nvhi-atsila-microservice/Jenkinsfile

pipeline {
    agent any

    parameters {
        booleanParam(
            name: 'FORCE_INFRASTRUCTURE_DEPLOY',
            defaultValue: false,
            description: 'Force infrastructure deployment regardless of change detection'
        )
        booleanParam(
            name: 'SKIP_QUALITY_GATES',
            defaultValue: false,
            description: 'Skip SonarQube quality gates (use with caution)'
        )
        booleanParam(
            name: 'DESTROY_INFRASTRUCTURE',
            defaultValue: false,
            description: 'Destroy all infrastructure (use with extreme caution)'
        )
    }

    environment {
        // Core configuration
        GITEA_REPO = 'https://code.jacquesingram.online/lenape/nvhi-atsila-microservice.git '
        GITEA_CREDS = '52ee0829-6e65-4951-925b-4186254c3f21'
        SONAR_HOST = 'https://sonar.jacquesingram.online '
        SONAR_TOKEN = credentials('sonar-token')
        // AWS configuration with ECR
        AWS_CRED_ID = 'aws-ci'
        AWS_ACCOUNT_ID = credentials('AWS_ACCOUNT_ID')
        AWS_REGION = 'us-east-2'
        ECR_REPO = 'nvhi-atsila-microservice'
        // Backend configuration
        TF_BACKEND_BUCKET = 'nvhi-atsila-tf-state'
        TF_BACKEND_PREFIX = 'ecs/terraform.tfstate'
        TF_DDB_TABLE = 'nvhi-atsila-locks'
        // Application variables
        TF_VAR_cluster_name = 'nvhi-atsila-cluster'
        TF_VAR_vpc_cidr = '10.0.0.0/16'
        TF_VAR_public_subnets = '10.0.1.0/24,10.0.2.0/24'
        TF_VAR_instance_type = 't2.micro'
        TF_VAR_key_pair_name = 'nvhi-atsila-deployer'
        TF_VAR_jenkins_ip_cidr = "0.0.0.0/0" // For demo; tighten in production
        TF_VAR_aws_region = "${AWS_REGION}"
        // Enhanced deployment tracking
        IMAGE_TAG = "v1.0.${BUILD_NUMBER}"
        // Initialize deployment type - will be set properly in stages
        DEPLOYMENT_TYPE = "APPLICATION"
        // Enterprise settings
        TF_IN_AUTOMATION = 'true'
        TF_INPUT = 'false'
        // Ansible configuration
        ANSIBLE_HOST_KEY_CHECKING = 'False'
        // Fix: Use relative path without leading slash
        ANSIBLE_CONFIG = './ansible/ansible.cfg'
        // Fix: Define log group as variable to avoid shell interpolation issues
        ECS_LOG_GROUP = "/ecs/nvhi-atsila-cluster"
    }

    stages {
        stage('Debug: Show File Structure') {
            steps {
                echo "📂 Current directory contents:"
                sh 'ls -la'
                echo "🔍 Full file tree:"
                sh 'find . -type f | sort'
            }
        }

        stage('Bootstrap Terraform Backend') {
            steps {
                script {
                    def tfBackendDir = "terraform-backend"

                    echo "🔐 Using Jenkins credentials to authenticate with AWS"
                    withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
                        echo "🔄 Checking/Bootstrapping Terraform backend..."
                        dir(tfBackendDir) {
                            def exitCode = sh(
                                script: """
                                    terraform init \\
                                        -var="aws_region=${TF_VAR_aws_region}" \\
                                        -var="backend_bucket_name=${TF_BACKEND_BUCKET}" \\
                                        -var="lock_table_name=${TF_DDB_TABLE}"
                                    terraform apply -auto-approve \\
                                        -var="aws_region=${TF_VAR_aws_region}" \\
                                        -var="backend_bucket_name=${TF_BACKEND_BUCKET}" \\
                                        -var="lock_table_name=${TF_DDB_TABLE}"
                                """,
                                returnStatus: true
                            )
                            
                            if (exitCode == 0) {
                                echo "✅ Terraform backend created successfully"
                            } else {
                                echo "⚠️ Terraform apply failed, checking if resources already exist..."
                                def bucketExists = sh(
                                    script: "aws s3api head-bucket --bucket ${TF_BACKEND_BUCKET} --region ${TF_VAR_aws_region} 2>/dev/null",
                                    returnStatus: true
                                ) == 0
                                def tableExists = sh(
                                    script: "aws dynamodb describe-table --table-name ${TF_DDB_TABLE} --region ${TF_VAR_aws_region} 2>/dev/null",
                                    returnStatus: true
                                ) == 0
                                
                                if (bucketExists && tableExists) {
                                    echo "✅ Terraform backend already exists - continuing..."
                                } else {
                                    echo "❌ Backend bootstrap failed and resources don't exist:"
                                    echo "   S3 Bucket exists: ${bucketExists}"
                                    echo "   DynamoDB Table exists: ${tableExists}"
                                    error("Manual intervention required.")
                                }
                            }
                        }
                    }
                }
            }
        }

        stage('Security Assessment & Checkout') {
            steps {
                checkout scm
                script {
                    // Check for infrastructure destruction first
                    if (params.DESTROY_INFRASTRUCTURE) {
                        env.DEPLOYMENT_TYPE = "DESTROY"
                        currentBuild.displayName = "DESTROY-${BUILD_NUMBER}"
                        echo "🚨 DESTROY MODE: Infrastructure destruction requested"
                        return
                    }

                    def infrastructureFiles = sh(
                        script: '''
                            if git rev-parse HEAD~1 >/dev/null 2>&1; then
                                git diff --name-only HEAD~1 2>/dev/null | grep -E "^terraform/" || echo "none"
                            else
                                echo "initial"
                            fi
                        ''',
                        returnStdout: true
                    ).trim()

                    // Check force parameter first - this overrides everything
                    if (params.FORCE_INFRASTRUCTURE_DEPLOY) {
                        env.DEPLOYMENT_TYPE = "INFRASTRUCTURE"
                        currentBuild.displayName = "INFRASTRUCTURE-FORCED-${BUILD_NUMBER}"
                        echo "🚨 FORCED: Infrastructure deployment requested via parameter"
                        echo "✅ Deployment type set to: INFRASTRUCTURE (forced)"
                    } else if (infrastructureFiles == "initial") {
                        env.DEPLOYMENT_TYPE = "INFRASTRUCTURE"
                        currentBuild.displayName = "INFRASTRUCTURE-INITIAL-${BUILD_NUMBER}"
                        echo "✅ First run detected. Deploying infrastructure."
                    } else if (infrastructureFiles != "none") {
                        env.DEPLOYMENT_TYPE = "INFRASTRUCTURE"
                        currentBuild.displayName = "INFRASTRUCTURE-CHANGED-${BUILD_NUMBER}"
                        echo "🚨 SECURITY NOTICE: Infrastructure changes detected - elevated permissions required"
                        echo "   Changed files: ${infrastructureFiles}"
                    } else {
                        env.DEPLOYMENT_TYPE = "APPLICATION"
                        currentBuild.displayName = "APPLICATION-${BUILD_NUMBER}"
                        echo "✅ SECURITY: Application-only deployment - using restricted permissions"
                    }

                    def gitCommit = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
                    def gitAuthor = sh(script: 'git log -1 --pretty=format:"%an"', returnStdout: true).trim()
                    currentBuild.description = "${env.DEPLOYMENT_TYPE} | ${env.IMAGE_TAG} | ${gitCommit.take(8)}"
                    echo "📋 SECURITY AUDIT TRAIL:"
                    echo "   • Deployment Type: ${env.DEPLOYMENT_TYPE}"
                    echo "   • Version: ${env.IMAGE_TAG}"
                    echo "   • Commit: ${gitCommit.take(8)}"
                    echo "   • Author: ${gitAuthor}"
                    echo "   • Container Registry: ECR (AWS-native, secure)"
                    echo "   • Architecture: Ansible-based deployment (enterprise security)"
                    echo "   • Security Model: Principle of Least Privilege"
                    echo "   • Timestamp: ${new Date()}"
                    echo "🔄 DEPLOYMENT TYPE CONFIRMATION: ${env.DEPLOYMENT_TYPE}"

                    writeFile file: 'deployment-audit.json', text: """{
                        "build_number": "${BUILD_NUMBER}",
                        "deployment_type": "${env.DEPLOYMENT_TYPE}",
                        "image_tag": "${env.IMAGE_TAG}",
                        "git_commit": "${gitCommit}",
                        "git_author": "${gitAuthor}",
                        "infrastructure_files_changed": "${infrastructureFiles}",
                        "container_registry": "ECR",
                        "architecture": "ansible_based_deployment",
                        "security_model": "principle_of_least_privilege",
                        "timestamp": "${new Date()}"
                    }"""
                    archiveArtifacts artifacts: 'deployment-audit.json', fingerprint: true
                }
            }
        }

        stage('Security & Quality Checks') {
            parallel {
                stage('SonarQube Security Analysis') {
                    when {
                        expression { !params.SKIP_QUALITY_GATES }
                    }
                    steps {
                        script {
                            def scannerHome = tool 'SonarQubeScanner'
                            withSonarQubeEnv('SonarQube') {
                                sh """
                                    echo "🔒 SECURITY: Running SonarQube security analysis..."
                                    ${scannerHome}/bin/sonar-scanner \\
                                      -Dsonar.projectKey=nvhi-atsila-microservice \\
                                      -Dsonar.sources=. \\
                                      -Dsonar.projectVersion=${BUILD_NUMBER} \\
                                      -Dsonar.login=${SONAR_TOKEN}
                                """
                            }
                            echo "✅ SECURITY: Code quality and security scan completed"
                        }
                    }
                }
                stage('Terraform Security Validation') {
                    steps {
                        script {
                            echo "🔒 SECURITY: Running Terraform security and validation checks..."
                            sh '''
                                echo "Validating Terraform configuration..."
                                cd terraform && terraform init -backend=false
                                terraform validate
                                echo "✅ Terraform validation passed"
                                echo "🔒 SECURITY: Checking infrastructure security compliance..."
                                grep -r "encrypted.*true" . --include="*.tf" && echo "✅ Encryption policies found" || echo "⚠️ Review encryption settings"
                                echo "🔒 SECURITY: Checking for open security groups..."
                                if grep -r "0.0.0.0/0" . --include="*.tf" --exclude-dir=".terraform" | grep -v "# Approved:"; then
                                    echo "⚠️ Review open access rules found"
                                else
                                    echo "✅ No unauthorized open access rules"
                                fi
                            '''
                            echo "✅ SECURITY: Infrastructure validation and security checks passed"
                        }
                    }
                }
            }
        }

        stage('Secure Container Build & Registry') {
            when {
                not { expression { env.DEPLOYMENT_TYPE == "DESTROY" } }
            }
            steps {
                withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
                    script {
                        echo "🔐 SECURITY: Using ECR for secure, AWS-native container registry"
                        
                        // Create ECR repository if it doesn't exist
                        echo "🔍 Checking/Creating ECR repository..."
                        sh """
                            if ! aws ecr describe-repositories --repository-names ${ECR_REPO} --region ${AWS_REGION} 2>/dev/null; then
                                echo "📦 Creating ECR repository: ${ECR_REPO}"
                                aws ecr create-repository --repository-name ${ECR_REPO} --region ${AWS_REGION}
                                echo "✅ ECR repository created successfully"
                            else
                                echo "✅ ECR repository already exists"
                            fi
                        """
                        
                        sh """
                            echo "🔐 Authenticating with ECR using temporary credentials..."
                            aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com
                        """
                        echo "🐳 Building secure container with metadata..."
                        sh """
                            docker build -t ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG} .
                            docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}
                            docker tag ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG} ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:latest
                            docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:latest
                        """
                        echo "✅ SECURITY: Container built and pushed to ECR successfully"
                        echo "   Image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}"
                        echo "   Registry: ECR (AWS-native, IAM-secured)"
                    }
                }
            }
        }

        stage('Infrastructure Readiness Check') {
            when {
                not { expression { env.DEPLOYMENT_TYPE == "DESTROY" } }
            }
            steps {
                withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
                    script {
                        echo "🔍 SECURITY: Checking if infrastructure is ready for deployment..."
                        echo "🔍 Current deployment type: ${env.DEPLOYMENT_TYPE}"
                        
                        // Only check readiness if deployment type is APPLICATION
                        if (env.DEPLOYMENT_TYPE == "APPLICATION") {
                            def serviceExists = sh(
                                script: """
                                    aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION} 2>/dev/null | grep -q 'ACTIVE' && echo 'true' || echo 'false'
                                """,
                                returnStdout: true
                            ).trim()
                            def instanceCount = sh(
                                script: """
                                    aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'length(containerInstanceArns)' --output text 2>/dev/null || echo '0'
                                """,
                                returnStdout: true
                            ).trim()
                            
                            echo "🔍 Service Exists: ${serviceExists}"
                            echo "🔍 Container Instances: ${instanceCount}"
                            
                            if (serviceExists == "false" || instanceCount == "0" || instanceCount == "null") {
                                echo "🚨 SECURITY NOTICE: Infrastructure not ready - forcing infrastructure deployment"
                                env.DEPLOYMENT_TYPE = "INFRASTRUCTURE"
                                currentBuild.displayName = "INFRASTRUCTURE-AUTO-${BUILD_NUMBER}"
                                currentBuild.description = "INFRASTRUCTURE (auto-detected) | ${env.IMAGE_TAG}"
                                echo "✅ Changed deployment type to: INFRASTRUCTURE"
                            }
                        } else {
                            echo "✅ Infrastructure deployment already scheduled - skipping readiness check"
                        }
                        
                        echo "📋 SECURITY: Infrastructure readiness assessment completed"
                        echo "   Final Deployment Type: ${env.DEPLOYMENT_TYPE}"
                    }
                }
            }
        }

        stage('Destroy Infrastructure') {
            when {
                expression { env.DEPLOYMENT_TYPE == "DESTROY" }
            }
            steps {
                withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
                    dir('terraform') {
                        script {
                            echo "🚨 DESTRUCTION: Destroying infrastructure..."
                            sh """
                                echo "🔄 Initializing Terraform with remote backend..."
                                terraform init \\
                                    -backend-config="bucket=${TF_BACKEND_BUCKET}" \\
                                    -backend-config="key=${TF_BACKEND_PREFIX}" \\
                                    -backend-config="region=${AWS_REGION}" \\
                                    -backend-config="dynamodb_table=${TF_DDB_TABLE}"
                                
                                echo "🔄 Planning infrastructure destruction..."
                                terraform plan -destroy \\
                                    -var="cluster_name=${TF_VAR_cluster_name}" \\
                                    -var="vpc_cidr=${TF_VAR_vpc_cidr}" \\
                                    -var="public_subnets=${TF_VAR_public_subnets}" \\
                                    -var="instance_type=${TF_VAR_instance_type}" \\
                                    -var="key_pair_name=${TF_VAR_key_pair_name}" \\
                                    -var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\
                                    -var="aws_region=${TF_VAR_aws_region}"
                                
                                echo "🔄 Destroying infrastructure..."
                                terraform destroy -auto-approve \\
                                    -var="cluster_name=${TF_VAR_cluster_name}" \\
                                    -var="vpc_cidr=${TF_VAR_vpc_cidr}" \\
                                    -var="public_subnets=${TF_VAR_public_subnets}" \\
                                    -var="instance_type=${TF_VAR_instance_type}" \\
                                    -var="key_pair_name=${TF_VAR_key_pair_name}" \\
                                    -var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\
                                    -var="aws_region=${TF_VAR_aws_region}"
                            """
                            echo "✅ Infrastructure destruction completed"
                        }
                    }
                }
            }
        }

        stage('Deploy Infrastructure') {
            when {
                anyOf {
                    expression { params.FORCE_INFRASTRUCTURE_DEPLOY == true }
                    expression { env.DEPLOYMENT_TYPE == "INFRASTRUCTURE" }
                }
            }
            steps {
                withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
                    dir('terraform') {
                        script {
                            echo "🔍 DEPLOYMENT: Force parameter = ${params.FORCE_INFRASTRUCTURE_DEPLOY}"
                            echo "🔍 DEPLOYMENT: Deployment type = ${env.DEPLOYMENT_TYPE}"
                            echo "🚨 SECURITY NOTICE: Infrastructure deployment requested"
                            echo "🏗️ ARCHITECTURE: Deploying ECS Cluster with Ansible-based deployment (enterprise security)"
                            echo "🔐 In production: This would require infrastructure-admin role"
                            echo "🚀 Attempting infrastructure deployment..."
                            
                            // Add error handling for Terraform operations
                            try {
                                sh """
                                    echo "🔄 Initializing Terraform with remote backend..."
                                    terraform init \\
                                        -backend-config="bucket=${TF_BACKEND_BUCKET}" \\
                                        -backend-config="key=${TF_BACKEND_PREFIX}" \\
                                        -backend-config="region=${AWS_REGION}" \\
                                        -backend-config="dynamodb_table=${TF_DDB_TABLE}"
                                    
                                    echo "🔄 Planning infrastructure changes..."
                                    terraform plan \\
                                        -var="cluster_name=${TF_VAR_cluster_name}" \\
                                        -var="vpc_cidr=${TF_VAR_vpc_cidr}" \\
                                        -var="public_subnets=${TF_VAR_public_subnets}" \\
                                        -var="instance_type=${TF_VAR_instance_type}" \\
                                        -var="key_pair_name=${TF_VAR_key_pair_name}" \\
                                        -var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\
                                        -var="aws_region=${TF_VAR_aws_region}"
                                    
                                    echo "🔄 Applying infrastructure changes..."
                                    terraform apply -auto-approve \\
                                        -var="cluster_name=${TF_VAR_cluster_name}" \\
                                        -var="vpc_cidr=${TF_VAR_vpc_cidr}" \\
                                        -var="public_subnets=${TF_VAR_public_subnets}" \\
                                        -var="instance_type=${TF_VAR_instance_type}" \\
                                        -var="key_pair_name=${TF_VAR_key_pair_name}" \\
                                        -var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\
                                        -var="aws_region=${TF_VAR_aws_region}"
                                """
                                echo "✅ SECURITY: Infrastructure deployment completed successfully"
                            } catch (Exception e) {
                                echo "❌ Infrastructure deployment failed: ${e.getMessage()}"
                                echo "📋 Checking current Terraform state..."
                                sh "terraform show || echo 'No state found'"
                                throw e
                            }
                        }
                    }
                }
            }
        }

        stage('Wait for ECS Agents') {
            when {
                anyOf {
                    expression { params.FORCE_INFRASTRUCTURE_DEPLOY == true }
                    expression { env.DEPLOYMENT_TYPE == "INFRASTRUCTURE" }
                }
            }
            steps {
                withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
                    script {
                        echo "⏳ Waiting for ECS agents to register with cluster..."
                        timeout(time: 10, unit: 'MINUTES') {
                            waitUntil {
                                def count = sh(
                                    script: """
                                        aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'length(containerInstanceArns)' --output text 2>/dev/null || echo '0'
                                    """,
                                    returnStdout: true
                                ).trim()
                                if (count != "0" && count != "null") {
                                    echo "✅ ECS agents registered: ${count} instance(s)"
                                    def activeCount = sh(
                                        script: """
                                            aws ecs describe-container-instances --cluster ${TF_VAR_cluster_name} --container-instances \$(aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'containerInstanceArns[*]' --output text) --region ${AWS_REGION} --query 'length(containerInstances[?status==\\`ACTIVE\\`])' --output text 2>/dev/null || echo '0'
                                        """,
                                        returnStdout: true
                                    ).trim()
                                    if (activeCount != "0" && activeCount != "null") {
                                        echo "✅ Active ECS instances: ${activeCount}"
                                        return true
                                    } else {
                                        echo "⏳ Waiting for instances to become ACTIVE..."
                                        sleep(20)
                                        return false
                                    }
                                } else {
                                    echo "⏳ No ECS agents registered yet..."
                                    sleep(20)
                                    return false
                                }
                            }
                        }
                    }
                }
            }
        }

        stage('Configure & Deploy Application with Ansible') {
            when {
                not { expression { env.DEPLOYMENT_TYPE == "DESTROY" } }
            }
            steps {
                script {
                    echo "🚀 ENTERPRISE: Deploying with Ansible (replacing SSM approach)"
                    
                    // Get infrastructure details from Terraform
                    def instanceId = ""
                    def publicIp = ""
                    def executionRoleArn = ""
                    
                    try {
                        instanceId = sh(
                            script: "cd terraform && terraform output -raw ecs_instance_id",
                            returnStdout: true
                        ).trim()
                        
                        publicIp = sh(
                            script: "cd terraform && terraform output -raw ecs_instance_public_ip", 
                            returnStdout: true
                        ).trim()
                        
                        executionRoleArn = sh(
                            script: "cd terraform && terraform output -raw ecs_task_execution_role_arn",
                            returnStdout: true
                        ).trim()
                        
                        echo "📍 Target Instance: ${instanceId} (${publicIp})"
                        echo "🔧 Execution Role: ${executionRoleArn}"
                    } catch (Exception e) {
                        echo "⚠️ Could not get all Terraform outputs: ${e.getMessage()}"
                        echo "⚠️ Some outputs may be missing, continuing with available data..."
                    }
                    
                    // Create Ansible working directory and files
                    sh "mkdir -p ansible/group_vars"
                    
                    // Fix: Create inventory with safer path handling
                    def inventoryContent = """[inventory_hosts]
ec2-instance ansible_host=${publicIp} ansible_user=ec2-user

[inventory_hosts:vars]
ansible_ssh_private_key_file=~/.ssh/id_rsa
ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o ServerAliveInterval=60'
ansible_python_interpreter=/usr/bin/python3
ansible_connection=ssh
ansible_ssh_retries=3
aws_region=${AWS_REGION}
"""
                    writeFile file: 'ansible/hosts', text: inventoryContent

                    // Fix: Create Ansible config with safer paths
                    def ansibleConfig = """[defaults]
inventory = hosts
host_key_checking = False
retry_files_enabled = False
gathering = smart
stdout_callback = yaml
timeout = 30
log_path = ./ansible.log

[ssh_connection]
ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=10
pipelining = True
"""
                    writeFile file: 'ansible/ansible.cfg', text: ansibleConfig

                    // Fix: Create group variables with safer variable handling
                    def groupVarsContent = """---
ecs_cluster_name: ${TF_VAR_cluster_name}
service_name: ${TF_VAR_cluster_name}-service  
task_family: ${TF_VAR_cluster_name}-task
container_name: ${ECR_REPO}
aws_region: ${AWS_REGION}
container_port: 8080
"""
                    writeFile file: 'ansible/group_vars/all.yml', text: groupVarsContent

                    // Test connectivity and execute deployment
                    withCredentials([
                        [$class: 'AmazonWebServicesCredentialsBinding', 
                         credentialsId: env.AWS_CRED_ID,
                         accessKeyVariable: 'AWS_ACCESS_KEY_ID',
                         secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
                    ]) {
                        // Fix: Use safer shell command construction
                        sh """
                            cd ansible
                            
                            # Set environment variables
                            export AWS_DEFAULT_REGION="${AWS_REGION}"
                            export ANSIBLE_HOST_KEY_CHECKING=False
                            export ANSIBLE_CONFIG="./ansible.cfg"
                            
                            # Wait for SSH connectivity
                            echo "🔍 Testing SSH connectivity to ${publicIp}..."
                            timeout 120 bash -c 'while ! nc -z ${publicIp} 22; do echo "Waiting for SSH..."; sleep 5; done'
                            
                            # Install Python dependencies if needed
                            pip3 install --user boto3 botocore jq > /dev/null 2>&1 || true
                            
                            # Test Ansible connectivity
                            echo "🔍 Testing Ansible connectivity..."
                            ansible inventory_hosts -m ping -i hosts -v
                            
                            if [ \$? -ne 0 ]; then
                                echo "❌ Ansible connectivity failed"
                                echo "Debugging SSH connection..."
                                ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${publicIp} 'echo "SSH test successful"' || {
                                    echo "SSH connection failed"
                                    exit 1
                                }
                                exit 1
                            fi
                            
                            echo "✅ Connectivity test passed"
                            
                            # Execute main deployment playbook
                            echo "🚀 Starting deployment..."
                            ansible-playbook configure_ecs.yml \\
                                -i hosts \\
                                -e "app_version=${IMAGE_TAG}" \\
                                -e "aws_account_id=${AWS_ACCOUNT_ID}" \\
                                -e "aws_region=${AWS_REGION}" \\
                                -e "task_execution_role_arn=${executionRoleArn}" \\
                                --timeout 600 \\
                                -v
                        """
                    }
                    
                    // Final verification
                    echo "🔍 Running final verification..."
                    sh """
                        echo "Testing application endpoint..."
                        for i in {1..10}; do
                            if curl -f -s "http://${publicIp}:8080/health"; then
                                echo "✅ Application health check passed"
                                break
                            else
                                echo "⏳ Health check attempt \$i/10..."
                                sleep 10
                            fi
                        done
                    """
                }
            }
            
            post {
                success {
                    script {
                        def publicIp = sh(
                            script: "cd terraform && terraform output -raw ecs_instance_public_ip",
                            returnStdout: true
                        ).trim()
                        
                        echo """
                        ========================================
                        🎉 DEPLOYMENT SUCCESSFUL!
                        ========================================
                        Application URL: http://${publicIp}:8080
                        Health Endpoint: http://${publicIp}:8080/health
                        Version: ${IMAGE_TAG}
                        Deployment Method: Ansible (Enterprise Security)
                        ========================================
                        """
                    }
                    
                    // Archive deployment artifacts
                    archiveArtifacts artifacts: 'ansible/ansible.log', allowEmptyArchive: true
                }
                
                failure {
                    echo "❌ DEPLOYMENT FAILED - Gathering debug information..."
                    
                    script {
                        // Fix: Use environment variable for log group to avoid shell interpolation issues
                        sh """
                            echo "=== ANSIBLE DEBUG INFORMATION ==="
                            cat ansible/ansible.log 2>/dev/null || echo "No Ansible log available"
                            
                            echo "=== ECS SERVICE STATUS ==="
                            aws ecs describe-services \\
                                --cluster "${TF_VAR_cluster_name}" \\
                                --services "${TF_VAR_cluster_name}-service" \\
                                --region "${AWS_REGION}" \\
                                --query 'services[0].{Status:status,Running:runningCount,Pending:pendingCount,Events:events[0:3]}' \\
                                --output json 2>/dev/null || echo "Could not get ECS service status"
                            
                            echo "=== ECS CLUSTER STATUS ==="
                            aws ecs describe-clusters \\
                                --clusters "${TF_VAR_cluster_name}" \\
                                --region "${AWS_REGION}" \\
                                --query 'clusters[0].{Status:status,ActiveInstances:activeContainerInstancesCount,Tasks:runningTasksCount}' \\
                                --output json 2>/dev/null || echo "Could not get ECS cluster status"
                            
                            echo "=== RECENT CONTAINER LOGS ==="
                            # Fix: Use environment variable for log group name
                            LATEST_STREAM=\$(aws logs describe-log-streams \\
                                --log-group-name "${ECS_LOG_GROUP}" \\
                                --region "${AWS_REGION}" \\
                                --order-by LastEventTime \\
                                --descending \\
                                --max-items 1 \\
                                --query 'logStreams[0].logStreamName' \\
                                --output text 2>/dev/null)
                            
                            if [ "\$LATEST_STREAM" != "None" ] && [ "\$LATEST_STREAM" != "" ]; then
                                echo "Latest log stream: \$LATEST_STREAM"
                                aws logs get-log-events \\
                                    --log-group-name "${ECS_LOG_GROUP}" \\
                                    --log-stream-name "\$LATEST_STREAM" \\
                                    --region "${AWS_REGION}" \\
                                    --start-from-head \\
                                    --query 'events[-20:].[timestamp,message]' \\
                                    --output table 2>/dev/null || echo "Could not retrieve logs"
                            else
                                echo "No log streams found"
                            fi
                        """
                    }
                    
                    // Offer rollback option
                    script {
                        try {
                            timeout(time: 5, unit: 'MINUTES') {
                                def rollbackChoice = input(
                                    message: 'Deployment failed. Would you like to rollback to the previous version?',
                                    parameters: [
                                        choice(choices: ['No', 'Yes'], description: 'Rollback?', name: 'ROLLBACK')
                                    ]
                                )
                                
                                if (rollbackChoice == 'Yes') {
                                    echo "🔄 Initiating automatic rollback..."
                                    withCredentials([
                                        [$class: 'AmazonWebServicesCredentialsBinding', 
                                         credentialsId: env.AWS_CRED_ID,
                                         accessKeyVariable: 'AWS_ACCESS_KEY_ID',
                                         secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
                                    ]) {
                                        sh """
                                            cd ansible
                                            ansible-playbook rollback.yml \\
                                                -e auto_rollback=true \\
                                                -v
                                        """
                                    }
                                }
                            }
                        } catch (Exception e) {
                            echo "Rollback prompt timed out or was cancelled"
                        }
                    }
                }
                
                always {
                    // Cleanup temporary files
                    sh """
                        rm -f ansible/hosts 2>/dev/null || true
                        rm -f ansible/ansible.cfg 2>/dev/null || true
                        rm -f ansible/group_vars/all.yml 2>/dev/null || true
                    """
                }
            }
        }
        
        stage('Verify Deployment') {
            when {
                not { expression { env.DEPLOYMENT_TYPE == "DESTROY" } }
            }
            steps {
                withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
                    script {
                        echo "🔍 VERIFICATION: Running comprehensive validation..."
                        
                        def publicIp = sh(
                            script: "cd terraform && terraform output -raw ecs_instance_public_ip",
                            returnStdout: true
                        ).trim()
                        
                        // Fix: Use safer URL construction and environment variables
                        sh """
                            echo "=== APPLICATION HEALTH CHECK ==="
                            curl -f -v "http://${publicIp}:8080/health"
                            
                            echo "=== ECS SERVICE VALIDATION ==="
                            aws ecs describe-services \\
                                --cluster "${TF_VAR_cluster_name}" \\
                                --services "${TF_VAR_cluster_name}-service" \\
                                --region "${AWS_REGION}" \\
                                --query 'services[0].{Status:status,TaskDefinition:taskDefinition,Running:runningCount,Desired:desiredCount}' \\
                                --output table
                            
                            echo "=== CONTAINER HEALTH CHECK ==="
                            # Check if containers are healthy
                            RUNNING_TASKS=\$(aws ecs list-tasks \\
                                --cluster "${TF_VAR_cluster_name}" \\
                                --service-name "${TF_VAR_cluster_name}-service" \\
                                --desired-status RUNNING \\
                                --region "${AWS_REGION}" \\
                                --query 'taskArns' \\
                                --output text)
                            
                            if [ -n "\$RUNNING_TASKS" ]; then
                                aws ecs describe-tasks \\
                                    --cluster "${TF_VAR_cluster_name}" \\
                                    --tasks \$RUNNING_TASKS \\
                                    --region "${AWS_REGION}" \\
                                    --query 'tasks[0].containers[0].{Name:name,Status:lastStatus,Health:healthStatus}' \\
                                    --output table
                            fi
                            
                            echo "=== LOG VALIDATION ==="
                            # Check for any errors in recent logs
                            LATEST_STREAM=\$(aws logs describe-log-streams \\
                                --log-group-name "${ECS_LOG_GROUP}" \\
                                --region "${AWS_REGION}" \\
                                --order-by LastEventTime \\
                                --descending \\
                                --max-items 1 \\
                                --query 'logStreams[0].logStreamName' \\
                                --output text 2>/dev/null)
                            
                            if [ "\$LATEST_STREAM" != "None" ] && [ "\$LATEST_STREAM" != "" ]; then
                                ERROR_COUNT=\$(aws logs get-log-events \\
                                    --log-group-name "${ECS_LOG_GROUP}" \\
                                    --log-stream-name "\$LATEST_STREAM" \\
                                    --region "${AWS_REGION}" \\
                                    --query 'events[?contains(message, \`ERROR\`) || contains(message, \`FATAL\`) || contains(message, \`Exception\`)].message' \\
                                    --output text | wc -l)
                                
                                if [ "\$ERROR_COUNT" -gt 0 ]; then
                                    echo "⚠️  Found \$ERROR_COUNT potential errors in logs - please review"
                                else
                                    echo "✅ No errors found in recent application logs"
                                fi
                            fi
                            
                            echo "✅ All validation checks completed successfully"
                        """
                        
                        // Update build description with URL
                        currentBuild.description = "${currentBuild.description} | URL: http://${publicIp}:8080"
                        
                        echo "✅ VERIFICATION: Deployment verification completed"
                    }
                }
            }
        }
    }
    
    post {
        always {
            script {
                echo "🧹 CLEANUP: Performing post-build cleanup..."
                
                // Archive deployment artifacts
                try {
                    archiveArtifacts artifacts: 'deployment-audit.json,task-definition.json', allowEmptyArchive: true
                } catch (Exception e) {
                    echo "⚠️ Could not archive artifacts: ${e.getMessage()}"
                }
                
                // Clean up Docker images to save space
                sh '''
                    echo "🧹 Cleaning up Docker images..."
                    docker system prune -f || echo "Docker cleanup failed"
                '''
                
                echo "📊 SUMMARY: Build completed"
                echo "   Build Number: ${BUILD_NUMBER}"
                echo "   Image Tag: ${IMAGE_TAG}"
                echo "   Deployment Type: ${env.DEPLOYMENT_TYPE}"
                echo "   Status: ${currentBuild.currentResult}"
            }
        }
        
        success {
            script {
                if (env.DEPLOYMENT_TYPE == "DESTROY") {
                    echo "🎉 SUCCESS: Infrastructure destroyed successfully!"
                } else {
                    echo "🎉 SUCCESS: Deployment completed successfully!"
                    echo "   Version ${IMAGE_TAG} deployed to ECS cluster ${TF_VAR_cluster_name}"
                    
                    // Get application URL for success message
                    def appUrl = ""
                    try {
                        appUrl = sh(
                            script: "cd terraform && terraform output -raw ecs_instance_public_ip 2>/dev/null || echo 'unknown'",
                            returnStdout: true
                        ).trim()
                        if (appUrl != "unknown" && appUrl != "") {
                            echo "🌐 Application available at: http://${appUrl}:8080"
                            echo "🏥 Health check: http://${appUrl}:8080/health"
                        }
                    } catch (Exception e) {
                        echo "⚠️ Could not determine application URL"
                    }
                }
            }
        }
        
        failure {
            script {
                echo "❌ FAILURE: Deployment failed"
                echo "   Check the logs above for error details"
                
                // Try to get some debug information
                try {
                    withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
                        echo "🔍 DEBUG: Checking ECS cluster status..."
                        sh """
                            aws ecs describe-clusters --clusters ${TF_VAR_cluster_name} --region ${AWS_REGION} || echo "Cluster check failed"
                            aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} || echo "Instance list failed"
                        """
                    }
                } catch (Exception e) {
                    echo "⚠️ Could not get debug information: ${e.getMessage()}"
                }
            }
        }
        
        unstable {
            script {
                echo "⚠️ UNSTABLE: Build completed with warnings"
            }
        }
    }
}