From 07f0cc3c117ad98f5e3c7e20b3d725ef9264b65e Mon Sep 17 00:00:00 2001 From: lenape Date: Mon, 14 Jul 2025 05:08:31 +0000 Subject: [PATCH] automated terminal push --- Jenkinsfile | 1717 ++++++++++++++++++++------------------------------- 1 file changed, 667 insertions(+), 1050 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3c4ed97..72e2d5e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,1068 +1,685 @@ pipeline { - agent any - - parameters { - booleanParam( - name: 'FORCE_INFRASTRUCTURE_DEPLOY', - defaultValue: false, - description: 'Force infrastructure deployment regardless of change detection' - ) - booleanParam( - name: 'SKIP_QUALITY_GATES', - defaultValue: false, - description: 'Skip SonarQube quality gates (use with caution)' - ) - } - - environment { - // Core configuration - GITEA_REPO = 'https://code.jacquesingram.online/lenape/nvhi-atsila-microservice.git' - GITEA_CREDS = '52ee0829-6e65-4951-925b-4186254c3f21' - SONAR_HOST = 'https://sonar.jacquesingram.online' - SONAR_TOKEN = credentials('sonar-token') + agent any - // AWS configuration with ECR (optimal choice) - AWS_CRED_ID = 'aws-ci' - AWS_ACCOUNT_ID = credentials('AWS_ACCOUNT_ID') - AWS_REGION = 'us-east-2' - ECR_REPO = 'nvhi-atsila-microservice' - - // Backend configuration - TF_BACKEND_BUCKET = 'nvhi-atsila-tf-state' - TF_BACKEND_PREFIX = 'ecs/terraform.tfstate' - TF_DDB_TABLE = 'nvhi-atsila-locks' - - // Application variables - TF_VAR_cluster_name = 'nvhi-atsila-cluster' - TF_VAR_vpc_cidr = '10.0.0.0/16' - TF_VAR_public_subnets = '10.0.1.0/24,10.0.2.0/24' - TF_VAR_instance_type = 't2.micro' - TF_VAR_key_pair_name = 'nvhi-atsila-deployer' - TF_VAR_jenkins_ip_cidr = "38.110.1.139/32" - TF_VAR_aws_region = "${AWS_REGION}" - - // Enhanced deployment tracking - IMAGE_TAG = "v1.0.${BUILD_NUMBER}" - DEPLOYMENT_TYPE = "APPLICATION" - - // Enterprise settings - TF_IN_AUTOMATION = 'true' - TF_INPUT = 'false' - } - - stages { - stage('Security Assessment & Checkout') { - steps { - checkout scm - - script { - // Enhanced change detection with security implications - def infrastructureFiles = sh( - script: ''' - if git rev-parse HEAD~1 >/dev/null 2>&1; then - git diff --name-only HEAD~1 2>/dev/null | grep -E "(terraform/|infrastructure)" || echo "none" - else - echo "none" - fi - ''', - returnStdout: true - ).trim() - - // Fixed: Proper deployment type assignment - if (params.FORCE_INFRASTRUCTURE_DEPLOY) { - env.DEPLOYMENT_TYPE = "INFRASTRUCTURE" - echo "🚨 FORCED: Infrastructure deployment requested via parameter" - } else if (infrastructureFiles != "none") { - env.DEPLOYMENT_TYPE = "INFRASTRUCTURE" - echo "🚨 SECURITY NOTICE: Infrastructure changes detected - elevated permissions required" - echo " Changed files: ${infrastructureFiles}" - } else { - env.DEPLOYMENT_TYPE = "APPLICATION" - echo "✅ SECURITY: Application-only deployment - using restricted permissions" - } - - // Professional audit logging - def gitCommit = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - def gitAuthor = sh(returnStdout: true, script: 'git log -1 --pretty=format:"%an"').trim() - - currentBuild.description = "${env.DEPLOYMENT_TYPE} | ${env.IMAGE_TAG} | ${gitCommit.take(8)}" - - echo "📋 SECURITY AUDIT TRAIL:" - echo " • Deployment Type: ${env.DEPLOYMENT_TYPE}" - echo " • Version: ${env.IMAGE_TAG}" - echo " • Commit: ${gitCommit.take(8)}" - echo " • Author: ${gitAuthor}" - echo " • Container Registry: ECR (AWS-native, secure)" - echo " • Architecture: SSM-based ECS access (secure, keyless)" - echo " • Security Model: Principle of Least Privilege" - echo " • Timestamp: ${new Date()}" - - // Archive deployment metadata for compliance - writeFile file: 'deployment-audit.json', text: """{ - "build_number": "${BUILD_NUMBER}", - "deployment_type": "${env.DEPLOYMENT_TYPE}", - "image_tag": "${env.IMAGE_TAG}", - "git_commit": "${gitCommit}", - "git_author": "${gitAuthor}", - "infrastructure_files_changed": "${infrastructureFiles}", - "container_registry": "ECR", - "architecture": "ssm_based_ecs_access", - "security_model": "principle_of_least_privilege", - "timestamp": "${new Date()}" - }""" - - archiveArtifacts artifacts: 'deployment-audit.json', fingerprint: true - } - } + parameters { + booleanParam( + name: 'FORCE_INFRASTRUCTURE_DEPLOY', + defaultValue: false, + description: 'Force infrastructure deployment regardless of change detection' + ) + booleanParam( + name: 'SKIP_QUALITY_GATES', + defaultValue: false, + description: 'Skip SonarQube quality gates (use with caution)' + ) } - stage('Security & Quality Checks') { - parallel { - stage('SonarQube Security Analysis') { - when { - expression { !params.SKIP_QUALITY_GATES } - } - steps { - script { - def scannerHome = tool 'SonarQubeScanner' - withSonarQubeEnv('SonarQube') { - sh """ - ${scannerHome}/bin/sonar-scanner \\ - -Dsonar.projectKey=nvhi-atsila-microservice \\ - -Dsonar.sources=. \\ - -Dsonar.projectVersion=${BUILD_NUMBER} - """ - } - echo "✅ SECURITY: Code quality and security scan completed" - } - } - } - - stage('Terraform Security Validation') { - steps { - script { - echo "🔒 SECURITY: Running Terraform security and validation checks..." - sh ''' - echo "Validating Terraform configuration..." - cd terraform && terraform init -backend=false - terraform validate - echo "✅ Terraform validation passed" - - echo "🔒 SECURITY: Checking infrastructure security compliance..." - grep -r "encrypted.*true" . --include="*.tf" && echo "✅ Encryption policies found" || echo "⚠️ Review encryption settings" - echo "🔒 SECURITY: Checking for open security groups..." - if grep -r "0.0.0.0/0" . --include="*.tf" --exclude-dir=.terraform | grep -v "# Approved:"; then - echo "⚠️ Review open access rules found" - else - echo "✅ No unauthorized open access rules" - fi - ''' - echo "✅ SECURITY: Infrastructure validation and security checks passed" - } - } - } - } + environment { + // Core configuration + GITEA_REPO = 'https://code.jacquesingram.online/lenape/nvhi-atsila-microservice.git' + GITEA_CREDS = '52ee0829-6e65-4951-925b-4186254c3f21' + SONAR_HOST = 'https://sonar.jacquesingram.online' + SONAR_TOKEN = credentials('sonar-token') + // AWS configuration with ECR + AWS_CRED_ID = 'aws-ci' + AWS_ACCOUNT_ID = credentials('AWS_ACCOUNT_ID') + AWS_REGION = 'us-east-2' + ECR_REPO = 'nvhi-atsila-microservice' + // Backend configuration + TF_BACKEND_BUCKET = 'nvhi-atsila-tf-state' + TF_BACKEND_PREFIX = 'ecs/terraform.tfstate' + TF_DDB_TABLE = 'nvhi-atsila-locks' + // Application variables + TF_VAR_cluster_name = 'nvhi-atsila-cluster' + TF_VAR_vpc_cidr = '10.0.0.0/16' + TF_VAR_public_subnets = '10.0.1.0/24,10.0.2.0/24' + TF_VAR_instance_type = 't2.micro' + TF_VAR_key_pair_name = 'nvhi-atsila-deployer' + TF_VAR_jenkins_ip_cidr = "38.110.1.139/32" + TF_VAR_aws_region = "${AWS_REGION}" + // Enhanced deployment tracking + IMAGE_TAG = "v1.0.${BUILD_NUMBER}" + DEPLOYMENT_TYPE = "APPLICATION" + + // Enterprise settings + TF_IN_AUTOMATION = 'true' + TF_INPUT = 'false' } - stage('Secure Container Build & Registry') { - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - script { - echo "🔐 SECURITY: Using ECR for secure, AWS-native container registry" - - // Get AWS Account ID safely for credential masking - def awsAccountId = env.AWS_ACCOUNT_ID - - sh ''' - echo "🔐 Authenticating with ECR using temporary credentials..." - aws ecr get-login-password --region $AWS_REGION \\ - | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com - ''' - - echo "🐳 Building secure container with metadata..." - def img = docker.build("${awsAccountId}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}") - - echo "📤 Pushing to secure ECR registry..." - img.push() - img.push("latest") // Also tag as latest for convenience - - echo "✅ SECURITY: Container built and pushed to ECR successfully" - echo " Image: ${awsAccountId}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}" - echo " Registry: ECR (AWS-native, IAM-secured)" - } - } - } - } + stages { + stage('Security Assessment & Checkout') { + steps { + checkout scm - stage('Bootstrap Backend Infrastructure') { - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - dir('terraform-backend') { - script { - echo "🏗️ SECURITY: Checking backend infrastructure with proper permissions..." - - def backendExists = sh( - script: ''' - echo "🔒 SECURITY: Verifying backend infrastructure access..." - if aws s3api head-bucket --bucket $TF_BACKEND_BUCKET 2>/dev/null && \\ - aws dynamodb describe-table --table-name $TF_DDB_TABLE 2>/dev/null; then - echo "true" - else - echo "false" - fi - ''', - returnStdout: true - ).trim() - - if (backendExists == "false") { - echo "🚨 SECURITY WARNING: Backend infrastructure missing - would require elevated permissions" - echo "📦 In production: This would trigger infrastructure admin role" - echo "🔐 For demo: Simulating backend creation check..." - } else { - echo "✅ SECURITY: Backend infrastructure verified and accessible" - echo " S3 Bucket: ${TF_BACKEND_BUCKET}" - echo " DynamoDB Table: ${TF_DDB_TABLE}" - } - } - } - } - } - } - - stage('Infrastructure State Management') { - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - dir('terraform') { - script { - if (env.DEPLOYMENT_TYPE == "INFRASTRUCTURE") { - echo "🔐 SECURITY ESCALATION: Infrastructure changes detected" - echo " In production: Would use infrastructure-admin role" - echo " Current: Using deployment role with limited permissions" - } else { - echo "✅ SECURITY: Application deployment using restricted permissions" - } - - echo "🔄 Managing infrastructure state for optimal ECS deployment..." - - sh ''' - # Initialize with secure remote backend - terraform init \\ - -backend-config="bucket=${TF_BACKEND_BUCKET}" \\ - -backend-config="key=${TF_BACKEND_PREFIX}" \\ - -backend-config="region=${AWS_REGION}" \\ - -backend-config="dynamodb_table=${TF_DDB_TABLE}" - - # Enterprise security: Backup current state - echo "💾 Creating secure state backup for disaster recovery..." - terraform state pull > "secure-state-backup-${BUILD_NUMBER}.json" - - # Upgrade providers to handle version conflicts - echo "⬆️ Upgrading providers with security validation..." - terraform init -upgrade - - # Create execution plan for ECS infrastructure - echo "📋 Creating infrastructure plan for ECS cluster and networking..." - terraform plan -out="secure-tfplan-${BUILD_NUMBER}" \\ - -var="cluster_name=${TF_VAR_cluster_name}" \\ - -var="vpc_cidr=${TF_VAR_vpc_cidr}" \\ - -var="public_subnets=${TF_VAR_public_subnets}" \\ - -var="instance_type=${TF_VAR_instance_type}" \\ - -var="key_pair_name=${TF_VAR_key_pair_name}" \\ - -var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\ - -var="aws_region=${TF_VAR_aws_region}" - ''' - - // Archive state backup and plan for enterprise audit trail - archiveArtifacts artifacts: "secure-state-backup-${BUILD_NUMBER}.json,secure-tfplan-${BUILD_NUMBER}", - fingerprint: true, - allowEmptyArchive: false - - echo "✅ SECURITY: Infrastructure planning completed with full audit trail" - } - } - } - } - } - - stage('Infrastructure Readiness Check') { - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - script { - echo "🔍 SECURITY: Checking if infrastructure is ready for deployment..." - - // Check if parameter forces infrastructure deployment - if (params.FORCE_INFRASTRUCTURE_DEPLOY) { - echo "🚨 FORCED: Infrastructure deployment requested via parameter" - env.DEPLOYMENT_TYPE = "INFRASTRUCTURE" - currentBuild.description = "INFRASTRUCTURE (forced) | ${env.IMAGE_TAG}" - return - } - - // Check if ECS service exists - def serviceExists = sh( - script: ''' - if aws ecs describe-services --cluster nvhi-atsila-cluster --services nvhi-atsila-cluster-service --region us-east-2 2>/dev/null | grep -q "ACTIVE"; then - echo "true" - else - echo "false" - fi - ''', - returnStdout: true - ).trim() - - // Check container instance count - def instanceCount = sh( - script: """ - aws ecs list-container-instances \\ - --cluster ${TF_VAR_cluster_name} \\ - --region ${AWS_REGION} \\ - --query 'length(containerInstanceArns)' \\ - --output text 2>/dev/null || echo "0" - """, - returnStdout: true - ).trim() - - if (serviceExists == "false" || instanceCount == "0" || instanceCount == "null") { - echo "🚨 SECURITY NOTICE: Infrastructure not ready - forcing deployment" - echo " Service Exists: ${serviceExists}" - echo " Container Instances: ${instanceCount}" - echo " This is normal for first deployment or after infrastructure cleanup" - - env.DEPLOYMENT_TYPE = "INFRASTRUCTURE" - currentBuild.description = "INFRASTRUCTURE (auto-detected) | ${env.IMAGE_TAG}" - } - - echo "📋 SECURITY: Infrastructure readiness assessment completed" - echo " ECS Service Exists: ${serviceExists}" - echo " Container Instances: ${instanceCount}" - echo " Final Deployment Type: ${env.DEPLOYMENT_TYPE}" - echo " Security Decision: ${env.DEPLOYMENT_TYPE == 'INFRASTRUCTURE' ? 'Infrastructure deployment required' : 'Application-only deployment'}" - } - } - } - } - - stage('Deploy Infrastructure') { - when { - expression { env.DEPLOYMENT_TYPE == "INFRASTRUCTURE" } - } - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - dir('terraform') { - script { - echo "🚨 SECURITY NOTICE: Infrastructure deployment requested" - echo "🏗️ ARCHITECTURE: Deploying ECS Cluster with SSM access (secure, keyless)" - echo "🔐 In production: This would require infrastructure-admin role" - echo "🚀 Attempting infrastructure deployment..." - - sh """ - # Apply the planned changes with security logging - echo "🔄 Applying infrastructure changes..." - terraform apply "secure-tfplan-${BUILD_NUMBER}" - - # Verify no unexpected drift with security validation - echo "🔍 Verifying deployment consistency and security compliance..." - terraform plan -detailed-exitcode \\ - -var="cluster_name=${TF_VAR_cluster_name}" \\ - -var="vpc_cidr=${TF_VAR_vpc_cidr}" \\ - -var="public_subnets=${TF_VAR_public_subnets}" \\ - -var="instance_type=${TF_VAR_instance_type}" \\ - -var="key_pair_name=${TF_VAR_key_pair_name}" \\ - -var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\ - -var="aws_region=${TF_VAR_aws_region}" || echo "⚠️ Infrastructure drift detected - review required" - """ - - echo "✅ SECURITY: Infrastructure deployment completed with compliance verification" - } - } - } - } - } - - stage('Wait for ECS Agents') { - when { - expression { env.DEPLOYMENT_TYPE == "INFRASTRUCTURE" } - } - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - script { - echo "⏳ Waiting for ECS agents to register with cluster..." - timeout(time: 5, unit: 'MINUTES') { - waitUntil { - def count = sh( - script: """ - aws ecs list-container-instances \\ - --cluster ${TF_VAR_cluster_name} \\ - --region ${AWS_REGION} \\ - --query 'length(containerInstanceArns)' \\ - --output text 2>/dev/null || echo "0" - """, - returnStdout: true - ).trim() - - if (count != "0" && count != "null") { - echo "✅ ECS agents registered: ${count} instance(s)" - - // Verify the instances are actually ACTIVE - def activeCount = sh( - script: """ - aws ecs describe-container-instances \\ - --cluster ${TF_VAR_cluster_name} \\ - --container-instances \$(aws ecs list-container-instances \\ - --cluster ${TF_VAR_cluster_name} \\ - --region ${AWS_REGION} \\ - --query 'containerInstanceArns[*]' \\ - --output text) \\ - --region ${AWS_REGION} \\ - --query 'length(containerInstances[?status==\`ACTIVE\`])' \\ - --output text 2>/dev/null || echo "0" - """, - returnStdout: true - ).trim() - - if (activeCount != "0" && activeCount != "null") { - echo "✅ Active ECS instances: ${activeCount}" - return true - } else { - echo "⏳ Waiting for instances to become ACTIVE..." - sleep(20) - return false - } - } else { - echo "⏳ No ECS agents registered yet..." - sleep(20) - return false - } - } - } - } - } - } - } - - stage('Configure & Deploy Application') { - parallel { - stage('Configure EC2 Instance via SSM') { - when { - expression { - // Only run if we have EC2 instances from terraform - def hasInstances = false - try { - def instanceId = sh( - script: "cd terraform && terraform output -raw ecs_instance_id 2>/dev/null || echo ''", - returnStdout: true - ).trim() - hasInstances = (instanceId != "" && instanceId != "null") - } catch (Exception e) { - echo "⚠️ No instances to configure: ${e.getMessage()}" - } - return hasInstances - } - } - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - script { - echo "🔧 ENTERPRISE: Configuring EC2 instance via SSM (no SSH required)" - - // Get instance ID from Terraform output - def instanceId = "" - def ec2_ip = "" - - try { - sh "test -d terraform || (echo 'Terraform directory not found' && exit 1)" - instanceId = sh( - script: "cd terraform && terraform output -raw ecs_instance_id", - returnStdout: true - ).trim() - ec2_ip = sh( - script: "cd terraform && terraform output -raw ecs_instance_public_ip", - returnStdout: true - ).trim() - } catch (Exception e) { - echo "⚠️ Could not get instance details: ${e.getMessage()}" - echo "⚠️ Skipping SSM configuration - no instances available" - return - } - - echo "📍 Target Instance: ${instanceId} (${ec2_ip})" - - // Wait for SSM agent to be ready - echo "⏳ Waiting for SSM agent to be ready..." - timeout(time: 10, unit: 'MINUTES') { - waitUntil { - script { - def ssmStatus = sh( - script: """ - aws ssm describe-instance-information \\ - --filters "Key=InstanceIds,Values=${instanceId}" \\ - --region ${AWS_REGION} \\ - --query 'InstanceInformationList[0].PingStatus' \\ - --output text 2>/dev/null || echo "Offline" - """, + script { + def infrastructureFiles = sh( + script: ''' + if git rev-parse HEAD~1 >/dev/null 2>&1; then + git diff --name-only HEAD~1 2>/dev/null | grep -E "(terraform/|infrastructure)" || echo "none" + else + echo "none" + fi + ''', returnStdout: true - ).trim() - - if (ssmStatus == "Online") { - echo "✅ SSM agent is online" - return true - } else { - echo "⏳ Waiting for SSM agent... (Status: ${ssmStatus})" - sleep(20) - return false - } - } - } - } - - // Configure ECS agent via SSM - echo "🔧 Configuring ECS agent via SSM..." - def commandId = sh( - script: """ - aws ssm send-command \\ - --instance-ids ${instanceId} \\ - --document-name "AWS-RunShellScript" \\ - --parameters 'commands=[ - "echo \\"=== ECS Configuration via SSM ===\\"", - "echo \\"Cluster: ${TF_VAR_cluster_name}\\"", - "echo \\"Time: \$(date)\\"", - "echo \\"Instance: \$(hostname)\\"", - "sudo systemctl status ecs --no-pager", - "sudo systemctl status docker --no-pager", - "curl -s http://localhost:51678/v1/metadata || echo \\"ECS agent not ready\\"", - "sudo systemctl restart ecs", - "sleep 15", - "sudo systemctl status ecs --no-pager", - "curl -s http://localhost:51678/v1/metadata || echo \\"ECS agent still starting\\"", - "echo \\"=== Configuration completed ===\\"" - ]' \\ - --region ${AWS_REGION} \\ - --output text \\ - --query 'Command.CommandId' - """, - returnStdout: true - ).trim() - - echo "📋 SSM Command ID: ${commandId}" - - // Wait for command completion - echo "⏳ Waiting for SSM command completion..." - sh """ - aws ssm wait command-executed \\ - --command-id ${commandId} \\ - --instance-id ${instanceId} \\ - --region ${AWS_REGION} - """ - - // Get command output - echo "📋 SSM Command Output:" - sh """ - aws ssm get-command-invocation \\ - --command-id ${commandId} \\ - --instance-id ${instanceId} \\ - --region ${AWS_REGION} \\ - --query 'StandardOutputContent' \\ - --output text - """ - - // Check for any errors - def commandStatus = sh( - script: """ - aws ssm get-command-invocation \\ - --command-id ${commandId} \\ - --instance-id ${instanceId} \\ - --region ${AWS_REGION} \\ - --query 'Status' \\ - --output text - """, - returnStdout: true - ).trim() - - if (commandStatus != "Success") { - echo "❌ SSM Command failed with status: ${commandStatus}" - // Get error output - sh """ - echo "Error Output:" - aws ssm get-command-invocation \\ - --command-id ${commandId} \\ - --instance-id ${instanceId} \\ - --region ${AWS_REGION} \\ - --query 'StandardErrorContent' \\ - --output text - """ - // Don't fail the build, continue with deployment - echo "⚠️ SSM configuration had issues but continuing with deployment" - } - - echo "✅ ENTERPRISE: EC2 instance configured via SSM successfully" - echo """ - 🔐 SSM Session Manager Access: - - To connect to the instance for troubleshooting: - - aws ssm start-session \\ - --target ${instanceId} \\ - --region ${AWS_REGION} - - Instance ID: ${instanceId} - Instance IP: ${ec2_ip} - """ - } - } - } - } - - stage('Deploy Application to ECS') { - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - script { - echo "🚢 SECURITY: Deploying application to ECS with ECR integration..." - echo "📦 ARCHITECTURE: Using ECR for secure, AWS-native container delivery" - echo "🔐 User can register tasks and update services, but cannot modify infrastructure" - - // Get AWS Account ID and Git commit safely for credential masking - def awsAccountId = env.AWS_ACCOUNT_ID - def gitCommitHash = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() - - // Create task definition JSON safely - def taskDefinition = """[{ - "name":"health-workload", - "image":"${awsAccountId}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}", - "essential":true, - "memory":512, - "portMappings":[{"containerPort":8080,"hostPort":8080}], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/${TF_VAR_cluster_name}", - "awslogs-region": "${AWS_REGION}", - "awslogs-stream-prefix": "ecs" - } - }, - "environment": [ - {"name": "BUILD_NUMBER", "value": "${BUILD_NUMBER}"}, - {"name": "GIT_COMMIT", "value": "${gitCommitHash}"}, - {"name": "DEPLOYMENT_TIME", "value": "${new Date().format('yyyy-MM-dd HH:mm:ss')}"}, - {"name": "CONTAINER_REGISTRY", "value": "ECR"}, - {"name": "ARCHITECTURE", "value": "ssm_based_ecs_access"} - ] - }]""" - - // Write task definition to file for safety - writeFile file: 'task-definition.json', text: taskDefinition - - sh """ - # Register new task definition with ECR image - aws ecs register-task-definition \\ - --family ${TF_VAR_cluster_name} \\ - --network-mode bridge \\ - --container-definitions file://task-definition.json \\ - --region ${AWS_REGION} - """ - - // Check if service exists and create/update accordingly - def serviceExists = sh( - script: """ - if aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION} 2>/dev/null | grep -q "ACTIVE"; then - echo "true" - else - echo "false" - fi - """, - returnStdout: true - ).trim() - - if (serviceExists == "false") { - echo "🆕 Creating new ECS service..." - sh """ - # Create new service since it doesn't exist - aws ecs create-service \\ - --cluster ${TF_VAR_cluster_name} \\ - --service-name ${TF_VAR_cluster_name}-service \\ - --task-definition ${TF_VAR_cluster_name} \\ - --desired-count 1 \\ - --launch-type EC2 \\ - --region ${AWS_REGION} - """ - } else { - echo "🔄 Updating existing ECS service..." - sh """ - # Update existing service - aws ecs update-service \\ - --cluster ${TF_VAR_cluster_name} \\ - --service ${TF_VAR_cluster_name}-service \\ - --force-new-deployment \\ - --region ${AWS_REGION} - """ - } - - // Wait for deployment with better timeout handling - echo "⏳ Waiting for secure service deployment to stabilize..." - timeout(time: 10, unit: 'MINUTES') { - try { - sh """ - aws ecs wait services-stable \\ - --cluster ${TF_VAR_cluster_name} \\ - --services ${TF_VAR_cluster_name}-service \\ - --region ${AWS_REGION} - """ - echo "✅ SECURITY: Application deployed successfully with ECR integration" - } catch (Exception e) { - echo "⚠️ Service deployment timeout - checking status..." - - // Get service status even if wait times out - def serviceStatus = sh( - script: """ - aws ecs describe-services \\ - --cluster ${TF_VAR_cluster_name} \\ - --services ${TF_VAR_cluster_name}-service \\ - --region ${AWS_REGION} \\ - --query 'services[0].deployments[0].rolloutState' \\ - --output text - """, - returnStdout: true ).trim() - - echo "Service deployment state: ${serviceStatus}" - - if (serviceStatus == "COMPLETED") { - echo "✅ Deployment completed successfully despite timeout" + + if (params.FORCE_INFRASTRUCTURE_DEPLOY) { + env.DEPLOYMENT_TYPE = "INFRASTRUCTURE" + echo "🚨 FORCED: Infrastructure deployment requested via parameter" + } else if (infrastructureFiles != "none") { + env.DEPLOYMENT_TYPE = "INFRASTRUCTURE" + echo "🚨 SECURITY NOTICE: Infrastructure changes detected - elevated permissions required" + echo " Changed files: ${infrastructureFiles}" } else { - echo "⚠️ Deployment still in progress: ${serviceStatus}" - echo "⚠️ Check ECS console for more details" + env.DEPLOYMENT_TYPE = "APPLICATION" + echo "✅ SECURITY: Application-only deployment - using restricted permissions" } - } - } - - echo "✅ SECURITY: Application deployment initiated successfully" - } - } - } - } - } - } - stage('Post-Deployment Validation') { - parallel { - stage('Health Check') { - steps { - script { - def ec2_ip = "" - try { - ec2_ip = sh( - script: "cd terraform && terraform output -raw ecs_instance_public_ip 2>/dev/null || echo 'unknown'", - returnStdout: true - ).trim() - } catch (Exception e) { - echo "⚠️ Could not get EC2 IP for health check" - ec2_ip = "unknown" - } - - echo "🏥 SECURITY: Running health validation on http://${ec2_ip}:8080/health" - echo "🔗 ARCHITECTURE: Direct access with SSM management (secure and efficient)" - - if (ec2_ip != "unknown" && ec2_ip != "" && ec2_ip != "null") { - timeout(time: 5, unit: 'MINUTES') { - waitUntil { + def gitCommit = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() + def gitAuthor = sh(returnStdout: true, script: 'git log -1 --pretty=format:"%an"').trim() + + currentBuild.description = "${env.DEPLOYMENT_TYPE} | ${env.IMAGE_TAG} | ${gitCommit.take(8)}" + + echo "📋 SECURITY AUDIT TRAIL:" + echo " • Deployment Type: ${env.DEPLOYMENT_TYPE}" + echo " • Version: ${env.IMAGE_TAG}" + echo " • Commit: ${gitCommit.take(8)}" + echo " • Author: ${gitAuthor}" + echo " • Container Registry: ECR (AWS-native, secure)" + echo " • Architecture: SSM-based ECS access (secure, keyless)" + echo " • Security Model: Principle of Least Privilege" + echo " • Timestamp: ${new Date()}" + + writeFile file: 'deployment-audit.json', text: """{ + "build_number": "${BUILD_NUMBER}", + "deployment_type": "${env.DEPLOYMENT_TYPE}", + "image_tag": "${env.IMAGE_TAG}", + "git_commit": "${gitCommit}", + "git_author": "${gitAuthor}", + "infrastructure_files_changed": "${infrastructureFiles}", + "container_registry": "ECR", + "architecture": "ssm_based_ecs_access", + "security_model": "principle_of_least_privilege", + "timestamp": "${new Date()}" + }""" + + archiveArtifacts artifacts: 'deployment-audit.json', fingerprint: true + } + } + } + + stage('Security & Quality Checks') { + parallel { + stage('SonarQube Security Analysis') { + when { + expression { !params.SKIP_QUALITY_GATES } + } + steps { + script { + def scannerHome = tool 'SonarQubeScanner' + withSonarQubeEnv('SonarQube') { + sh "${scannerHome}/bin/sonar-scanner -Dsonar.projectKey=nvhi-atsila-microservice -Dsonar.sources=. -Dsonar.projectVersion=${BUILD_NUMBER}" + } + echo "✅ SECURITY: Code quality and security scan completed" + } + } + } + + stage('Terraform Security Validation') { + steps { + script { + echo "🔒 SECURITY: Running Terraform security and validation checks..." + sh ''' + echo "Validating Terraform configuration..." + cd terraform && terraform init -backend=false + terraform validate + echo "✅ Terraform validation passed" + + echo "🔒 SECURITY: Checking infrastructure security compliance..." + grep -r "encrypted.*true" . --include="*.tf" && echo "✅ Encryption policies found" || echo "⚠️ Review encryption settings" + echo "🔒 SECURITY: Checking for open security groups..." + if grep -r "0.0.0.0/0" . --include="*.tf" --exclude-dir=.terraform | grep -v "# Approved:"; then + echo "⚠️ Review open access rules found" + else + echo "✅ No unauthorized open access rules" + fi + ''' + echo "✅ SECURITY: Infrastructure validation and security checks passed" + } + } + } + } + } + + stage('Secure Container Build & Registry') { + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { script { - def response = sh( - script: "curl -s -o /dev/null -w '%{http_code}' http://${ec2_ip}:8080/health || echo '000'", - returnStdout: true - ).trim() - - echo "🔍 SECURITY: Health check response: ${response}" - if (response == "200") { - echo "✅ SECURITY: Application health check passed - service is secure and operational" - return true - } else { - echo "⏳ SECURITY: Waiting for application to be ready (${response})" - sleep(10) - return false - } - } - } - } - } else { - echo "⚠️ SECURITY: EC2 IP not available - skipping health check" - echo "⚠️ This may happen if infrastructure wasn't deployed in this run" - } - } - } - } - - stage('Smoke Tests') { - steps { - script { - echo "💨 SECURITY: Running comprehensive smoke tests..." - def ec2_ip = "" - try { - sh "test -d terraform || (echo 'Terraform directory not found' && exit 1)" - ec2_ip = sh( - script: "cd terraform && terraform output -raw ecs_instance_public_ip 2>/dev/null || echo 'unknown'", - returnStdout: true - ).trim() - } catch (Exception e) { - echo "⚠️ Could not get EC2 IP for smoke tests: ${e.getMessage()}" - ec2_ip = "unknown" - } - - if (ec2_ip != "unknown" && ec2_ip != "" && ec2_ip != "null") { - sh """ - echo "🔒 SECURITY: Testing application endpoints and security headers..." - curl -I http://${ec2_ip}:8080/health || echo "Service may still be starting..." - - echo "🔍 SECURITY: Verifying ECR image deployment and metadata..." - curl -s http://${ec2_ip}:8080/health || echo "Application responding" - - echo "🛡️ SECURITY: Validating network security and access controls..." - echo " Testing only allowed ports are accessible" - echo " Verifying ECR integration working correctly" - echo " Confirming SSM-based access security model" - - echo "✅ SECURITY: All smoke tests and security validations passed" - """ - } else { - echo "⚠️ SECURITY: EC2 IP not available - skipping smoke tests" - echo "⚠️ This may happen if infrastructure wasn't deployed in this run" - } - } - } - } - - stage('Container Instance Validation') { - steps { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - script { - echo "🔍 Validating ECS container instances..." - - def instanceArns = sh( - script: """ - aws ecs list-container-instances \\ - --cluster ${TF_VAR_cluster_name} \\ - --region ${AWS_REGION} \\ - --query 'containerInstanceArns' \\ - --output json 2>/dev/null || echo '[]' - """, - returnStdout: true - ).trim() - - if (instanceArns != '[]' && instanceArns != 'null') { - echo "📋 Container Instances Found:" - sh """ - aws ecs describe-container-instances \\ - --cluster ${TF_VAR_cluster_name} \\ - --container-instances ${instanceArns} \\ - --region ${AWS_REGION} \\ - --query 'containerInstances[*].[containerInstanceArn,status,runningTasksCount,pendingTasksCount]' \\ - --output table - """ - - // Check running tasks - def runningTasks = sh( - script: """ - aws ecs list-tasks \\ - --cluster ${TF_VAR_cluster_name} \\ - --desired-status RUNNING \\ - --region ${AWS_REGION} \\ - --query 'length(taskArns)' \\ - --output text 2>/dev/null || echo "0" - """, - returnStdout: true - ).trim() - - echo "📊 Running Tasks: ${runningTasks}" - - if (runningTasks == "0") { - echo "⚠️ No running tasks found - deployment may still be in progress" - } else { - echo "✅ ${runningTasks} task(s) running successfully" - } - } else { - echo "⚠️ No container instances found in cluster" - } - } - } - } - } - } - } - } + echo "🔐 SECURITY: Using ECR for secure, AWS-native container registry" + def awsAccountId = env.AWS_ACCOUNT_ID - post { - always { - script { - echo "📊 SECURITY: Collecting deployment artifacts and performing secure cleanup..." - - // Archive comprehensive deployment artifacts for audit - archiveArtifacts artifacts: 'deployment-audit.json,task-definition.json', allowEmptyArchive: true - - // Secure workspace cleanup - cleanWs(deleteDirs: true, notFailBuild: true) - - echo "🔒 SECURITY: Deployment artifacts archived and workspace securely cleaned" - } - } - - success { - script { - def ec2_ip = "" - def instanceId = "" - def gitCommitHash = "" - def runningTasks = "0" - - try { - sh "test -d terraform || echo 'Terraform directory not found'" - ec2_ip = sh( - script: "cd terraform && terraform output -raw ecs_instance_public_ip 2>/dev/null || echo 'unknown'", - returnStdout: true - ).trim() - instanceId = sh( - script: "cd terraform && terraform output -raw ecs_instance_id 2>/dev/null || echo 'unknown'", - returnStdout: true - ).trim() - gitCommitHash = sh(script: 'git rev-parse HEAD 2>/dev/null || echo "unknown"', returnStdout: true).trim().take(8) - - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - runningTasks = sh( - script: """ - aws ecs list-tasks \\ - --cluster ${TF_VAR_cluster_name} \\ - --desired-status RUNNING \\ - --region ${AWS_REGION} \\ - --query 'length(taskArns)' \\ - --output text 2>/dev/null || echo "0" - """, - returnStdout: true - ).trim() - } - } catch (Exception e) { - ec2_ip = "unknown" - instanceId = "unknown" - gitCommitHash = "unknown" + sh """ + echo "🔐 Authenticating with ECR using temporary credentials..." + aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com + """ + + echo "🐳 Building secure container with metadata..." + def img = docker.build("${awsAccountId}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}") + + echo "📤 Pushing to secure ECR registry..." + img.push() + img.push("latest") + + echo "✅ SECURITY: Container built and pushed to ECR successfully" + echo " Image: ${awsAccountId}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}" + echo " Registry: ECR (AWS-native, IAM-secured)" + } + } + } } - - echo "🎉 SSM-BASED SECURE DEPLOYMENT SUCCESSFUL!" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "📋 DEPLOYMENT SUMMARY (SSM-OPTIMIZED FOR SECURITY):" - echo " • Container Registry: ECR (AWS-native, secure) ✅" - echo " • Architecture: SSM-based ECS access (keyless, secure) ✅" - echo " • Infrastructure: ECS + VPC + Security Groups (SSM-enabled) ✅" - echo " • Application Version: ${IMAGE_TAG}" - echo " • Application URL: http://${ec2_ip}:8080" - echo " • Health Endpoint: http://${ec2_ip}:8080/health" - echo " • ECR Image: ${env.AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}" - echo " • Security Compliance: ✅ PASSED (No SSH keys required)" - echo " • Git Commit: ${gitCommitHash}" - echo " • Deployment Type: ${env.DEPLOYMENT_TYPE}" - echo " • Running Tasks: ${runningTasks}" - echo " • Instance Access: SSM Session Manager (${instanceId}) ✅" - echo " • Cost Optimization: Free tier friendly ✅" - echo "" - echo "🔐 SSM ACCESS COMMANDS:" - echo " • Connect to instance: aws ssm start-session --target ${instanceId} --region ${AWS_REGION}" - echo " • View ECS logs: aws logs tail /ecs/${TF_VAR_cluster_name} --follow --region ${AWS_REGION}" - echo " • Check task status: aws ecs list-tasks --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION}" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - currentBuild.description = "✅ ${env.DEPLOYMENT_TYPE} | ECR | ${IMAGE_TAG} | ${ec2_ip}" - } - } - - failure { - script { - def failureStage = "" - def instanceCount = "unknown" - - try { - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: env.AWS_CRED_ID - ]]) { - instanceCount = sh( - script: """ - aws ecs list-container-instances \\ - --cluster ${TF_VAR_cluster_name} \\ - --region ${AWS_REGION} \\ - --query 'length(containerInstanceArns)' \\ - --output text 2>/dev/null || echo "unknown" - """, - returnStdout: true - ).trim() - } - } catch (Exception e) { - // Ignore errors in post section + + stage('Infrastructure Readiness Check') { + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { + script { + echo "🔍 SECURITY: Checking if infrastructure is ready for deployment..." + + if (params.FORCE_INFRASTRUCTURE_DEPLOY) { + echo "🚨 FORCED: Infrastructure deployment requested via parameter" + env.DEPLOYMENT_TYPE = "INFRASTRUCTURE" + currentBuild.description = "INFRASTRUCTURE (forced) | ${env.IMAGE_TAG}" + } + + def serviceExists = sh( + script: "aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION} 2>/dev/null | grep -q 'ACTIVE' && echo 'true' || echo 'false'", + returnStdout: true + ).trim() + + def instanceCount = sh( + script: "aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'length(containerInstanceArns)' --output text 2>/dev/null || echo '0'", + returnStdout: true + ).trim() + + if (serviceExists == "false" || instanceCount == "0" || instanceCount == "null") { + echo "🚨 SECURITY NOTICE: Infrastructure not ready - forcing deployment" + echo " Service Exists: ${serviceExists}" + echo " Container Instances: ${instanceCount}" + env.DEPLOYMENT_TYPE = "INFRASTRUCTURE" + currentBuild.description = "INFRASTRUCTURE (auto-detected) | ${env.IMAGE_TAG}" + } + + echo "📋 SECURITY: Infrastructure readiness assessment completed" + echo " ECS Service Exists: ${serviceExists}" + echo " Container Instances: ${instanceCount}" + echo " Final Deployment Type: ${env.DEPLOYMENT_TYPE}" + } + } + } } - - echo "❌ DEPLOYMENT FAILED!" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "🔍 FAILURE ANALYSIS:" - echo " • Deployment Type: ${env.DEPLOYMENT_TYPE}" - echo " • Container Instances Available: ${instanceCount}" - echo " • Failed Stage: ${env.STAGE_NAME ?: 'Unknown'}" - echo "" - echo "📋 COMMON ISSUES AND SOLUTIONS:" - echo "" - if (instanceCount == "0" || instanceCount == "unknown") { - echo "❌ NO CONTAINER INSTANCES FOUND" - echo " • Run with FORCE_INFRASTRUCTURE_DEPLOY=true parameter" - echo " • Or check if EC2 instances are terminating unexpectedly" - echo " • Verify IAM role has required ECS permissions" + + stage('Deploy Infrastructure') { + when { + expression { env.DEPLOYMENT_TYPE == "INFRASTRUCTURE" } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { + dir('terraform') { + script { + echo "🚨 SECURITY NOTICE: Infrastructure deployment requested" + echo "🏗️ ARCHITECTURE: Deploying ECS Cluster with SSM access (secure, keyless)" + echo "🔐 In production: This would require infrastructure-admin role" + echo "🚀 Attempting infrastructure deployment..." + + sh """ + echo "🔄 Applying infrastructure changes..." + terraform apply "secure-tfplan-${BUILD_NUMBER}" + echo "🔍 Verifying deployment consistency and security compliance..." + terraform plan -detailed-exitcode -var="cluster_name=${TF_VAR_cluster_name}" -var="vpc_cidr=${TF_VAR_vpc_cidr}" -var="public_subnets=${TF_VAR_public_subnets}" -var="instance_type=${TF_VAR_instance_type}" -var="key_pair_name=${TF_VAR_key_pair_name}" -var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" -var="aws_region=${TF_VAR_aws_region}" || echo "⚠️ Infrastructure drift detected - review required" + """ + + echo "✅ SECURITY: Infrastructure deployment completed with compliance verification" + } + } + } + } + } + + stage('Wait for ECS Agents') { + when { + expression { env.DEPLOYMENT_TYPE == "INFRASTRUCTURE" } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { + script { + echo "⏳ Waiting for ECS agents to register with cluster..." + timeout(time: 5, unit: 'MINUTES') { + waitUntil { + def count = sh( + script: "aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'length(containerInstanceArns)' --output text 2>/dev/null || echo '0'", + returnStdout: true + ).trim() + + if (count != "0" && count != "null") { + echo "✅ ECS agents registered: ${count} instance(s)" + def activeCount = sh( + script: "aws ecs describe-container-instances --cluster ${TF_VAR_cluster_name} --container-instances \$(aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'containerInstanceArns[*]' --output text) --region ${AWS_REGION} --query 'length(containerInstances[?status==\\`ACTIVE\\`])' --output text 2>/dev/null || echo '0'", + returnStdout: true + ).trim() + + if (activeCount != "0" && activeCount != "null") { + echo "✅ Active ECS instances: ${activeCount}" + return true + } else { + echo "⏳ Waiting for instances to become ACTIVE..." + sleep(20) + return false + } + } else { + echo "⏳ No ECS agents registered yet..." + sleep(20) + return false + } + } + } + } + } + } + } + + stage('Configure & Deploy Application') { + parallel { + stage('Configure EC2 Instance via SSM') { + when { + expression { + def hasInstances = false + try { + def instanceId = sh( + script: "cd terraform && terraform output -raw ecs_instance_id 2>/dev/null || echo ''", + returnStdout: true + ).trim() + hasInstances = (instanceId != "" && instanceId != "null") + } catch (Exception e) { + echo "⚠️ No instances to configure: ${e.getMessage()}" + } + return hasInstances + } + } + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { + script { + echo "🔧 ENTERPRISE: Configuring EC2 instance via SSM (no SSH required)" + def instanceId = "" + def ec2_ip = "" + + try { + sh "test -d terraform || (echo 'Terraform directory not found' && exit 1)" + instanceId = sh( + script: "cd terraform && terraform output -raw ecs_instance_id", + returnStdout: true + ).trim() + ec2_ip = sh( + script: "cd terraform && terraform output -raw ecs_instance_public_ip", + returnStdout: true + ).trim() + } catch (Exception e) { + echo "⚠️ Could not get instance details: ${e.getMessage()}" + echo "⚠️ Skipping SSM configuration - no instances available" + return + } + + echo "📍 Target Instance: ${instanceId} (${ec2_ip})" + echo "⏳ Waiting for SSM agent to be ready..." + timeout(time: 10, unit: 'MINUTES') { + waitUntil { + def ssmStatus = sh( + script: "aws ssm describe-instance-information --filters \"Key=InstanceIds,Values=${instanceId}\" --region ${AWS_REGION} --query 'InstanceInformationList[0].PingStatus' --output text 2>/dev/null || echo 'Offline'", + returnStdout: true + ).trim() + + if (ssmStatus == "Online") { + echo "✅ SSM agent is online" + return true + } else { + echo "⏳ Waiting for SSM agent... (Status: ${ssmStatus})" + sleep(20) + return false + } + } + } + + echo "🔧 Configuring ECS agent via SSM..." + def commandId = sh( + script: "aws ssm send-command --instance-ids ${instanceId} --document-name \"AWS-RunShellScript\" --parameters 'commands=[\"echo === ECS Configuration via SSM ===\",\"echo Cluster: ${TF_VAR_cluster_name}\",\"echo Time: \$(date)\",\"echo Instance: \$(hostname)\",\"sudo systemctl status ecs --no-pager\",\"sudo systemctl status docker --no-pager\",\"curl -s http://localhost:51678/v1/metadata || echo \\\"ECS agent not ready\\\"\",\"sudo systemctl restart ecs\",\"sleep 15\",\"sudo systemctl status ecs --no-pager\",\"curl -s http://localhost:51678/v1/metadata || echo \\\"ECS agent still starting\\\"\",\"echo === Configuration completed ===\"]' --region ${AWS_REGION} --output text --query 'Command.CommandId'", + returnStdout: true + ).trim() + + echo "📋 SSM Command ID: ${commandId}" + echo "⏳ Waiting for SSM command completion..." + sh "aws ssm wait command-executed --command-id ${commandId} --instance-id ${instanceId} --region ${AWS_REGION}" + + echo "📋 SSM Command Output:" + sh "aws ssm get-command-invocation --command-id ${commandId} --instance-id ${instanceId} --region ${AWS_REGION} --query 'StandardOutputContent' --output text" + + def commandStatus = sh( + script: "aws ssm get-command-invocation --command-id ${commandId} --instance-id ${instanceId} --region ${AWS_REGION} --query 'Status' --output text", + returnStdout: true + ).trim() + + if (commandStatus != "Success") { + echo "❌ SSM Command failed with status: ${commandStatus}" + sh "echo 'Error Output:'; aws ssm get-command-invocation --command-id ${commandId} --instance-id ${instanceId} --region ${AWS_REGION} --query 'StandardErrorContent' --output text" + echo "⚠️ SSM configuration had issues but continuing with deployment" + } + + echo "✅ ENTERPRISE: EC2 instance configured via SSM successfully" + echo "🔐 SSM Session Manager Access: To connect to the instance for troubleshooting, use: aws ssm start-session --target ${instanceId} --region ${AWS_REGION}" + } + } + } + } + + stage('Deploy Application to ECS') { + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { + script { + echo "🚢 SECURITY: Deploying application to ECS with ECR integration..." + def awsAccountId = env.AWS_ACCOUNT_ID + def gitCommitHash = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() + + def taskDefinition = """[{ + "name": "health-workload", + "image": "${awsAccountId}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}", + "essential": true, + "memory": 512, + "portMappings": [{"containerPort": 8080, "hostPort": 8080}], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/${TF_VAR_cluster_name}", + "awslogs-region": "${AWS_REGION}", + "awslogs-stream-prefix": "ecs" + } + }, + "environment": [ + {"name": "BUILD_NUMBER", "value": "${BUILD_NUMBER}"}, + {"name": "GIT_COMMIT", "value": "${gitCommitHash}"}, + {"name": "DEPLOYMENT_TIME", "value": "${new Date().format('yyyy-MM-dd HH:mm:ss')}"}, + {"name": "CONTAINER_REGISTRY", "value": "ECR"}, + {"name": "ARCHITECTURE", "value": "ssm_based_ecs_access"} + ] + }]""" + + writeFile file: 'task-definition.json', text: taskDefinition + + sh "aws ecs register-task-definition --family ${TF_VAR_cluster_name} --network-mode bridge --container-definitions file://task-definition.json --region ${AWS_REGION}" + + def serviceExists = sh( + script: "aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION} 2>/dev/null | grep -q 'ACTIVE' && echo 'true' || echo 'false'", + returnStdout: true + ).trim() + + if (serviceExists == "false") { + echo "🆕 Creating new ECS service..." + sh "aws ecs create-service --cluster ${TF_VAR_cluster_name} --service-name ${TF_VAR_cluster_name}-service --task-definition ${TF_VAR_cluster_name} --desired-count 1 --launch-type EC2 --region ${AWS_REGION}" + } else { + echo "🔄 Updating existing ECS service..." + sh "aws ecs update-service --cluster ${TF_VAR_cluster_name} --service ${TF_VAR_cluster_name}-service --force-new-deployment --region ${AWS_REGION}" + } + + echo "⏳ Waiting for secure service deployment to stabilize..." + timeout(time: 10, unit: 'MINUTES') { + try { + sh "aws ecs wait services-stable --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION}" + echo "✅ SECURITY: Application deployed successfully with ECR integration" + } catch (Exception e) { + echo "⚠️ Service deployment timeout - checking status..." + def serviceStatus = sh( + script: "aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION} --query 'services[0].deployments[0].rolloutState' --output text", + returnStdout: true + ).trim() + echo "Service deployment state: ${serviceStatus}" + if (serviceStatus == "COMPLETED") { + echo "✅ Deployment completed successfully despite timeout" + } else { + echo "⚠️ Deployment still in progress: ${serviceStatus}" + echo "⚠️ Check ECS console for more details" + } + } + } + echo "✅ SECURITY: Application deployment initiated successfully" + } + } + } + } + } + } + + stage('Post-Deployment Validation') { + parallel { + stage('Health Check') { + steps { + script { + def ec2_ip = "" + try { + ec2_ip = sh( + script: "cd terraform && terraform output -raw ecs_instance_public_ip 2>/dev/null || echo 'unknown'", + returnStdout: true + ).trim() + } catch (Exception e) { + echo "⚠️ Could not get EC2 IP for health check" + ec2_ip = "unknown" + } + + echo "🏥 SECURITY: Running health validation on http://${ec2_ip}:8080/health" + echo "🔗 ARCHITECTURE: Direct access with SSM management (secure and efficient)" + + if (ec2_ip != "unknown" && ec2_ip != "" && ec2_ip != "null") { + timeout(time: 5, unit: 'MINUTES') { + waitUntil { + def response = sh( + script: "curl -s -o /dev/null -w '%{http_code}' http://${ec2_ip}:8080/health || echo '000'", + returnStdout: true + ).trim() + echo "🔍 SECURITY: Health check response: ${response}" + if (response == "200") { + echo "✅ SECURITY: Application health check passed - service is secure and operational" + return true + } else { + echo "⏳ SECURITY: Waiting for application to be ready (${response})" + sleep(10) + return false + } + } + } + } else { + echo "⚠️ SECURITY: EC2 IP not available - skipping health check" + echo "⚠️ This may happen if infrastructure wasn't deployed in this run" + } + } + } + } + + stage('Smoke Tests') { + steps { + script { + echo "💨 SECURITY: Running comprehensive smoke tests..." + def ec2_ip = "" + try { + sh "test -d terraform || (echo 'Terraform directory not found' && exit 1)" + ec2_ip = sh( + script: "cd terraform && terraform output -raw ecs_instance_public_ip 2>/dev/null || echo 'unknown'", + returnStdout: true + ).trim() + } catch (Exception e) { + echo "⚠️ Could not get EC2 IP for smoke tests: ${e.getMessage()}" + ec2_ip = "unknown" + } + + if (ec2_ip != "unknown" && ec2_ip != "" && ec2_ip != "null") { + sh """ + echo "🔒 SECURITY: Testing application endpoints and security headers..." + curl -I http://${ec2_ip}:8080/health || echo "Service may still be starting..." + echo "🔍 SECURITY: Verifying ECR image deployment and metadata..." + curl -s http://${ec2_ip}:8080/health || echo "Application responding" + echo "🛡️ SECURITY: Validating network security and access controls..." + echo "✅ SECURITY: All smoke tests and security validations passed" + """ + } else { + echo "⚠️ SECURITY: EC2 IP not available - skipping smoke tests" + echo "⚠️ This may happen if infrastructure wasn't deployed in this run" + } + } + } + } + + stage('Container Instance Validation') { + steps { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { + script { + echo "🔍 Validating ECS container instances..." + def instanceArns = sh( + script: "aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'containerInstanceArns' --output json 2>/dev/null || echo '[]'", + returnStdout: true + ).trim() + + if (instanceArns != '[]' && instanceArns != 'null') { + echo "📋 Container Instances Found:" + sh "aws ecs describe-container-instances --cluster ${TF_VAR_cluster_name} --container-instances ${instanceArns} --region ${AWS_REGION} --query 'containerInstances[*].[containerInstanceArn,status,runningTasksCount,pendingTasksCount]' --output table" + + def runningTasks = sh( + script: "aws ecs list-tasks --cluster ${TF_VAR_cluster_name} --desired-status RUNNING --region ${AWS_REGION} --query 'length(taskArns)' --output text 2>/dev/null || echo '0'", + returnStdout: true + ).trim() + echo "📊 Running Tasks: ${runningTasks}" + if (runningTasks == "0") { + echo "⚠️ No running tasks found - deployment may still be in progress" + } else { + echo "✅ ${runningTasks} task(s) running successfully" + } + } else { + echo "⚠️ No container instances found in cluster" + } + } + } + } + } + } } - echo "" - echo "🔧 TROUBLESHOOTING COMMANDS:" - echo " • Check ECS cluster: aws ecs describe-clusters --clusters ${TF_VAR_cluster_name} --region ${AWS_REGION}" - echo " • List instances: aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION}" - echo " • Check services: aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION}" - echo " • View ECS events: aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION} --query 'services[0].events[:5]'" - echo "" - echo "💡 RECOVERY OPTIONS:" - echo " 1. Force infrastructure deployment: Run pipeline with FORCE_INFRASTRUCTURE_DEPLOY=true" - echo " 2. Check AWS Console for any manually terminated resources" - echo " 3. Review Terraform state: May need to run 'terraform refresh' locally" - echo " 4. Check CloudWatch logs: /ecs/${TF_VAR_cluster_name}" - echo "" - echo "📁 ARTIFACTS AVAILABLE:" - echo " • Security audit trail: deployment-audit.json" - echo " • State backup: secure-state-backup-${BUILD_NUMBER}.json" - echo " • Terraform plan: secure-tfplan-${BUILD_NUMBER}" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - currentBuild.description = "❌ Failed: ${env.DEPLOYMENT_TYPE} | ${env.STAGE_NAME}" - - // Optional: Send notification - // mail to: 'devops-team@example.com', - // subject: "Jenkins Build Failed: ${env.JOB_NAME} #${env.BUILD_NUMBER}", - // body: "The build failed at stage: ${env.STAGE_NAME}\n\nCheck the console output: ${env.BUILD_URL}" - } } - } -} \ No newline at end of file + + post { + always { + script { + echo "📊 SECURITY: Collecting deployment artifacts and performing secure cleanup..." + archiveArtifacts artifacts: 'deployment-audit.json,task-definition.json', allowEmptyArchive: true + cleanWs(deleteDirs: true, notFailBuild: true) + echo "🔒 SECURITY: Deployment artifacts archived and workspace securely cleaned" + } + } + + success { + script { + def ec2_ip = "" + def instanceId = "" + def gitCommitHash = "" + def runningTasks = "0" + + try { + sh "test -d terraform || echo 'Terraform directory not found'" + ec2_ip = sh( + script: "cd terraform && terraform output -raw ecs_instance_public_ip 2>/dev/null || echo 'unknown'", + returnStdout: true + ).trim() + instanceId = sh( + script: "cd terraform && terraform output -raw ecs_instance_id 2>/dev/null || echo 'unknown'", + returnStdout: true + ).trim() + gitCommitHash = sh(script: 'git rev-parse HEAD 2>/dev/null || echo "unknown"', returnStdout: true).trim().take(8) + + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { + runningTasks = sh( + script: "aws ecs list-tasks --cluster ${TF_VAR_cluster_name} --desired-status RUNNING --region ${AWS_REGION} --query 'length(taskArns)' --output text 2>/dev/null || echo '0'", + returnStdout: true + ).trim() + } + } catch (Exception e) { + ec2_ip = "unknown" + instanceId = "unknown" + gitCommitHash = "unknown" + } + + echo "🎉 SSM-BASED SECURE DEPLOYMENT SUCCESSFUL!" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "📋 DEPLOYMENT SUMMARY (SSM-OPTIMIZED FOR SECURITY):" + echo " • Container Registry: ECR (AWS-native, secure) ✅" + echo " • Architecture: SSM-based ECS access (keyless, secure) ✅" + echo " • Infrastructure: ECS + VPC + Security Groups (SSM-enabled) ✅" + echo " • Application Version: ${IMAGE_TAG}" + echo " • Application URL: http://${ec2_ip}:8080" + echo " • Health Endpoint: http://${ec2_ip}:8080/health" + echo " • ECR Image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}" + echo " • Security Compliance: ✅ PASSED (No SSH keys required)" + echo " • Git Commit: ${gitCommitHash}" + echo " • Deployment Type: ${env.DEPLOYMENT_TYPE}" + echo " • Running Tasks: ${runningTasks}" + echo " • Instance Access: SSM Session Manager (${instanceId}) ✅" + echo " • Cost Optimization: Free tier friendly ✅" + echo "🔐 SSM ACCESS COMMANDS:" + echo " • Connect to instance: aws ssm start-session --target ${instanceId} --region ${AWS_REGION}" + echo " • View ECS logs: aws logs tail /ecs/${TF_VAR_cluster_name} --follow --region ${AWS_REGION}" + echo " • Check task status: aws ecs list-tasks --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION}" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + currentBuild.description = "✅ ${env.DEPLOYMENT_TYPE} | ECR | ${IMAGE_TAG} | ${ec2_ip}" + } + } + + failure { + script { + def failureStage = "" + def instanceCount = "unknown" + + try { + withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) { + instanceCount = sh( + script: "aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'length(containerInstanceArns)' --output text 2>/dev/null || echo 'unknown'", + returnStdout: true + ).trim() + } + } catch (Exception e) { + // Ignore errors in post section + } + + echo "❌ DEPLOYMENT FAILED!" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "🔍 FAILURE ANALYSIS:" + echo " • Deployment Type: ${env.DEPLOYMENT_TYPE}" + echo " • Container Instances Available: ${instanceCount}" + echo " • Failed Stage: ${env.STAGE_NAME ?: 'Unknown'}" + echo "📋 COMMON ISSUES AND SOLUTIONS:" + if (instanceCount == "0" || instanceCount == "unknown") { + echo "❌ NO CONTAINER INSTANCES FOUND" + echo " • Run with FORCE_INFRASTRUCTURE_DEPLOY=true parameter" + echo " • Or check if EC2 instances are terminating unexpectedly" + echo " • Verify IAM role has required ECS permissions" + } + echo "🔧 TROUBLESHOOTING COMMANDS:" + echo " • Check ECS cluster: aws ecs describe-clusters --clusters ${TF_VAR_cluster_name} --region ${AWS_REGION}" + echo " • List instances: aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION}" + echo " • Check services: aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION}" + echo "💡 RECOVERY OPTIONS:" + echo " 1. Force infrastructure deployment: Run pipeline with FORCE_INFRASTRUCTURE_DEPLOY=true" + echo " 2. Check AWS Console for any manually terminated resources" + echo " 3. Review Terraform state: May need to run 'terraform refresh' locally" + echo " 4. Check CloudWatch logs: /ecs/${TF_VAR_cluster_name}" + echo "📁 ARTIFACTS AVAILABLE:" + echo " • Security audit trail: deployment-audit.json" + echo " • State backup: secure-state-backup-${BUILD_NUMBER}.json" + echo " • Terraform plan: secure-tfplan-${BUILD_NUMBER}" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + currentBuild.description = "❌ Failed: ${env.DEPLOYMENT_TYPE} | ${env.STAGE_NAME}" + } + } + } +}