921 lines
48 KiB
Groovy
921 lines
48 KiB
Groovy
pipeline {
|
|
agent any
|
|
|
|
parameters {
|
|
booleanParam(
|
|
name: 'FORCE_INFRASTRUCTURE_DEPLOY',
|
|
defaultValue: false,
|
|
description: 'Force infrastructure deployment regardless of change detection'
|
|
)
|
|
booleanParam(
|
|
name: 'SKIP_QUALITY_GATES',
|
|
defaultValue: false,
|
|
description: 'Skip SonarQube quality gates (use with caution)'
|
|
)
|
|
booleanParam(
|
|
name: 'DESTROY_INFRASTRUCTURE',
|
|
defaultValue: false,
|
|
description: 'Destroy all infrastructure (use with extreme caution)'
|
|
)
|
|
}
|
|
|
|
environment {
|
|
// Core configuration
|
|
GITEA_REPO = 'https://code.jacquesingram.online/lenape/nvhi-atsila-microservice.git '
|
|
GITEA_CREDS = '52ee0829-6e65-4951-925b-4186254c3f21'
|
|
SONAR_HOST = 'https://sonar.jacquesingram.online '
|
|
SONAR_TOKEN = credentials('sonar-token')
|
|
// AWS configuration with ECR
|
|
AWS_CRED_ID = 'aws-ci'
|
|
AWS_ACCOUNT_ID = credentials('AWS_ACCOUNT_ID')
|
|
AWS_REGION = 'us-east-2'
|
|
ECR_REPO = 'nvhi-atsila-microservice'
|
|
// Backend configuration
|
|
TF_BACKEND_BUCKET = 'nvhi-atsila-tf-state'
|
|
TF_BACKEND_PREFIX = 'ecs/terraform.tfstate'
|
|
TF_DDB_TABLE = 'nvhi-atsila-locks'
|
|
// Application variables
|
|
TF_VAR_cluster_name = 'nvhi-atsila-cluster'
|
|
TF_VAR_vpc_cidr = '10.0.0.0/16'
|
|
TF_VAR_public_subnets = '10.0.1.0/24,10.0.2.0/24'
|
|
TF_VAR_instance_type = 't2.micro'
|
|
TF_VAR_key_pair_name = 'nvhi-atsila-deployer'
|
|
TF_VAR_jenkins_ip_cidr = "0.0.0.0/0" // For demo; tighten in production
|
|
TF_VAR_aws_region = "${AWS_REGION}"
|
|
// Enhanced deployment tracking
|
|
IMAGE_TAG = "v1.0.${BUILD_NUMBER}"
|
|
// Initialize deployment type - will be set properly in stages
|
|
DEPLOYMENT_TYPE = "APPLICATION"
|
|
// Enterprise settings
|
|
TF_IN_AUTOMATION = 'true'
|
|
TF_INPUT = 'false'
|
|
// Ansible configuration
|
|
ANSIBLE_HOST_KEY_CHECKING = 'False'
|
|
ANSIBLE_CONFIG = './ansible/ansible.cfg'
|
|
ECS_LOG_GROUP = "/ecs/nvhi-atsila-cluster"
|
|
}
|
|
|
|
stages {
|
|
stage('Debug: Show File Structure') {
|
|
steps {
|
|
echo "📂 Current directory contents:"
|
|
sh 'ls -la'
|
|
echo "🔍 Full file tree:"
|
|
sh 'find . -type f | sort'
|
|
}
|
|
}
|
|
|
|
stage('Bootstrap Terraform Backend') {
|
|
steps {
|
|
script {
|
|
def tfBackendDir = "terraform-backend"
|
|
|
|
echo "🔐 Using Jenkins credentials to authenticate with AWS"
|
|
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
|
|
echo "🔄 Checking/Bootstrapping Terraform backend..."
|
|
dir(tfBackendDir) {
|
|
def exitCode = sh(
|
|
script: """
|
|
terraform init \\
|
|
-var="aws_region=${TF_VAR_aws_region}" \\
|
|
-var="backend_bucket_name=${TF_BACKEND_BUCKET}" \\
|
|
-var="lock_table_name=${TF_DDB_TABLE}"
|
|
terraform apply -auto-approve \\
|
|
-var="aws_region=${TF_VAR_aws_region}" \\
|
|
-var="backend_bucket_name=${TF_BACKEND_BUCKET}" \\
|
|
-var="lock_table_name=${TF_DDB_TABLE}"
|
|
""",
|
|
returnStatus: true
|
|
)
|
|
|
|
if (exitCode == 0) {
|
|
echo "✅ Terraform backend created successfully"
|
|
} else {
|
|
echo "⚠️ Terraform apply failed, checking if resources already exist..."
|
|
def bucketExists = sh(
|
|
script: "aws s3api head-bucket --bucket ${TF_BACKEND_BUCKET} --region ${TF_VAR_aws_region} 2>/dev/null",
|
|
returnStatus: true
|
|
) == 0
|
|
def tableExists = sh(
|
|
script: "aws dynamodb describe-table --table-name ${TF_DDB_TABLE} --region ${TF_VAR_aws_region} 2>/dev/null",
|
|
returnStatus: true
|
|
) == 0
|
|
|
|
if (bucketExists && tableExists) {
|
|
echo "✅ Terraform backend already exists - continuing..."
|
|
} else {
|
|
echo "❌ Backend bootstrap failed and resources don't exist:"
|
|
echo " S3 Bucket exists: ${bucketExists}"
|
|
echo " DynamoDB Table exists: ${tableExists}"
|
|
error("Manual intervention required.")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Security Assessment & Checkout') {
|
|
steps {
|
|
checkout scm
|
|
script {
|
|
// Check for infrastructure destruction first
|
|
if (params.DESTROY_INFRASTRUCTURE) {
|
|
env.DEPLOYMENT_TYPE = "DESTROY"
|
|
currentBuild.displayName = "DESTROY-${BUILD_NUMBER}"
|
|
echo "🚨 DESTROY MODE: Infrastructure destruction requested"
|
|
return
|
|
}
|
|
|
|
def infrastructureFiles = sh(
|
|
script: '''
|
|
if git rev-parse HEAD~1 >/dev/null 2>&1; then
|
|
git diff --name-only HEAD~1 2>/dev/null | grep -E "^terraform/" || echo "none"
|
|
else
|
|
echo "initial"
|
|
fi
|
|
''',
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
// Check force parameter first - this overrides everything
|
|
if (params.FORCE_INFRASTRUCTURE_DEPLOY) {
|
|
env.DEPLOYMENT_TYPE = "INFRASTRUCTURE"
|
|
currentBuild.displayName = "INFRASTRUCTURE-FORCED-${BUILD_NUMBER}"
|
|
echo "🚨 FORCED: Infrastructure deployment requested via parameter"
|
|
echo "✅ Deployment type set to: INFRASTRUCTURE (forced)"
|
|
} else if (infrastructureFiles == "initial") {
|
|
env.DEPLOYMENT_TYPE = "INFRASTRUCTURE"
|
|
currentBuild.displayName = "INFRASTRUCTURE-INITIAL-${BUILD_NUMBER}"
|
|
echo "✅ First run detected. Deploying infrastructure."
|
|
} else if (infrastructureFiles != "none") {
|
|
env.DEPLOYMENT_TYPE = "INFRASTRUCTURE"
|
|
currentBuild.displayName = "INFRASTRUCTURE-CHANGED-${BUILD_NUMBER}"
|
|
echo "🚨 SECURITY NOTICE: Infrastructure changes detected - elevated permissions required"
|
|
echo " Changed files: ${infrastructureFiles}"
|
|
} else {
|
|
env.DEPLOYMENT_TYPE = "APPLICATION"
|
|
currentBuild.displayName = "APPLICATION-${BUILD_NUMBER}"
|
|
echo "✅ SECURITY: Application-only deployment - using restricted permissions"
|
|
}
|
|
|
|
def gitCommit = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
|
|
def gitAuthor = sh(script: 'git log -1 --pretty=format:"%an"', returnStdout: true).trim()
|
|
currentBuild.description = "${env.DEPLOYMENT_TYPE} | ${env.IMAGE_TAG} | ${gitCommit.take(8)}"
|
|
echo "📋 SECURITY AUDIT TRAIL:"
|
|
echo " • Deployment Type: ${env.DEPLOYMENT_TYPE}"
|
|
echo " • Version: ${env.IMAGE_TAG}"
|
|
echo " • Commit: ${gitCommit.take(8)}"
|
|
echo " • Author: ${gitAuthor}"
|
|
echo " • Container Registry: ECR (AWS-native, secure)"
|
|
echo " • Architecture: Ansible-based deployment (enterprise security)"
|
|
echo " • Security Model: Principle of Least Privilege"
|
|
echo " • Timestamp: ${new Date()}"
|
|
echo "🔄 DEPLOYMENT TYPE CONFIRMATION: ${env.DEPLOYMENT_TYPE}"
|
|
|
|
writeFile file: 'deployment-audit.json', text: """{
|
|
"build_number": "${BUILD_NUMBER}",
|
|
"deployment_type": "${env.DEPLOYMENT_TYPE}",
|
|
"image_tag": "${env.IMAGE_TAG}",
|
|
"git_commit": "${gitCommit}",
|
|
"git_author": "${gitAuthor}",
|
|
"infrastructure_files_changed": "${infrastructureFiles}",
|
|
"container_registry": "ECR",
|
|
"architecture": "ansible_based_deployment",
|
|
"security_model": "principle_of_least_privilege",
|
|
"timestamp": "${new Date()}"
|
|
}"""
|
|
archiveArtifacts artifacts: 'deployment-audit.json', fingerprint: true
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Security & Quality Checks') {
|
|
parallel {
|
|
stage('SonarQube Security Analysis') {
|
|
when {
|
|
expression { !params.SKIP_QUALITY_GATES }
|
|
}
|
|
steps {
|
|
script {
|
|
def scannerHome = tool 'SonarQubeScanner'
|
|
withSonarQubeEnv('SonarQube') {
|
|
sh """
|
|
echo "🔒 SECURITY: Running SonarQube security analysis..."
|
|
${scannerHome}/bin/sonar-scanner \\
|
|
-Dsonar.projectKey=nvhi-atsila-microservice \\
|
|
-Dsonar.sources=. \\
|
|
-Dsonar.projectVersion=${BUILD_NUMBER} \\
|
|
-Dsonar.login=${SONAR_TOKEN}
|
|
"""
|
|
}
|
|
echo "✅ SECURITY: Code quality and security scan completed"
|
|
}
|
|
}
|
|
}
|
|
stage('Terraform Security Validation') {
|
|
steps {
|
|
script {
|
|
echo "🔒 SECURITY: Running Terraform security and validation checks..."
|
|
sh '''
|
|
echo "Validating Terraform configuration..."
|
|
cd terraform && terraform init -backend=false
|
|
terraform validate
|
|
echo "✅ Terraform validation passed"
|
|
echo "🔒 SECURITY: Checking infrastructure security compliance..."
|
|
grep -r "encrypted.*true" . --include="*.tf" && echo "✅ Encryption policies found" || echo "⚠️ Review encryption settings"
|
|
echo "🔒 SECURITY: Checking for open security groups..."
|
|
if grep -r "0.0.0.0/0" . --include="*.tf" --exclude-dir=".terraform" | grep -v "# Approved:"; then
|
|
echo "⚠️ Review open access rules found"
|
|
else
|
|
echo "✅ No unauthorized open access rules"
|
|
fi
|
|
'''
|
|
echo "✅ SECURITY: Infrastructure validation and security checks passed"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Secure Container Build & Registry') {
|
|
when {
|
|
not { expression { env.DEPLOYMENT_TYPE == "DESTROY" } }
|
|
}
|
|
steps {
|
|
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
|
|
script {
|
|
echo "🔐 SECURITY: Using ECR for secure, AWS-native container registry"
|
|
|
|
// Create ECR repository if it doesn't exist
|
|
echo "🔍 Checking/Creating ECR repository..."
|
|
sh """
|
|
if ! aws ecr describe-repositories --repository-names ${ECR_REPO} --region ${AWS_REGION} 2>/dev/null; then
|
|
echo "📦 Creating ECR repository: ${ECR_REPO}"
|
|
aws ecr create-repository --repository-name ${ECR_REPO} --region ${AWS_REGION}
|
|
echo "✅ ECR repository created successfully"
|
|
else
|
|
echo "✅ ECR repository already exists"
|
|
fi
|
|
"""
|
|
|
|
sh """
|
|
echo "🔐 Authenticating with ECR using temporary credentials..."
|
|
aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com
|
|
"""
|
|
echo "🐳 Building secure container with metadata..."
|
|
sh """
|
|
docker build -t ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG} .
|
|
docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}
|
|
docker tag ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG} ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:latest
|
|
docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:latest
|
|
"""
|
|
echo "✅ SECURITY: Container built and pushed to ECR successfully"
|
|
echo " Image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${ECR_REPO}:${IMAGE_TAG}"
|
|
echo " Registry: ECR (AWS-native, IAM-secured)"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Infrastructure Readiness Check') {
|
|
when {
|
|
not { expression { env.DEPLOYMENT_TYPE == "DESTROY" } }
|
|
}
|
|
steps {
|
|
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
|
|
script {
|
|
echo "🔍 SECURITY: Checking if infrastructure is ready for deployment..."
|
|
echo "🔍 Current deployment type: ${env.DEPLOYMENT_TYPE}"
|
|
|
|
// Only check readiness if deployment type is APPLICATION
|
|
if (env.DEPLOYMENT_TYPE == "APPLICATION") {
|
|
def serviceExists = sh(
|
|
script: """
|
|
aws ecs describe-services --cluster ${TF_VAR_cluster_name} --services ${TF_VAR_cluster_name}-service --region ${AWS_REGION} 2>/dev/null | grep -q 'ACTIVE' && echo 'true' || echo 'false'
|
|
""",
|
|
returnStdout: true
|
|
).trim()
|
|
def instanceCount = sh(
|
|
script: """
|
|
aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'length(containerInstanceArns)' --output text 2>/dev/null || echo '0'
|
|
""",
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
echo "🔍 Service Exists: ${serviceExists}"
|
|
echo "🔍 Container Instances: ${instanceCount}"
|
|
|
|
if (serviceExists == "false" || instanceCount == "0" || instanceCount == "null") {
|
|
echo "🚨 SECURITY NOTICE: Infrastructure not ready - forcing infrastructure deployment"
|
|
env.DEPLOYMENT_TYPE = "INFRASTRUCTURE"
|
|
currentBuild.displayName = "INFRASTRUCTURE-AUTO-${BUILD_NUMBER}"
|
|
currentBuild.description = "INFRASTRUCTURE (auto-detected) | ${env.IMAGE_TAG}"
|
|
echo "✅ Changed deployment type to: INFRASTRUCTURE"
|
|
}
|
|
} else {
|
|
echo "✅ Infrastructure deployment already scheduled - skipping readiness check"
|
|
}
|
|
|
|
echo "📋 SECURITY: Infrastructure readiness assessment completed"
|
|
echo " Final Deployment Type: ${env.DEPLOYMENT_TYPE}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Destroy Infrastructure') {
|
|
when {
|
|
expression { env.DEPLOYMENT_TYPE == "DESTROY" }
|
|
}
|
|
steps {
|
|
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
|
|
dir('terraform') {
|
|
script {
|
|
echo "🚨 DESTRUCTION: Destroying infrastructure..."
|
|
sh """
|
|
echo "🔄 Initializing Terraform with remote backend..."
|
|
terraform init \\
|
|
-backend-config="bucket=${TF_BACKEND_BUCKET}" \\
|
|
-backend-config="key=${TF_BACKEND_PREFIX}" \\
|
|
-backend-config="region=${AWS_REGION}" \\
|
|
-backend-config="dynamodb_table=${TF_DDB_TABLE}"
|
|
|
|
echo "🔄 Planning infrastructure destruction..."
|
|
terraform plan -destroy \\
|
|
-var="cluster_name=${TF_VAR_cluster_name}" \\
|
|
-var="vpc_cidr=${TF_VAR_vpc_cidr}" \\
|
|
-var="public_subnets=${TF_VAR_public_subnets}" \\
|
|
-var="instance_type=${TF_VAR_instance_type}" \\
|
|
-var="key_pair_name=${TF_VAR_key_pair_name}" \\
|
|
-var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\
|
|
-var="aws_region=${TF_VAR_aws_region}"
|
|
|
|
echo "🔄 Destroying infrastructure..."
|
|
terraform destroy -auto-approve \\
|
|
-var="cluster_name=${TF_VAR_cluster_name}" \\
|
|
-var="vpc_cidr=${TF_VAR_vpc_cidr}" \\
|
|
-var="public_subnets=${TF_VAR_public_subnets}" \\
|
|
-var="instance_type=${TF_VAR_instance_type}" \\
|
|
-var="key_pair_name=${TF_VAR_key_pair_name}" \\
|
|
-var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\
|
|
-var="aws_region=${TF_VAR_aws_region}"
|
|
"""
|
|
echo "✅ Infrastructure destruction completed"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Deploy Infrastructure') {
|
|
when {
|
|
anyOf {
|
|
expression { params.FORCE_INFRASTRUCTURE_DEPLOY == true }
|
|
expression { env.DEPLOYMENT_TYPE == "INFRASTRUCTURE" }
|
|
}
|
|
}
|
|
steps {
|
|
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
|
|
dir('terraform') {
|
|
script {
|
|
echo "🔍 DEPLOYMENT: Force parameter = ${params.FORCE_INFRASTRUCTURE_DEPLOY}"
|
|
echo "🔍 DEPLOYMENT: Deployment type = ${env.DEPLOYMENT_TYPE}"
|
|
echo "🚨 SECURITY NOTICE: Infrastructure deployment requested"
|
|
echo "🏗️ ARCHITECTURE: Deploying ECS Cluster with Ansible-based deployment (enterprise security)"
|
|
echo "🔐 In production: This would require infrastructure-admin role"
|
|
echo "🚀 Attempting infrastructure deployment..."
|
|
|
|
// Add error handling for Terraform operations
|
|
try {
|
|
sh """
|
|
echo "🔄 Initializing Terraform with remote backend..."
|
|
terraform init \\
|
|
-backend-config="bucket=${TF_BACKEND_BUCKET}" \\
|
|
-backend-config="key=${TF_BACKEND_PREFIX}" \\
|
|
-backend-config="region=${AWS_REGION}" \\
|
|
-backend-config="dynamodb_table=${TF_DDB_TABLE}"
|
|
|
|
echo "🔄 Planning infrastructure changes..."
|
|
terraform plan \\
|
|
-var="cluster_name=${TF_VAR_cluster_name}" \\
|
|
-var="vpc_cidr=${TF_VAR_vpc_cidr}" \\
|
|
-var="public_subnets=${TF_VAR_public_subnets}" \\
|
|
-var="instance_type=${TF_VAR_instance_type}" \\
|
|
-var="key_pair_name=${TF_VAR_key_pair_name}" \\
|
|
-var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\
|
|
-var="aws_region=${TF_VAR_aws_region}"
|
|
|
|
echo "🔄 Applying infrastructure changes..."
|
|
terraform apply -auto-approve \\
|
|
-var="cluster_name=${TF_VAR_cluster_name}" \\
|
|
-var="vpc_cidr=${TF_VAR_vpc_cidr}" \\
|
|
-var="public_subnets=${TF_VAR_public_subnets}" \\
|
|
-var="instance_type=${TF_VAR_instance_type}" \\
|
|
-var="key_pair_name=${TF_VAR_key_pair_name}" \\
|
|
-var="jenkins_ip_cidr=${TF_VAR_jenkins_ip_cidr}" \\
|
|
-var="aws_region=${TF_VAR_aws_region}"
|
|
"""
|
|
echo "✅ SECURITY: Infrastructure deployment completed successfully"
|
|
} catch (Exception e) {
|
|
echo "❌ Infrastructure deployment failed: ${e.getMessage()}"
|
|
echo "📋 Checking current Terraform state..."
|
|
sh "terraform show || echo 'No state found'"
|
|
throw e
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Wait for ECS Agents') {
|
|
when {
|
|
anyOf {
|
|
expression { params.FORCE_INFRASTRUCTURE_DEPLOY == true }
|
|
expression { env.DEPLOYMENT_TYPE == "INFRASTRUCTURE" }
|
|
}
|
|
}
|
|
steps {
|
|
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
|
|
script {
|
|
echo "⏳ Waiting for ECS agents to register with cluster..."
|
|
timeout(time: 10, unit: 'MINUTES') {
|
|
waitUntil {
|
|
def count = sh(
|
|
script: """
|
|
aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'length(containerInstanceArns)' --output text 2>/dev/null || echo '0'
|
|
""",
|
|
returnStdout: true
|
|
).trim()
|
|
if (count != "0" && count != "null") {
|
|
echo "✅ ECS agents registered: ${count} instance(s)"
|
|
// Fixed: Simplified active count check to avoid backtick escaping issues
|
|
def activeCount = sh(
|
|
script: """
|
|
aws ecs describe-container-instances \\
|
|
--cluster ${TF_VAR_cluster_name} \\
|
|
--container-instances \$(aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} --query 'containerInstanceArns[*]' --output text) \\
|
|
--region ${AWS_REGION} \\
|
|
--output text | grep -c ACTIVE || echo '0'
|
|
""",
|
|
returnStdout: true
|
|
).trim()
|
|
if (activeCount != "0" && activeCount != "null") {
|
|
echo "✅ Active ECS instances: ${activeCount}"
|
|
return true
|
|
} else {
|
|
echo "⏳ Waiting for instances to become ACTIVE..."
|
|
sleep(20)
|
|
return false
|
|
}
|
|
} else {
|
|
echo "⏳ No ECS agents registered yet..."
|
|
sleep(20)
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Configure & Deploy Application with Ansible') {
|
|
when {
|
|
not { expression { env.DEPLOYMENT_TYPE == "DESTROY" } }
|
|
}
|
|
steps {
|
|
script {
|
|
echo "🚀 ENTERPRISE: Deploying with Ansible (replacing SSM approach)"
|
|
|
|
// Get infrastructure details from Terraform
|
|
def instanceId = ""
|
|
def publicIp = ""
|
|
def executionRoleArn = ""
|
|
|
|
try {
|
|
instanceId = sh(
|
|
script: "cd terraform && terraform output -raw ecs_instance_id",
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
publicIp = sh(
|
|
script: "cd terraform && terraform output -raw ecs_instance_public_ip",
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
executionRoleArn = sh(
|
|
script: "cd terraform && terraform output -raw ecs_task_execution_role_arn",
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
echo "📍 Target Instance: ${instanceId} (${publicIp})"
|
|
echo "🔧 Execution Role: ${executionRoleArn}"
|
|
} catch (Exception e) {
|
|
echo "⚠️ Could not get all Terraform outputs: ${e.getMessage()}"
|
|
echo "⚠️ Some outputs may be missing, continuing with available data..."
|
|
}
|
|
|
|
// Create Ansible working directory and files
|
|
sh "mkdir -p ansible/group_vars"
|
|
|
|
// Create dynamic inventory file
|
|
def inventoryContent = """[inventory_hosts]
|
|
ec2-instance ansible_host=${publicIp} ansible_user=ec2-user
|
|
|
|
[inventory_hosts:vars]
|
|
ansible_ssh_private_key_file=~/.ssh/id_rsa
|
|
ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o ServerAliveInterval=60'
|
|
ansible_python_interpreter=/usr/bin/python3
|
|
ansible_connection=ssh
|
|
ansible_ssh_retries=3
|
|
aws_region=${AWS_REGION}
|
|
"""
|
|
writeFile file: 'ansible/hosts', text: inventoryContent
|
|
|
|
// Create Ansible configuration
|
|
def ansibleConfig = """[defaults]
|
|
inventory = hosts
|
|
host_key_checking = False
|
|
retry_files_enabled = False
|
|
gathering = smart
|
|
stdout_callback = yaml
|
|
timeout = 30
|
|
log_path = ./ansible.log
|
|
|
|
[ssh_connection]
|
|
ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=10
|
|
pipelining = True
|
|
"""
|
|
writeFile file: 'ansible/ansible.cfg', text: ansibleConfig
|
|
|
|
// Create group variables
|
|
def groupVarsContent = """---
|
|
ecs_cluster_name: ${TF_VAR_cluster_name}
|
|
service_name: ${TF_VAR_cluster_name}-service
|
|
task_family: ${TF_VAR_cluster_name}-task
|
|
container_name: ${ECR_REPO}
|
|
aws_region: ${AWS_REGION}
|
|
container_port: 8080
|
|
"""
|
|
writeFile file: 'ansible/group_vars/all.yml', text: groupVarsContent
|
|
|
|
// Test connectivity and execute deployment
|
|
withCredentials([
|
|
[$class: 'AmazonWebServicesCredentialsBinding',
|
|
credentialsId: env.AWS_CRED_ID,
|
|
accessKeyVariable: 'AWS_ACCESS_KEY_ID',
|
|
secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
|
|
]) {
|
|
sh """
|
|
cd ansible
|
|
|
|
# Set environment variables
|
|
export AWS_DEFAULT_REGION="${AWS_REGION}"
|
|
export ANSIBLE_HOST_KEY_CHECKING=False
|
|
export ANSIBLE_CONFIG="./ansible.cfg"
|
|
|
|
# Wait for SSH connectivity
|
|
echo "🔍 Testing SSH connectivity to ${publicIp}..."
|
|
timeout 120 bash -c 'while ! nc -z ${publicIp} 22; do echo "Waiting for SSH..."; sleep 5; done'
|
|
|
|
# Install Python dependencies if needed
|
|
pip3 install --user boto3 botocore jq > /dev/null 2>&1 || true
|
|
|
|
# Test Ansible connectivity
|
|
echo "🔍 Testing Ansible connectivity..."
|
|
ansible inventory_hosts -m ping -i hosts -v
|
|
|
|
if [ \$? -ne 0 ]; then
|
|
echo "❌ Ansible connectivity failed"
|
|
echo "Debugging SSH connection..."
|
|
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${publicIp} 'echo "SSH test successful"' || {
|
|
echo "SSH connection failed"
|
|
exit 1
|
|
}
|
|
exit 1
|
|
fi
|
|
|
|
echo "✅ Connectivity test passed"
|
|
|
|
# Execute main deployment playbook
|
|
echo "🚀 Starting deployment..."
|
|
ansible-playbook configure_ecs.yml \\
|
|
-i hosts \\
|
|
-e "app_version=${IMAGE_TAG}" \\
|
|
-e "aws_account_id=${AWS_ACCOUNT_ID}" \\
|
|
-e "aws_region=${AWS_REGION}" \\
|
|
-e "task_execution_role_arn=${executionRoleArn}" \\
|
|
--timeout 600 \\
|
|
-v
|
|
"""
|
|
}
|
|
|
|
// Final verification
|
|
echo "🔍 Running final verification..."
|
|
sh """
|
|
echo "Testing application endpoint..."
|
|
for i in {1..10}; do
|
|
if curl -f -s "http://${publicIp}:8080/health"; then
|
|
echo "✅ Application health check passed"
|
|
break
|
|
else
|
|
echo "⏳ Health check attempt \$i/10..."
|
|
sleep 10
|
|
fi
|
|
done
|
|
"""
|
|
}
|
|
}
|
|
|
|
post {
|
|
success {
|
|
script {
|
|
def publicIp = sh(
|
|
script: "cd terraform && terraform output -raw ecs_instance_public_ip",
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
echo """
|
|
========================================
|
|
🎉 DEPLOYMENT SUCCESSFUL!
|
|
========================================
|
|
Application URL: http://${publicIp}:8080
|
|
Health Endpoint: http://${publicIp}:8080/health
|
|
Version: ${IMAGE_TAG}
|
|
Deployment Method: Ansible (Enterprise Security)
|
|
========================================
|
|
"""
|
|
}
|
|
|
|
// Archive deployment artifacts
|
|
archiveArtifacts artifacts: 'ansible/ansible.log', allowEmptyArchive: true
|
|
}
|
|
|
|
failure {
|
|
echo "❌ DEPLOYMENT FAILED - Gathering debug information..."
|
|
|
|
script {
|
|
sh """
|
|
echo "=== ANSIBLE DEBUG INFORMATION ==="
|
|
cat ansible/ansible.log 2>/dev/null || echo "No Ansible log available"
|
|
|
|
echo "=== ECS SERVICE STATUS ==="
|
|
aws ecs describe-services \\
|
|
--cluster "${TF_VAR_cluster_name}" \\
|
|
--services "${TF_VAR_cluster_name}-service" \\
|
|
--region "${AWS_REGION}" \\
|
|
--query 'services[0].{Status:status,Running:runningCount,Pending:pendingCount,Events:events[0:3]}' \\
|
|
--output json 2>/dev/null || echo "Could not get ECS service status"
|
|
|
|
echo "=== ECS CLUSTER STATUS ==="
|
|
aws ecs describe-clusters \\
|
|
--clusters "${TF_VAR_cluster_name}" \\
|
|
--region "${AWS_REGION}" \\
|
|
--query 'clusters[0].{Status:status,ActiveInstances:activeContainerInstancesCount,Tasks:runningTasksCount}' \\
|
|
--output json 2>/dev/null || echo "Could not get ECS cluster status"
|
|
|
|
echo "=== RECENT CONTAINER LOGS ==="
|
|
LATEST_STREAM=\$(aws logs describe-log-streams \\
|
|
--log-group-name "${ECS_LOG_GROUP}" \\
|
|
--region "${AWS_REGION}" \\
|
|
--order-by LastEventTime \\
|
|
--descending \\
|
|
--max-items 1 \\
|
|
--query 'logStreams[0].logStreamName' \\
|
|
--output text 2>/dev/null)
|
|
|
|
if [ "\$LATEST_STREAM" != "None" ] && [ "\$LATEST_STREAM" != "" ]; then
|
|
echo "Latest log stream: \$LATEST_STREAM"
|
|
aws logs get-log-events \\
|
|
--log-group-name "${ECS_LOG_GROUP}" \\
|
|
--log-stream-name "\$LATEST_STREAM" \\
|
|
--region "${AWS_REGION}" \\
|
|
--start-from-head \\
|
|
--query 'events[-20:].[timestamp,message]' \\
|
|
--output table 2>/dev/null || echo "Could not retrieve logs"
|
|
else
|
|
echo "No log streams found"
|
|
fi
|
|
"""
|
|
}
|
|
|
|
// Offer rollback option
|
|
script {
|
|
try {
|
|
timeout(time: 5, unit: 'MINUTES') {
|
|
def rollbackChoice = input(
|
|
message: 'Deployment failed. Would you like to rollback to the previous version?',
|
|
parameters: [
|
|
choice(choices: ['No', 'Yes'], description: 'Rollback?', name: 'ROLLBACK')
|
|
]
|
|
)
|
|
|
|
if (rollbackChoice == 'Yes') {
|
|
echo "🔄 Initiating automatic rollback..."
|
|
withCredentials([
|
|
[$class: 'AmazonWebServicesCredentialsBinding',
|
|
credentialsId: env.AWS_CRED_ID,
|
|
accessKeyVariable: 'AWS_ACCESS_KEY_ID',
|
|
secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']
|
|
]) {
|
|
sh """
|
|
cd ansible
|
|
ansible-playbook rollback.yml \\
|
|
-e auto_rollback=true \\
|
|
-v
|
|
"""
|
|
}
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
echo "Rollback prompt timed out or was cancelled"
|
|
}
|
|
}
|
|
}
|
|
|
|
always {
|
|
// Cleanup temporary files
|
|
sh """
|
|
rm -f ansible/hosts 2>/dev/null || true
|
|
rm -f ansible/ansible.cfg 2>/dev/null || true
|
|
rm -f ansible/group_vars/all.yml 2>/dev/null || true
|
|
"""
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Verify Deployment') {
|
|
when {
|
|
not { expression { env.DEPLOYMENT_TYPE == "DESTROY" } }
|
|
}
|
|
steps {
|
|
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
|
|
script {
|
|
echo "🔍 VERIFICATION: Running comprehensive validation..."
|
|
|
|
def publicIp = sh(
|
|
script: "cd terraform && terraform output -raw ecs_instance_public_ip",
|
|
returnStdout: true
|
|
).trim()
|
|
|
|
sh """
|
|
echo "=== APPLICATION HEALTH CHECK ==="
|
|
curl -f -v "http://${publicIp}:8080/health"
|
|
|
|
echo "=== ECS SERVICE VALIDATION ==="
|
|
aws ecs describe-services \\
|
|
--cluster "${TF_VAR_cluster_name}" \\
|
|
--services "${TF_VAR_cluster_name}-service" \\
|
|
--region "${AWS_REGION}" \\
|
|
--query 'services[0].{Status:status,TaskDefinition:taskDefinition,Running:runningCount,Desired:desiredCount}' \\
|
|
--output table
|
|
|
|
echo "=== CONTAINER HEALTH CHECK ==="
|
|
# Check if containers are healthy
|
|
RUNNING_TASKS=\$(aws ecs list-tasks \\
|
|
--cluster "${TF_VAR_cluster_name}" \\
|
|
--service-name "${TF_VAR_cluster_name}-service" \\
|
|
--desired-status RUNNING \\
|
|
--region "${AWS_REGION}" \\
|
|
--query 'taskArns' \\
|
|
--output text)
|
|
|
|
if [ -n "\$RUNNING_TASKS" ]; then
|
|
aws ecs describe-tasks \\
|
|
--cluster "${TF_VAR_cluster_name}" \\
|
|
--tasks \$RUNNING_TASKS \\
|
|
--region "${AWS_REGION}" \\
|
|
--query 'tasks[0].containers[0].{Name:name,Status:lastStatus,Health:healthStatus}' \\
|
|
--output table
|
|
fi
|
|
|
|
echo "=== LOG VALIDATION ==="
|
|
# Fixed: Simplified log analysis to avoid complex escaping
|
|
LATEST_STREAM=\$(aws logs describe-log-streams \\
|
|
--log-group-name "${ECS_LOG_GROUP}" \\
|
|
--region "${AWS_REGION}" \\
|
|
--order-by LastEventTime \\
|
|
--descending \\
|
|
--max-items 1 \\
|
|
--query 'logStreams[0].logStreamName' \\
|
|
--output text 2>/dev/null)
|
|
|
|
if [ "\$LATEST_STREAM" != "None" ] && [ "\$LATEST_STREAM" != "" ]; then
|
|
echo "Checking logs for errors in stream: \$LATEST_STREAM"
|
|
# Simple approach: get recent log messages and check for errors with grep
|
|
aws logs get-log-events \\
|
|
--log-group-name "${ECS_LOG_GROUP}" \\
|
|
--log-stream-name "\$LATEST_STREAM" \\
|
|
--region "${AWS_REGION}" \\
|
|
--start-from-head \\
|
|
--query 'events[-20:].message' \\
|
|
--output text > /tmp/recent_logs.txt 2>/dev/null || echo "Could not get logs"
|
|
|
|
if [ -f /tmp/recent_logs.txt ]; then
|
|
ERROR_COUNT=\$(grep -c -i "error\\|fatal\\|exception" /tmp/recent_logs.txt 2>/dev/null || echo "0")
|
|
if [ "\$ERROR_COUNT" -gt 0 ]; then
|
|
echo "⚠️ Found \$ERROR_COUNT potential errors in logs - please review"
|
|
echo "Recent error lines:"
|
|
grep -i "error\\|fatal\\|exception" /tmp/recent_logs.txt | head -5 || true
|
|
else
|
|
echo "✅ No errors found in recent application logs"
|
|
fi
|
|
rm -f /tmp/recent_logs.txt
|
|
fi
|
|
fi
|
|
|
|
echo "✅ All validation checks completed successfully"
|
|
"""
|
|
|
|
// Update build description with URL
|
|
currentBuild.description = "${currentBuild.description} | URL: http://${publicIp}:8080"
|
|
|
|
echo "✅ VERIFICATION: Deployment verification completed"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
post {
|
|
always {
|
|
script {
|
|
echo "🧹 CLEANUP: Performing post-build cleanup..."
|
|
|
|
// Archive deployment artifacts
|
|
try {
|
|
archiveArtifacts artifacts: 'deployment-audit.json,task-definition.json', allowEmptyArchive: true
|
|
} catch (Exception e) {
|
|
echo "⚠️ Could not archive artifacts: ${e.getMessage()}"
|
|
}
|
|
|
|
// Clean up Docker images to save space
|
|
sh '''
|
|
echo "🧹 Cleaning up Docker images..."
|
|
docker system prune -f || echo "Docker cleanup failed"
|
|
'''
|
|
|
|
echo "📊 SUMMARY: Build completed"
|
|
echo " Build Number: ${BUILD_NUMBER}"
|
|
echo " Image Tag: ${IMAGE_TAG}"
|
|
echo " Deployment Type: ${env.DEPLOYMENT_TYPE}"
|
|
echo " Status: ${currentBuild.currentResult}"
|
|
}
|
|
}
|
|
|
|
success {
|
|
script {
|
|
if (env.DEPLOYMENT_TYPE == "DESTROY") {
|
|
echo "🎉 SUCCESS: Infrastructure destroyed successfully!"
|
|
} else {
|
|
echo "🎉 SUCCESS: Deployment completed successfully!"
|
|
echo " Version ${IMAGE_TAG} deployed to ECS cluster ${TF_VAR_cluster_name}"
|
|
|
|
// Get application URL for success message
|
|
def appUrl = ""
|
|
try {
|
|
appUrl = sh(
|
|
script: "cd terraform && terraform output -raw ecs_instance_public_ip 2>/dev/null || echo 'unknown'",
|
|
returnStdout: true
|
|
).trim()
|
|
if (appUrl != "unknown" && appUrl != "") {
|
|
echo "🌐 Application available at: http://${appUrl}:8080"
|
|
echo "🏥 Health check: http://${appUrl}:8080/health"
|
|
}
|
|
} catch (Exception e) {
|
|
echo "⚠️ Could not determine application URL"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
failure {
|
|
script {
|
|
echo "❌ FAILURE: Deployment failed"
|
|
echo " Check the logs above for error details"
|
|
|
|
// Try to get some debug information
|
|
try {
|
|
withCredentials([[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: env.AWS_CRED_ID]]) {
|
|
echo "🔍 DEBUG: Checking ECS cluster status..."
|
|
sh """
|
|
aws ecs describe-clusters --clusters ${TF_VAR_cluster_name} --region ${AWS_REGION} || echo "Cluster check failed"
|
|
aws ecs list-container-instances --cluster ${TF_VAR_cluster_name} --region ${AWS_REGION} || echo "Instance list failed"
|
|
"""
|
|
}
|
|
} catch (Exception e) {
|
|
echo "⚠️ Could not get debug information: ${e.getMessage()}"
|
|
}
|
|
}
|
|
}
|
|
|
|
unstable {
|
|
script {
|
|
echo "⚠️ UNSTABLE: Build completed with warnings"
|
|
}
|
|
}
|
|
}
|
|
} |