Compare commits
20 Commits
Author | SHA1 | Date | |
---|---|---|---|
4c3c26b196 | |||
bec3bd2ee8 | |||
0dc8aecc1d | |||
6fa5cdf5a5 | |||
61647118e5 | |||
|
36fbf4788c | ||
|
7ec72165ee | ||
|
a610c8b0dc | ||
|
cdb0d19de7 | ||
|
000876cccd | ||
|
fee40bba7b | ||
|
56ffc5a83c | ||
|
321c0785bb | ||
|
c2a9b26828 | ||
|
c9aee7ef7f | ||
|
cc3bd26b71 | ||
|
d06f94f41f | ||
|
015ac7adc3 | ||
|
8ef1eeed96 | ||
|
f10c615863 |
37
Jenkinsfile
vendored
37
Jenkinsfile
vendored
@@ -56,12 +56,15 @@ pipeline {
|
|||||||
// Use Docker for consistent, isolated build environment
|
// Use Docker for consistent, isolated build environment
|
||||||
docker.image(env.BUILD_IMAGE).inside('-e HOME=/tmp -e PIP_CACHE_DIR=/tmp/.pip') {
|
docker.image(env.BUILD_IMAGE).inside('-e HOME=/tmp -e PIP_CACHE_DIR=/tmp/.pip') {
|
||||||
sh '''
|
sh '''
|
||||||
|
# Add local bin to PATH
|
||||||
|
export PATH="/tmp/.local/bin:$PATH"
|
||||||
|
|
||||||
# Corporate standard: CodeArtifact primary, PyPI fallback
|
# Corporate standard: CodeArtifact primary, PyPI fallback
|
||||||
export PIP_INDEX_URL="${CODEART_URL}"
|
export PIP_INDEX_URL="${CODEART_URL}"
|
||||||
export PIP_EXTRA_INDEX_URL="https://pypi.org/simple/"
|
export PIP_EXTRA_INDEX_URL="https://pypi.org/simple/"
|
||||||
|
|
||||||
# Install build dependencies
|
# Install build dependencies
|
||||||
pip install --upgrade setuptools wheel twine
|
pip install --user --upgrade setuptools wheel twine
|
||||||
|
|
||||||
# Build the package
|
# Build the package
|
||||||
python3 setup.py sdist bdist_wheel
|
python3 setup.py sdist bdist_wheel
|
||||||
@@ -99,11 +102,14 @@ pipeline {
|
|||||||
script {
|
script {
|
||||||
docker.image(env.BUILD_IMAGE).inside('-e HOME=/tmp -e PIP_CACHE_DIR=/tmp/.pip') {
|
docker.image(env.BUILD_IMAGE).inside('-e HOME=/tmp -e PIP_CACHE_DIR=/tmp/.pip') {
|
||||||
sh '''
|
sh '''
|
||||||
|
# Add local bin to PATH
|
||||||
|
export PATH="/tmp/.local/bin:$PATH"
|
||||||
|
|
||||||
# Corporate standard: CodeArtifact primary, PyPI fallback
|
# Corporate standard: CodeArtifact primary, PyPI fallback
|
||||||
export PIP_INDEX_URL="${CODEART_URL}"
|
export PIP_INDEX_URL="${CODEART_URL}"
|
||||||
export PIP_EXTRA_INDEX_URL="https://pypi.org/simple/"
|
export PIP_EXTRA_INDEX_URL="https://pypi.org/simple/"
|
||||||
|
|
||||||
pip install safety
|
pip install --user safety
|
||||||
safety check --json --output safety-report.json || true
|
safety check --json --output safety-report.json || true
|
||||||
'''
|
'''
|
||||||
}
|
}
|
||||||
@@ -119,20 +125,24 @@ pipeline {
|
|||||||
script {
|
script {
|
||||||
docker.image(env.BUILD_IMAGE).inside('-e HOME=/tmp -e PIP_CACHE_DIR=/tmp/.pip') {
|
docker.image(env.BUILD_IMAGE).inside('-e HOME=/tmp -e PIP_CACHE_DIR=/tmp/.pip') {
|
||||||
sh '''
|
sh '''
|
||||||
|
# Add local bin to PATH
|
||||||
|
export PATH="/tmp/.local/bin:$PATH"
|
||||||
|
|
||||||
# Corporate standard: CodeArtifact primary, PyPI fallback
|
# Corporate standard: CodeArtifact primary, PyPI fallback
|
||||||
export PIP_INDEX_URL="${CODEART_URL}"
|
export PIP_INDEX_URL="${CODEART_URL}"
|
||||||
export PIP_EXTRA_INDEX_URL="https://pypi.org/simple/"
|
export PIP_EXTRA_INDEX_URL="https://pypi.org/simple/"
|
||||||
|
|
||||||
# Install test dependencies if they exist
|
# Install test dependencies if they exist
|
||||||
if [ -f requirements-test.txt ]; then
|
if [ -f requirements-test.txt ]; then
|
||||||
pip install -r requirements-test.txt
|
pip install --user -r requirements-test.txt
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Install the built package for testing
|
# Install the built package for testing
|
||||||
pip install dist/*.whl
|
pip install --user dist/*.whl
|
||||||
|
|
||||||
# Run tests if they exist
|
# Run tests if they exist
|
||||||
if [ -f pytest.ini ] || [ -d tests ]; then
|
if [ -f pytest.ini ] || [ -d tests ]; then
|
||||||
|
pip install --user pytest
|
||||||
python -m pytest --junitxml=test-results.xml || true
|
python -m pytest --junitxml=test-results.xml || true
|
||||||
fi
|
fi
|
||||||
'''
|
'''
|
||||||
@@ -144,7 +154,7 @@ pipeline {
|
|||||||
// Publish test results if they exist
|
// Publish test results if they exist
|
||||||
script {
|
script {
|
||||||
if (fileExists('test-results.xml')) {
|
if (fileExists('test-results.xml')) {
|
||||||
publishTestResults testResultsPattern: 'test-results.xml'
|
junit 'test-results.xml'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -152,17 +162,20 @@ pipeline {
|
|||||||
}
|
}
|
||||||
|
|
||||||
stage('Publish') {
|
stage('Publish') {
|
||||||
when {
|
|
||||||
// Only publish on main branch or release tags
|
|
||||||
anyOf {
|
|
||||||
branch 'main'
|
|
||||||
tag pattern: 'v\\d+\\.\\d+\\.\\d+', comparator: 'REGEXP'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
docker.image(env.BUILD_IMAGE).inside('-e HOME=/tmp') {
|
docker.image(env.BUILD_IMAGE).inside('-e HOME=/tmp') {
|
||||||
sh '''
|
sh '''
|
||||||
|
# Add local bin to PATH
|
||||||
|
export PATH="/tmp/.local/bin:$PATH"
|
||||||
|
|
||||||
|
# Corporate standard: CodeArtifact primary, PyPI fallback
|
||||||
|
export PIP_INDEX_URL="${CODEART_URL}"
|
||||||
|
export PIP_EXTRA_INDEX_URL="https://pypi.org/simple/"
|
||||||
|
|
||||||
|
# Install twine
|
||||||
|
pip install --user twine
|
||||||
|
|
||||||
# Configure twine for CodeArtifact in /tmp
|
# Configure twine for CodeArtifact in /tmp
|
||||||
cat > /tmp/.pypirc <<EOF
|
cat > /tmp/.pypirc <<EOF
|
||||||
[distutils]
|
[distutils]
|
||||||
|
161
README.md
161
README.md
@@ -1,2 +1,161 @@
|
|||||||
# hello-codeartifact
|
# AWS CodeArtifact Python Package Pipeline
|
||||||
|
|
||||||
|
This repository contains a Jenkins pipeline for building and publishing Python packages to AWS CodeArtifact.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
This CI/CD pipeline automates the process of building, testing, securing, and publishing Python packages to a private AWS CodeArtifact repository. It ensures consistent builds, security compliance, and reliable package distribution in an enterprise environment.
|
||||||
|
|
||||||
|
## Pipeline Architecture
|
||||||
|
|
||||||
|
### Environment Configuration
|
||||||
|
- Leverages Jenkins Credentials for secure storage of AWS configuration
|
||||||
|
- Uses Docker containers (Python 3.11-slim) for consistent build environments
|
||||||
|
- Configures AWS CodeArtifact as primary package index with PyPI fallback
|
||||||
|
|
||||||
|
### Pipeline Stages
|
||||||
|
|
||||||
|
#### 1. **Checkout**
|
||||||
|
- Retrieves source code from Git repository
|
||||||
|
- Standard SCM checkout process
|
||||||
|
|
||||||
|
#### 2. **Authenticate & Configure**
|
||||||
|
- Generates short-lived AWS CodeArtifact authentication tokens (12-hour expiration)
|
||||||
|
- Constructs secure repository URLs with embedded credentials
|
||||||
|
- Uses AWS IAM roles via Jenkins AWS plugin for authentication
|
||||||
|
|
||||||
|
#### 3. **Build**
|
||||||
|
- Executes within isolated Docker container
|
||||||
|
- Installs build dependencies (setuptools, wheel, twine)
|
||||||
|
- Produces both source distribution (.tar.gz) and wheel (.whl) artifacts
|
||||||
|
- Configures PATH for pip user installations
|
||||||
|
|
||||||
|
#### 4. **Security Scan** (Parallel Execution)
|
||||||
|
- **Trivy Scanner**: Performs filesystem vulnerability analysis
|
||||||
|
- **Safety Check**: Analyzes Python package dependencies
|
||||||
|
- Fails pipeline on HIGH/CRITICAL severity vulnerabilities
|
||||||
|
- Archives all security reports for compliance
|
||||||
|
|
||||||
|
#### 5. **Test**
|
||||||
|
- Installs built package in clean environment
|
||||||
|
- Executes pytest test suite if present
|
||||||
|
- Generates JUnit XML reports for Jenkins integration
|
||||||
|
- Continues pipeline even if tests fail (for visibility)
|
||||||
|
|
||||||
|
#### 6. **Publish**
|
||||||
|
- Uploads package artifacts to AWS CodeArtifact repository
|
||||||
|
- Uses temporary `.pypirc` configuration
|
||||||
|
- Publishes all artifacts from dist/ directory
|
||||||
|
|
||||||
|
### Post-Build Actions
|
||||||
|
- Archives build artifacts for traceability
|
||||||
|
- Cleans workspace to prevent credential leakage
|
||||||
|
- Sends notifications on success/failure (when configured)
|
||||||
|
|
||||||
|
## Technical Implementation
|
||||||
|
|
||||||
|
### Security Features
|
||||||
|
- Temporary authentication tokens minimize credential exposure
|
||||||
|
- Non-root container execution
|
||||||
|
- Parallel security scanning for comprehensive coverage
|
||||||
|
- Workspace cleanup prevents sensitive data persistence
|
||||||
|
|
||||||
|
### Docker Strategy
|
||||||
|
- Each stage runs in isolated container using `docker.image().inside()`
|
||||||
|
- Shared workspace volume for artifact passing between stages
|
||||||
|
- Non-root execution with `HOME=/tmp` configuration
|
||||||
|
|
||||||
|
### Key Design Decisions
|
||||||
|
|
||||||
|
1. **AWS CodeArtifact Integration**
|
||||||
|
- Private package hosting for security
|
||||||
|
- AWS IAM integration for authentication
|
||||||
|
- Automatic PyPI package caching
|
||||||
|
- Enterprise compliance support
|
||||||
|
|
||||||
|
2. **Containerized Builds**
|
||||||
|
- Eliminates Python version conflicts
|
||||||
|
- Ensures reproducible builds
|
||||||
|
- Provides clean build environment
|
||||||
|
|
||||||
|
3. **Parallel Security Scanning**
|
||||||
|
- Reduces overall pipeline execution time
|
||||||
|
- Multiple vulnerability detection methods
|
||||||
|
- Comprehensive security coverage
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### Required Jenkins Credentials
|
||||||
|
- `AWS_REGION`: AWS region for CodeArtifact
|
||||||
|
- `AWS_ACCOUNT_ID`: AWS account identifier
|
||||||
|
- `CODEART_DOMAIN`: CodeArtifact domain name
|
||||||
|
- `CODEART_REPO`: CodeArtifact repository name
|
||||||
|
- `jenkins-codeartifact`: AWS IAM credentials
|
||||||
|
|
||||||
|
### Pipeline Options
|
||||||
|
```groovy
|
||||||
|
disableConcurrentBuilds() // Prevents parallel execution
|
||||||
|
buildDiscarder(logRotator(numToKeepStr: '50')) // Retains 50 builds
|
||||||
|
timeout(time: 30, unit: 'MINUTES') // 30-minute execution limit
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Installing Published Packages
|
||||||
|
```bash
|
||||||
|
# Configure AWS CodeArtifact
|
||||||
|
aws codeartifact login --tool pip \
|
||||||
|
--domain YOUR_DOMAIN \
|
||||||
|
--repository YOUR_REPO \
|
||||||
|
--domain-owner YOUR_ACCOUNT_ID
|
||||||
|
|
||||||
|
# Install package
|
||||||
|
pip install hello-codeartifact
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using the Package
|
||||||
|
```python
|
||||||
|
from hello_pkg import greet
|
||||||
|
|
||||||
|
message = greet()
|
||||||
|
print(message) # Output: Hello, CodeArtifact!
|
||||||
|
```
|
||||||
|
|
||||||
|
## Monitoring and Reporting
|
||||||
|
|
||||||
|
- **Build Artifacts**: Stored in Jenkins for each build
|
||||||
|
- **Test Results**: JUnit XML reports integrated with Jenkins
|
||||||
|
- **Security Reports**: JSON reports from Trivy and Safety
|
||||||
|
- **Build History**: 50 builds retained for audit purposes
|
||||||
|
|
||||||
|
## Future Enhancements
|
||||||
|
|
||||||
|
- Implement semantic versioning automation
|
||||||
|
- Add code coverage reporting
|
||||||
|
- Integrate SBOM (Software Bill of Materials) generation
|
||||||
|
- Implement multi-environment deployment strategies
|
||||||
|
- Add performance benchmarking
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Common Issues
|
||||||
|
- **Authentication Failures**: Verify AWS credentials and IAM permissions
|
||||||
|
- **Build Failures**: Check Docker availability and Python syntax
|
||||||
|
- **Security Scan Failures**: Review vulnerability reports in archived artifacts
|
||||||
|
|
||||||
|
### Debug Commands
|
||||||
|
```bash
|
||||||
|
# Verify CodeArtifact connection
|
||||||
|
aws codeartifact list-packages --domain DOMAIN --repository REPO
|
||||||
|
|
||||||
|
# Test package installation
|
||||||
|
pip install hello-codeartifact --index-url CODEARTIFACT_URL
|
||||||
|
```
|
||||||
|
|
||||||
|
## Contributing
|
||||||
|
|
||||||
|
When modifying the pipeline:
|
||||||
|
1. Test changes in a development branch
|
||||||
|
2. Ensure security scans pass
|
||||||
|
3. Verify package publishes correctly
|
||||||
|
4. Update documentation as needed
|
4
setup.py
4
setup.py
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
|
|||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="hello-codeartifact",
|
name="hello-codeartifact",
|
||||||
version="0.1.0",
|
version="0.1.3",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
description="A hello‑world PyPI package for CodeArtifact demo",
|
description="A hello-world PyPI package for CodeArtifact demo",
|
||||||
)
|
)
|
||||||
|
Reference in New Issue
Block a user