graph TD
 A[Code Commit] --> B[Security Scan]
 B --> C[Unit Tests]
 C --> D[Build Containers] 
 D --> E[Integration Tests]
 E --> F[Security Scanning]
 F --> G[Deploy to Staging]
 G --> H[End-to-End Tests]
 H --> I[Performance Tests]
 I --> J[Blue-Green Production]
 J --> K[Health Monitoring]
 K --> L[Rollback if Issues]

// Jenkinsfile
pipeline {
 agent none  environment {
 DOCKER_REGISTRY = "registry.company.com"
 IMAGE_TAG = "${BUILD_NUMBER}-${GIT_COMMIT.substring(0,8)}"
 SECURITY_SCAN_THRESHOLD = "HIGH"
 }  stages {
 stage('Parallel Security & Build') {
 parallel {
 stage('Security Scanning') {
 agent { label 'security-scanner' }
 steps {
 script {
 securityScan()
 }
 }
 }  stage('Build Preparation') {
 agent { label 'docker-builder' }
 steps {
 checkout scm
 script {
 prepareBuildEnvironment()
 }
 }
 }
 }
 }  stage('Container Build') {
 agent { label 'docker-builder' }
 steps {
 script {
 buildAllContainers()
 }
 }
 }  stage('Testing Suite') {
 parallel {
 stage('Unit Tests') {
 agent { label 'test-runner' }
 steps { runUnitTests() }
 }  stage('Integration Tests') {
 agent { label 'integration-test' }
 steps { runIntegrationTests() }
 }  stage('Security Container Scan') {
 agent { label 'security-scanner' }
 steps { scanContainerImages() }
 }
 }
 }  stage('Staging Deployment') {
 agent { label 'deploy-staging' }
 steps {
 script {
 deployToStaging()
 runStagingTests()
 }
 }
 }  stage('Production Deployment') {
 when {
 allOf {
 branch 'main'
 expression { currentBuild.result != 'FAILURE' }
 }
 }
 agent { label 'deploy-production' }
 steps {
 script {
 deployToProduction()
 }
 }
 post {
 failure {
 script {
 rollbackProduction()
 }
 }
 }
 }
 }  post {
 always {
 cleanupWorkspace()
 }
 failure {
 notifyTeam('FAILURE')
 }
 success {
 notifyTeam('SUCCESS')
 }
 }
}

def securityScan() {
 // Source code security scanning
 sh """
 # SAST scanning with SonarQube
 sonar-scanner \
 -Dsonar.projectKey=ims-cscf \
 -Dsonar.sources=. \
 -Dsonar.host.url=${SONAR_URL} \
 -Dsonar.login=${SONAR_TOKEN}
 """  // Dependency vulnerability scanning
 sh """
 # Check for known vulnerabilities in dependencies
 safety check --json --output vulnerabilities.json  # Fail build if high severity vulnerabilities found
 python3 check_vulnerabilities.py --threshold ${SECURITY_SCAN_THRESHOLD}
 """  // License compliance checking
 sh """
 # Ensure all dependencies have compatible licenses
 pip-licenses --format=json --output-file licenses.json
 python3 validate_licenses.py
 """
} def scanContainerImages() {
 // Container image security scanning with Trivy
 sh """
 for image in ims dns mysql; do
 trivy image --severity HIGH,CRITICAL \
 --format json \
 --output \${image}-scan.json \
 ${DOCKER_REGISTRY}/ims-\${image}:${IMAGE_TAG}  # Fail if critical vulnerabilities found
 if [ \$(jq '.Results[].Vulnerabilities | length' \${image}-scan.json) -gt 0 ]; then
 echo "Critical vulnerabilities found in \${image} image"
 exit 1
 fi
 done
 """
}

def buildAllContainers() {
 // Parallel container builds to optimize pipeline time
 parallel {
 'ims-container': {
 buildContainer('ims', './ims')
 },
 'dns-container': {
 buildContainer('dns', './dns') 
 },
 'mysql-container': {
 buildContainer('mysql', './mysql')
 }
 }
} def buildContainer(String serviceName, String contextPath) {
 sh """
 # Build with consistent tagging strategy
 docker build \
 --build-arg BUILD_NUMBER=${BUILD_NUMBER} \
 --build-arg GIT_COMMIT=${GIT_COMMIT} \
 --build-arg BUILD_DATE=\$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
 --tag ${DOCKER_REGISTRY}/ims-${serviceName}:${IMAGE_TAG} \
 --tag ${DOCKER_REGISTRY}/ims-${serviceName}:latest \
 ${contextPath}  # Push to registry
 docker push ${DOCKER_REGISTRY}/ims-${serviceName}:${IMAGE_TAG}
 docker push ${DOCKER_REGISTRY}/ims-${serviceName}:latest
 """  // Scan image immediately after build
 sh """
 docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \
 aquasec/trivy:latest image \
 --severity HIGH,CRITICAL \
 ${DOCKER_REGISTRY}/ims-${serviceName}:${IMAGE_TAG}
 """
}

# Optimized Dockerfile with build caching
FROM ubuntu:20.04 AS builder # Cache package lists
RUN --mount=type=cache,target=/var/cache/apt \
 --mount=type=cache,target=/var/lib/apt \
 apt-get update && apt-get install -y \
 build-essential gcc make # Cache compiled dependencies 
COPY requirements.txt /tmp/
RUN --mount=type=cache,target=/root/.cache/pip \
 pip install -r /tmp/requirements.txt # Build application
COPY src/ /build/
WORKDIR /build
RUN --mount=type=cache,target=/build/cache \
 make clean && make all FROM ubuntu:20.04 AS runtime
# ... runtime configuration

def runUnitTests() {
 sh """
 # Unit tests for configuration validation
 python3 -m pytest tests/unit/ \
 --junitxml=unit-tests.xml \
 --cov=src/ \
 --cov-report=xml  # Configuration syntax validation
 for config in ims/files/kamailio_*/*.cfg.tpl; do
 kamailio -c -f \$config
 done
 """  publishTestResults testResultsPattern: 'unit-tests.xml'
 publishCoverageResults()
} def runIntegrationTests() {
 sh """
 # Start test environment
 docker-compose -f docker-compose.test.yml up -d  # Wait for services to be ready
 timeout 60 bash -c 'until docker-compose -f docker-compose.test.yml \
 exec icscf kamctl stats >/dev/null 2>&1; do sleep 2; done'  # Run integration tests
 python3 -m pytest tests/integration/ \
 --junitxml=integration-tests.xml  # Cleanup
 docker-compose -f docker-compose.test.yml down -v
 """  publishTestResults testResultsPattern: 'integration-tests.xml'
}

#!/bin/bash
# performance-tests.sh # SIP call load testing
sipp -sn uac \
 -s 1234567890 \
 -d 10000 \
 -l 100 \
 -r 10 \
 -rp 1000 \
 ${PCSCF_IP}:5060 # Database performance testing 
sysbench oltp_read_write \
 --mysql-host=${MYSQL_IP} \
 --mysql-port=3306 \
 --mysql-user=test \
 --mysql-password=<REDACTED> \
 --mysql-db=icscf \
 --tables=10 \
 --table-size=10000 \
 --threads=16 \
 --time=60 \
 run # Network latency testing
for service in icscf pcscf scscf; do
 curl -o /dev/null -s -w "Connect: %{time_connect}s\nTotal: %{time_total}s\n" \
 http://${service}:8080/health
done

def deployToProduction() {
 // Determine current active environment
 def currentEnv = getCurrentProductionEnvironment()
 def targetEnv = (currentEnv == 'blue') ? 'green' : 'blue'  echo "Current production: ${currentEnv}, deploying to: ${targetEnv}"  try {
 // Deploy to inactive environment
 sh """
 # Update environment-specific configuration
 export TARGET_ENV=${targetEnv}
 envsubst < docker-compose.${targetEnv}.yml.tpl > docker-compose.${targetEnv}.yml  # Deploy new version
 docker-compose -f docker-compose.${targetEnv}.yml up -d
 """  // Health check new environment
 waitForHealthy(targetEnv)  // Run production smoke tests
 runSmokeTests(targetEnv)  // Switch traffic to new environment
 switchTraffic(targetEnv)  // Monitor for issues
 monitorHealth(targetEnv, 300) // 5 minute monitoring window  // Cleanup old environment if successful
 sh "docker-compose -f docker-compose.${currentEnv}.yml down"  } catch (Exception e) {
 echo "Deployment failed: ${e.message}"
 rollbackProduction(currentEnv)
 throw e
 }
} def waitForHealthy(String environment) {
 timeout(time: 10, unit: 'MINUTES') {
 waitUntil {
 script {
 def result = sh(
 script: "docker-compose -f docker-compose.${environment}.yml ps --services | xargs -I {} docker-compose -f docker-compose.${environment}.yml exec {} health-check.sh",
 returnStatus: true
 )
 return result == 0
 }
 }
 }
} def switchTraffic(String targetEnv) {
 // Update load balancer configuration
 sh """
 # Update HAProxy configuration
 sed -i 's/server.*:5060/server ${targetEnv} ${targetEnv}-pcscf:5060 check/' /etc/haproxy/haproxy.cfg  # Gracefully reload HAProxy
 systemctl reload haproxy  # Update DNS records if needed
 aws route53 change-resource-record-sets \
 --hosted-zone-id ${HOSTED_ZONE_ID} \
 --change-batch file://dns-update-${targetEnv}.json
 """
}

def canaryDeployment() {
 // Deploy to canary environment (10% traffic)
 deployCanary()  // Monitor key metrics for 30 minutes
 def metricsOk = monitorCanaryMetrics(30)  if (metricsOk) {
 // Gradually increase traffic
 increaseCanaryTraffic([25, 50, 75, 100])
 } else {
 // Rollback canary deployment
 rollbackCanary()
 currentBuild.result = 'FAILURE'
 }
} def monitorCanaryMetrics(int durationMinutes) {
 // Monitor critical metrics using Prometheus queries
 def queries = [
 'call_success_rate': 'rate(sip_calls_successful[5m]) / rate(sip_calls_total[5m])',
 'response_time_p95': 'histogram_quantile(0.95, rate(sip_response_time_bucket[5m]))',
 'error_rate': 'rate(sip_errors_total[5m])'
 ]  for (int i = 0; i < durationMinutes; i++) {
 queries.each { metric, query ->
 def value = queryPrometheus(query)
 if (!isMetricHealthy(metric, value)) {
 echo "Metric ${metric} failed threshold: ${value}"
 return false
 }
 }
 sleep(60) // Wait 1 minute between checks
 }
 return true
}

def collectPipelineMetrics() {
 // Collect build metrics
 def buildMetrics = [
 build_duration: currentBuild.duration,
 test_count: getTestCount(),
 coverage_percentage: getCoverage(),
 vulnerability_count: getVulnerabilityCount(),
 deployment_success: currentBuild.result == 'SUCCESS'
 ]  // Send metrics to monitoring system
 sh """
 curl -X POST ${METRICS_ENDPOINT} \
 -H 'Content-Type: application/json' \
 -d '${JsonOutput.toJson(buildMetrics)}'
 """
} def setupApplicationMonitoring() {
 // Deploy monitoring stack alongside application
 sh """
 # Deploy Prometheus configuration
 kubectl apply -f monitoring/prometheus-config.yaml  # Deploy Grafana dashboards
 kubectl apply -f monitoring/grafana-dashboards.yaml  # Configure alerting rules
 kubectl apply -f monitoring/alert-rules.yaml
 """
}

#!/bin/bash
# health-monitor.sh - Continuous health monitoring SERVICES=("icscf" "pcscf" "scscf" "dns" "mysql")
ALERT_WEBHOOK="${ALERT_WEBHOOK_URL}" while true; do
 for service in "${SERVICES[@]}"; do
 # Check service health
 if ! docker-compose exec $service health-check.sh; then
 # Service unhealthy, trigger alert
 curl -X POST $ALERT_WEBHOOK \
 -H 'Content-Type: application/json' \
 -d "{\"service\": \"$service\", \"status\": \"unhealthy\", \"timestamp\": \"$(date -Iseconds)\"}"  # Attempt automatic recovery
 echo "Attempting to restart $service..."
 docker-compose restart $service  # Wait for recovery
 sleep 30  # Check if recovery was successful
 if docker-compose exec $service health-check.sh; then
 curl -X POST $ALERT_WEBHOOK \
 -H 'Content-Type: application/json' \
 -d "{\"service\": \"$service\", \"status\": \"recovered\", \"timestamp\": \"$(date -Iseconds)\"}"
 fi
 fi
 done  sleep 60 # Check every minute
done

pipeline {
 environment {
 // Use Jenkins credentials binding
 MYSQL_PASSWORD = credentials('mysql-production-password')
 TLS_CERTIFICATE = credentials('ims-tls-certificate')
 DOCKER_REGISTRY_TOKEN = credentials('docker-registry-token')
 }  stages {
 stage('Deploy') {
 steps {
 withCredentials([
 usernamePassword(credentialsId: 'mysql-credentials', 
 usernameVariable: 'DB_USER', 
 passwordVariable: 'DB_PASS')
 ]) {
 script {
 // Secrets available as environment variables
 deployWithSecrets()
 }
 }
 }
 }
 }
}

def auditDeployment() {
 // Record deployment event
 def auditEvent = [
 timestamp: new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'"),
 user: env.BUILD_USER_ID ?: 'system',
 action: 'deployment',
 environment: env.DEPLOY_ENVIRONMENT,
 version: env.IMAGE_TAG,
 commit: env.GIT_COMMIT,
 pipeline_id: env.BUILD_ID
 ]  // Store audit record
 sh """
 echo '${JsonOutput.toJson(auditEvent)}' >> /var/log/deployment-audit.log  # Also send to centralized audit system
 curl -X POST ${AUDIT_SYSTEM_URL} \
 -H 'Content-Type: application/json' \
 -H 'Authorization: Bearer <REDACTED>
 -d '${JsonOutput.toJson(auditEvent)}'
 """
}

# environments/production.yml
environments:
 production:
 replicas:
 icscf: 3
 pcscf: 5
 scscf: 3
 resources:
 icscf:
 cpu: "2.0"
 memory: "2Gi"
 pcscf:
 cpu: "4.0"
 memory: "4Gi"
 networking:
 subnet: "10.0.0.0/16"
 load_balancer: true
 security:
 tls_enabled: true
 security_scanning: true
 monitoring:
 enabled: true
 retention_days: 90

def loadEnvironmentConfig(String environment) {
 def config = readYaml file: "environments/${environment}.yml"  // Set environment variables based on configuration
 config.environments[environment].each { key, value ->
 if (value instanceof Map) {
 value.each { subkey, subvalue ->
 env["${key.toUpperCase()}_${subkey.toUpperCase()}"] = subvalue.toString()
 }
 } else {
 env[key.toUpperCase()] = value.toString()
 }
 }
}

pipeline {
 options {
 // Build caching
 buildDiscarder(logRotator(numToKeepStr: '50'))  // Workspace caching
 skipStagesAfterUnstable()  // Artifact caching
 artifactNumToKeep(10)
 }  stages {
 stage('Build') {
 steps {
 // Docker layer caching
 sh '''
 # Enable Docker BuildKit for better caching
 export DOCKER_BUILDKIT=1  # Use cache from previous builds
 docker build \
 --cache-from ${DOCKER_REGISTRY}/ims-build-cache:latest \
 --build-arg BUILDKIT_INLINE_CACHE=1 \
 -t ${DOCKER_REGISTRY}/ims-build-cache:latest \
 .
 '''
 }
 }
 }
}

def monitorDeployment() {
 // Monitor key metrics after deployment
 def monitoringDuration = 600 // 10 minutes
 def checkInterval = 30 // 30 seconds
 def checks = monitoringDuration / checkInterval  for (int i = 0; i < checks; i++) {
 def metrics = collectHealthMetrics()  if (isDeploymentUnhealthy(metrics)) {
 echo "Unhealthy deployment detected, initiating rollback"
 rollbackProduction()
 currentBuild.result = 'FAILURE'
 break
 }  sleep(checkInterval)
 }
} def rollbackProduction() {
 // Get previous successful deployment
 def previousVersion = getPreviousSuccessfulVersion()  echo "Rolling back to version: ${previousVersion}"  // Rollback using blue-green switch
 sh """
 # Switch traffic back to previous environment
 kubectl patch service ims-gateway \
 -p '{"spec":{"selector":{"version":"${previousVersion}"}}}'  # Scale down failed deployment
 kubectl scale deployment ims-current --replicas=0  # Update deployment status
 kubectl annotate deployment ims-current \
 rollback.reason="Health check failed" \
 rollback.timestamp="$(date -Iseconds)" \
 rollback.initiator="${BUILD_USER_ID}"
 """  // Notify team of rollback
 notifyRollback(previousVersion)
}

def optimizeResources() {
 // Analyze resource usage patterns
 sh """
 # Collect resource metrics
 kubectl top pods --containers > resource-usage.txt  # Generate optimization recommendations
 python3 scripts/resource-optimizer.py \
 --usage-file resource-usage.txt \
 --recommendations-file recommendations.json
 """  // Apply recommendations if auto-optimization enabled
 if (env.AUTO_OPTIMIZE_RESOURCES == 'true') {
 applyResourceOptimizations()
 }
}