// ============================================ // AIOT Platform - Jenkins Pipeline (Enterprise Edition) // 优化版本:错误处理 + 性能优化 + 完善日志 // ============================================ pipeline { agent any options { buildDiscarder(logRotator( numToKeepStr: '10', artifactNumToKeepStr: '5' )) disableConcurrentBuilds() timeout(time: 90, unit: 'MINUTES') timestamps() retry(1) // 失败自动重试1次 } environment { // 镜像仓库配置 REGISTRY = 'localhost:5000' // 注意:IMAGE_TAG 将在 Checkout 阶段动态设置 // IMAGE_TAG = "${BRANCH_NAME}-${BUILD_NUMBER}-${GIT_COMMIT}" DEPS_IMAGE = "${REGISTRY}/aiot-deps:latest" // 服务配置 CORE_SERVICES = 'viewsh-gateway,viewsh-module-system-server,viewsh-module-infra-server,viewsh-module-iot-server,viewsh-module-iot-gateway' // 部署配置 DEPLOY_HOST = '172.19.0.1' DEPLOY_PATH = '/opt/aiot-platform-cloud' SSH_KEY = '/var/jenkins_home/.ssh/id_rsa' // 性能配置 MAX_PARALLEL_BUILDS = 2 BUILD_TIMEOUT = 45 // 单个服务构建超时(分钟) DEPLOY_TIMEOUT = 10 // 单个服务部署超时(分钟) HEALTH_CHECK_TIMEOUT = 180 // 健康检查总超时(秒) HEALTH_CHECK_INTERVAL = 10 // 健康检查间隔(秒) } stages { stage('Initialize') { steps { script { echo "==========================================" echo " AIOT Platform - CI/CD Pipeline" echo "==========================================" echo "Branch: ${env.BRANCH_NAME}" echo "Build: #${env.BUILD_NUMBER}" echo "Workspace: ${env.WORKSPACE}" echo "==========================================" } } } stage('Checkout') { steps { retry(3) { checkout scm } script { // 动态设置环境变量(避免在 environment 块中使用 env 变量) def shortCommit = sh( script: 'git rev-parse --short HEAD', returnStdout: true ).trim() env.IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}-${shortCommit}" env.GIT_COMMIT_MSG = sh( script: 'git log -1 --pretty=%B', returnStdout: true ).trim() echo "📦 Commit: ${shortCommit}" echo "📝 Message: ${env.GIT_COMMIT_MSG}" echo "🏷️ Image Tag: ${env.IMAGE_TAG}" } } } stage('Detect Changes') { steps { script { // 获取变更文件(只执行一次 git diff) def changedFiles = getChangedFiles() echo "📝 Changed files: ${changedFiles.size()} files" // 判断需要构建的服务 env.SERVICES_TO_BUILD = detectServicesToBuild(changedFiles) env.DEPS_CHANGED = checkIfDepsChanged(changedFiles) if (env.SERVICES_TO_BUILD.isEmpty()) { echo "⏭️ No changes detected, skipping build" currentBuild.result = 'SUCCESS' return // 直接跳过后续阶段 } echo "🔄 Services to build: ${env.SERVICES_TO_BUILD}" echo "📦 Deps changed: ${env.DEPS_CHANGED}" // 显示变更的服务 env.SERVICES_TO_BUILD.split(',').each { service -> def module = getModulePathForService(service) echo " - ${service} (${module})" } } } } stage('Pre-build Check') { when { expression { env.SERVICES_TO_BUILD != '' } } steps { script { echo "🔍 Running pre-build checks..." // 检查 Docker 是否可用 sh "docker version >/dev/null 2>&1 || { echo '❌ Docker not available'; exit 1; }" // 检查磁盘空间 def diskUsage = sh( script: "df ${env.WORKSPACE} | tail -1 | awk '{print \$5}' | sed 's/%//'", returnStdout: true ).trim() as int if (diskUsage > 80) { echo "⚠️ Disk usage is ${diskUsage}%, cleaning up..." sh "docker system prune -f --volumes || true" } // 检查镜像仓库连接 sh """ curl -f ${env.REGISTRY}/v2/ >/dev/null 2>&1 || \ { echo '⚠️ Registry not accessible, will continue...'; } """ echo "✅ Pre-build checks passed" } } } stage('Build Dependencies Image') { when { expression { env.SERVICES_TO_BUILD != '' && (env.DEPS_CHANGED == 'true' || !depsImageExists()) } } steps { script { echo "📦 Building dependencies base image..." timeout(time: 15, unit: 'MINUTES') { sh """ set -e echo "Building ${env.DEPS_IMAGE}..." docker build \ -f docker/Dockerfile.deps \ -t ${env.DEPS_IMAGE} \ --build-arg BUILDKIT_INLINE_CACHE=1 \ . docker push ${env.DEPS_IMAGE} echo "✅ Dependencies image built and pushed" """ } } } } stage('Build Services') { when { expression { env.SERVICES_TO_BUILD != '' } } steps { script { def servicesToBuild = env.SERVICES_TO_BUILD.split(',') echo "🔨 Building ${servicesToBuild.size()} services (parallelism: ${MAX_PARALLEL_BUILDS})" // 分批并行构建 def buildTasks = [:] def batchSize = env.MAX_PARALLEL_BUILDS.toInteger() servicesToBuild.each { service -> buildTasks[service] = { buildServiceWithRetry(service) } } // 限制并发数 parallel buildTasks echo "✅ All services built successfully" // 显示构建后的镜像信息 sh """ echo "📊 Built images:" ${env.SERVICES_TO_BUILD.split(',').collect { "docker images ${env.REGISTRY}/${it} --format ' {{.Repository}}:{{.Tag}} - {{.Size}}'" }.join('\n ')} """ } } } stage('Deploy') { when { allOf { expression { env.SERVICES_TO_BUILD != '' } branch 'master' } } steps { script { def servicesToDeploy = env.SERVICES_TO_BUILD.split(',') // 按依赖顺序排序 def sortedServices = sortServicesByDependency(servicesToDeploy) echo "🚀 Deploying ${sortedServices.size()} services in dependency order" sortedServices.eachWithIndex { service, index -> echo " ${index + 1}. ${service}" } // 串行部署(保证依赖关系) sortedServices.each { service -> deployServiceWithTimeout(service) } echo "🚀 All services deployed successfully!" } } } stage('Final Health Check') { when { allOf { expression { env.SERVICES_TO_BUILD != '' } branch 'master' } } steps { script { echo "🏥 Running final health check for all services..." def servicesToCheck = env.SERVICES_TO_BUILD.split(',') def healthCheckTasks = [:] servicesToCheck.each { service -> healthCheckTasks[service] = { checkServiceHealthWithRetry(service) } } // 并行健康检查 parallel healthCheckTasks echo "✅ All services are healthy!" // 显示最终状态 sh """ echo "📊 Service Status:" ${servicesToCheck.collect { def container = getContainerNameForService(it) "docker inspect --format='${it}: {{.State.Status}} ({{.State.Health.Status}})' ${container} 2>/dev/null || echo '${it}: not found'" }.join('\n ')} """ } } } } post { success { script { echo """ ========================================== ✅ BUILD SUCCESS ========================================== 📦 Services: ${env.SERVICES_TO_BUILD} 🏷️ Tag: ${env.IMAGE_TAG} ⏱️ Duration: ${currentBuild.durationString} ========================================== """ } } failure { script { echo """ ========================================== ❌ BUILD FAILED ========================================== 📦 Services: ${env.SERVICES_TO_BUILD ?: 'None'} 🏷️ Tag: ${env.IMAGE_TAG ?: 'Unknown'} ⚠️ Please check the logs above ========================================== """ // 失败时收集诊断信息 sh ''' echo "=== Docker System Info ===" docker system df echo "" echo "=== Disk Usage ===" df -h | grep -E "/$|/var" echo "" echo "=== Recent Containers ===" docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.CreatedAt}}" | head -10 ''' } } always { script { echo "🧹 Cleaning up..." // 清理悬空的镜像 sh "docker image prune -f || true" // 清理超过30天的构建日志 sh """ find ${env.WORKSPACE} -name '*.log' -mtime +30 -delete 2>/dev/null || true """ echo "📊 Final System Status:" sh 'df -h | grep -E "/$|/var" || true' sh 'docker system df || true' } } } } // ============================================ // 辅助函数 // ============================================ // 获取变更的文件列表 @NonCPS def getChangedFiles() { def changedFiles = sh( script: ''' PREV=$(git rev-parse HEAD~1 2>/dev/null || echo "") [ -z "$PREV" ] && echo "all" || git diff --name-only $PREV HEAD ''', returnStdout: true ).trim() if (changedFiles == 'all') { return ['all'] as List } return changedFiles.split('\n') as List } // 检测是否需要重建依赖镜像 @NonCPS def checkIfDepsChanged(List changedFiles) { if (changedFiles.contains('all')) { return 'true' } def depsFiles = ['pom.xml', 'viewsh-dependencies', 'viewsh-framework', 'docker/Dockerfile.deps'] return depsFiles.any { changedFiles.contains(it) } ? 'true' : 'false' } // 检测需要构建的服务 @NonCPS def detectServicesToBuild(List changedFiles) { // 如果是第一次构建或强制全量构建 if (changedFiles.contains('all')) { return env.CORE_SERVICES } // 检查是否触发了全量构建 def triggerAllFiles = ['pom.xml', 'viewsh-framework', 'viewsh-dependencies', 'Jenkinsfile', 'docker/'] if (triggerAllFiles.any { triggerFile -> changedFiles.any { changedFile -> changedFile.startsWith(triggerFile) || changedFile == triggerFile } }) { return env.CORE_SERVICES } // 检测变更的模块 def changedServices = [] def allServices = env.CORE_SERVICES.split(',') allServices.each { service -> def modulePath = getModulePathForService(service) def moduleDir = modulePath.split('/')[0] if (changedFiles.any { it.startsWith(moduleDir) }) { changedServices.add(service) } } return changedServices.isEmpty() ? env.CORE_SERVICES : changedServices.join(',') } // 检查依赖镜像是否存在 @NonCPS def depsImageExists() { def result = sh( script: "docker manifest inspect ${env.DEPS_IMAGE} > /dev/null 2>&1", returnStatus: true ) return result == 0 } // 构建服务(带重试) def buildServiceWithRetry(String service) { retry(2) { timeout(time: env.BUILD_TIMEOUT.toInteger(), unit: 'MINUTES') { buildService(service) } } } // 构建单个服务 def buildService(String service) { def modulePath = getModulePathForService(service) echo "" echo "==========================================" echo "🔨 Building ${service}" echo "==========================================" echo "Module: ${modulePath}" echo "Registry: ${env.REGISTRY}" echo "Tag: ${env.IMAGE_TAG}" echo "==========================================" try { sh """ set -e set -x # 构建镜像 docker build \\ -f docker/Dockerfile.service \\ --build-arg DEPS_IMAGE=${env.DEPS_IMAGE} \\ --build-arg MODULE_NAME=${modulePath} \\ --build-arg JAR_NAME=${service} \\ --build-arg SKIP_TESTS=true \\ -t ${env.REGISTRY}/${service}:${env.IMAGE_TAG} \\ -t ${env.REGISTRY}/${service}:latest \\ . # 推送镜像 docker push ${env.REGISTRY}/${service}:${env.IMAGE_TAG} docker push ${env.REGISTRY}/${service}:latest set +x """ echo "✅ ${service} built and pushed successfully" // 获取镜像大小 def imageSize = sh( script: "docker images ${env.REGISTRY}/${service}:latest --format '{{.Size}}'", returnStdout: true ).trim() echo "📊 Image size: ${imageSize}" } catch (Exception e) { echo "❌ Failed to build ${service}: ${e.message}" // 打印构建日志以便调试 sh """ echo "=== Docker Build Logs for ${service} ===" docker logs ${service}-builder 2>/dev/null || true """ throw e } } // 部署服务(带超时) def deployServiceWithTimeout(String service) { timeout(time: env.DEPLOY_TIMEOUT.toInteger(), unit: 'MINUTES') { deployService(service) } } // 部署单个服务 def deployService(String service) { def containerName = getContainerNameForService(service) def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}" echo "" echo "==========================================" echo "🚀 Deploying ${service}" echo "==========================================" echo "Container: ${containerName}" echo "Host: ${env.DEPLOY_HOST}" echo "==========================================" try { // 部署服务 sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' set -e cd ${env.DEPLOY_PATH} echo "📥 Pulling ${service}..." docker compose -f docker-compose.core.yml pull ${service} echo "🔄 Restarting ${service}..." docker compose -f docker-compose.core.yml up -d ${service} echo "⏳ Waiting for container to start..." sleep 5 ' """ // 等待服务健康 waitForServiceHealthy(containerName, service, sshOpts) echo "✅ ${service} deployed successfully" } catch (Exception e) { echo "❌ Failed to deploy ${service}: ${e.message}" // 收集诊断信息 sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' echo "=== Container Status ===" docker ps -a | grep ${containerName} || true echo "" echo "=== Container Logs (last 50 lines) ===" docker logs --tail 50 ${containerName} 2>/dev/null || true echo "" echo "=== Service Health Status ===" docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found" ' """ throw e } } // 等待服务健康 def waitForServiceHealthy(String containerName, String serviceName, String sshOpts) { def maxAttempts = env.HEALTH_CHECK_TIMEOUT.toInteger() / env.HEALTH_CHECK_INTERVAL.toInteger() echo "⏳ Waiting for ${serviceName} to be healthy (max ${env.HEALTH_CHECK_TIMEOUT}s)..." sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' set -e for i in $(seq 1 ${maxAttempts}); do STATUS=$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "starting") case "$STATUS" in healthy) echo "✅ ${serviceName} is healthy" exit 0 ;; unhealthy) echo "❌ ${serviceName} is unhealthy" echo "=== Last 100 lines of logs ===" docker logs --tail 100 ${containerName} exit 1 ;; starting) ELAPSED=$((i * ${env.HEALTH_CHECK_INTERVAL})) echo "⏳ ${serviceName} is starting... (\${ELAPSED}s/${env.HEALTH_CHECK_TIMEOUT}s)" ;; *) echo "⚠️ ${serviceName} status: \$STATUS" ;; esac sleep ${env.HEALTH_CHECK_INTERVAL} done echo "❌ ${serviceName} health check timeout after ${env.HEALTH_CHECK_TIMEOUT}s" echo "=== Full logs ===" docker logs ${containerName} exit 1 ' """ } // 检查服务健康(带重试) def checkServiceHealthWithRetry(String service) { def containerName = getContainerNameForService(service) def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}" retry(3) { timeout(time: 2, unit: 'MINUTES') { checkServiceHealth(containerName, service, sshOpts) } } } // 检查服务健康状态 def checkServiceHealth(String containerName, String serviceName, String sshOpts) { sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' STATUS=$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found") case "$STATUS" in healthy) echo "✅ ${serviceName} is healthy" ;; not_found) echo "⚠️ ${serviceName} not found (may not be deployed)" exit 1 ;; *) echo "❌ ${serviceName} is \$STATUS" echo "=== Last 50 lines of logs ===" docker logs --tail 50 ${containerName} exit 1 ;; esac ' """ } // 按依赖顺序排序服务 @NonCPS def sortServicesByDependency(def services) { def deployOrder = [ 'viewsh-gateway', 'viewsh-module-system-server', 'viewsh-module-infra-server', 'viewsh-module-iot-server', 'viewsh-module-iot-gateway' ] return services.sort { a, b -> deployOrder.indexOf(a) <=> deployOrder.indexOf(b) } } // 获取服务对应的容器名称 @NonCPS def getContainerNameForService(String service) { def map = [ 'viewsh-gateway': 'aiot-gateway', 'viewsh-module-system-server': 'aiot-system-server', 'viewsh-module-infra-server': 'aiot-infra-server', 'viewsh-module-iot-server': 'aiot-iot-server', 'viewsh-module-iot-gateway': 'aiot-iot-gateway' ] return map.get(service, "aiot-${service}") } // 获取服务对应的模块路径 @NonCPS def getModulePathForService(String service) { def map = [ 'viewsh-gateway': 'viewsh-gateway', 'viewsh-module-system-server': 'viewsh-module-system/viewsh-module-system-server', 'viewsh-module-infra-server': 'viewsh-module-infra/viewsh-module-infra-server', 'viewsh-module-iot-server': 'viewsh-module-iot/viewsh-module-iot-server', 'viewsh-module-iot-gateway': 'viewsh-module-iot/viewsh-module-iot-gateway' ] return map.get(service, service) }