// ============================================ // AIOT Platform - Jenkins Pipeline (Optimized Edition) // ============================================ pipeline { agent any options { buildDiscarder(logRotator( numToKeepStr: '10', artifactNumToKeepStr: '5' )) disableConcurrentBuilds() timeout(time: 90, unit: 'MINUTES') timestamps() retry(1) } environment { // 镜像仓库配置(Infra 服务器内网地址,Prod 服务器可通过内网拉取) REGISTRY = '172.17.16.7:5000' DEPS_IMAGE = "${REGISTRY}/aiot-deps:latest" // 服务配置 CORE_SERVICES = 'viewsh-gateway,viewsh-module-system-server,viewsh-module-infra-server,viewsh-module-iot-server,viewsh-module-iot-gateway,viewsh-module-ops-server' // 部署配置(Prod 服务器内网地址) DEPLOY_HOST = '172.17.16.14' DEPLOY_PATH = '/opt/aiot-platform-cloud' SSH_KEY = '/var/jenkins_home/.ssh/id_rsa' // 性能配置 - 将动态调整 BUILD_TIMEOUT = 45 DEPLOY_TIMEOUT = 10 HEALTH_CHECK_TIMEOUT = 180 HEALTH_CHECK_INTERVAL = 10 // 【优化1】Maven 缓存配置 MAVEN_CACHE_VOLUME = 'jenkins-maven-cache' MAVEN_OPTS = '-Dmaven.repo.local=/var/jenkins_home/.m2/repository' // 【优化7】回滚配置 ROLLBACK_ENABLED = 'true' KEEP_PREVIOUS_IMAGES = 'true' } stages { stage('Initialize') { steps { script { // 【优化6】记录开始时间 env.PIPELINE_START_TIME = System.currentTimeMillis() echo "==========================================" echo " AIOT Platform - CI/CD Pipeline (Optimized)" echo "==========================================" echo "Branch: ${env.BRANCH_NAME}" echo "Build: #${env.BUILD_NUMBER}" echo "Workspace: ${env.WORKSPACE}" echo "Start Time: ${new Date()}" echo "==========================================" // 【优化2】动态检测系统资源 detectSystemResources() } } } stage('Checkout') { steps { script { def stageStartTime = System.currentTimeMillis() retry(3) { checkout scm } def shortCommit = sh( script: 'git rev-parse --short HEAD', returnStdout: true ).trim() // 清理分支名:将斜杠替换为连字符,确保 Docker 标签格式有效 def sanitizedBranchName = env.BRANCH_NAME.replaceAll('/', '-') env.IMAGE_TAG = "${sanitizedBranchName}-${env.BUILD_NUMBER}-${shortCommit}" env.PREVIOUS_IMAGE_TAG = getPreviousImageTag() env.GIT_COMMIT_MSG = sh( script: 'git log -1 --pretty=%B', returnStdout: true ).trim() echo "📦 Commit: ${shortCommit}" echo "📝 Message: ${env.GIT_COMMIT_MSG}" echo "🏷️ Current Tag: ${env.IMAGE_TAG}" echo "🔖 Previous Tag: ${env.PREVIOUS_IMAGE_TAG}" // 【优化6】记录阶段耗时 recordStageMetrics('Checkout', stageStartTime) } } } stage('Detect Changes') { steps { script { def stageStartTime = System.currentTimeMillis() def changedFiles = getChangedFiles() echo "📝 Changed files: ${changedFiles.size()} files" env.SERVICES_TO_BUILD = detectServicesToBuild(changedFiles) env.DEPS_CHANGED = checkIfDepsChanged(changedFiles) // Check if deps image exists if (env.DEPS_CHANGED == 'false') { def depsExists = sh( script: "docker manifest inspect ${env.DEPS_IMAGE} > /dev/null 2>&1", returnStatus: true ) if (depsExists != 0) { env.DEPS_CHANGED = 'true' echo "📦 Deps image not found, will rebuild" } } if (env.SERVICES_TO_BUILD.isEmpty()) { echo "⏭️ No changes detected, skipping build" currentBuild.result = 'SUCCESS' return } echo "🔄 Services to build: ${env.SERVICES_TO_BUILD}" echo "📦 Deps changed: ${env.DEPS_CHANGED}" env.SERVICES_TO_BUILD.split(',').each { service -> def module = getModulePathForService(service) echo " - ${service} (${module})" } recordStageMetrics('Detect Changes', stageStartTime) } } } stage('Pre-build Check') { when { expression { env.SERVICES_TO_BUILD != '' } } steps { script { def stageStartTime = System.currentTimeMillis() echo "🔍 Running pre-build checks..." // 检查 Docker sh "docker version >/dev/null 2>&1 || { echo '❌ Docker not available'; exit 1; }" // 【优化1】检查并创建 Maven 缓存卷 sh """ if ! docker volume inspect ${env.MAVEN_CACHE_VOLUME} >/dev/null 2>&1; then echo "📦 Creating Maven cache volume: ${env.MAVEN_CACHE_VOLUME}" docker volume create ${env.MAVEN_CACHE_VOLUME} else echo "✅ Maven cache volume exists: ${env.MAVEN_CACHE_VOLUME}" fi """ // 检查磁盘空间 def diskUsage = sh( script: "df ${env.WORKSPACE} | tail -1 | awk '{print \$5}' | sed 's/%//'", returnStdout: true ).trim() as int if (diskUsage > 80) { echo "⚠️ Disk usage is ${diskUsage}%, cleaning up..." sh "docker system prune -f --volumes || true" } // 检查镜像仓库连接 sh """ curl -f ${env.REGISTRY}/v2/ >/dev/null 2>&1 || \ { echo '⚠️ Registry not accessible, will continue...'; } """ echo "✅ Pre-build checks passed" recordStageMetrics('Pre-build Check', stageStartTime) } } } stage('Build Dependencies Image') { when { expression { env.SERVICES_TO_BUILD != '' && env.DEPS_CHANGED == 'true' } } steps { script { def stageStartTime = System.currentTimeMillis() echo "📦 Building dependencies base image with Maven cache..." timeout(time: 15, unit: 'MINUTES') { // 【优化1】使用 Maven 缓存卷加速依赖下载 sh """ set -e echo "Building ${env.DEPS_IMAGE} with cache..." docker build \ -f docker/Dockerfile.deps \ -t ${env.DEPS_IMAGE} \ --build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg MAVEN_OPTS="${env.MAVEN_OPTS}" \ . docker push ${env.DEPS_IMAGE} echo "✅ Dependencies image built and pushed" """ } recordStageMetrics('Build Dependencies Image', stageStartTime) } } } stage('Build Services') { when { expression { env.SERVICES_TO_BUILD != '' } } steps { script { def stageStartTime = System.currentTimeMillis() def servicesToBuild = env.SERVICES_TO_BUILD.split(',') echo "🔨 Building ${servicesToBuild.size()} services (parallelism: ${env.MAX_PARALLEL_BUILDS})" // 【优化2】动态并行构建 def buildTasks = [:] servicesToBuild.each { service -> buildTasks[service] = { buildServiceWithRetry(service) } } parallel buildTasks echo "✅ All services built successfully" // 显示构建后的镜像信息 sh """ echo "📊 Built images:" ${env.SERVICES_TO_BUILD.split(',').collect { "docker images ${env.REGISTRY}/${it} --format ' {{.Repository}}:{{.Tag}} - {{.Size}}'" }.join('\n ')} """ recordStageMetrics('Build Services', stageStartTime) } } } stage('Deploy') { when { allOf { expression { env.SERVICES_TO_BUILD != '' } branch 'master' } } steps { script { def stageStartTime = System.currentTimeMillis() def servicesToDeploy = env.SERVICES_TO_BUILD.split(',') def sortedServices = sortServicesByDependency(servicesToDeploy) echo "🚀 Deploying ${sortedServices.size()} services in dependency order" sortedServices.eachWithIndex { service, index -> echo " ${index + 1}. ${service}" } // 【优化7】部署前备份当前镜像标签 if (env.ROLLBACK_ENABLED == 'true') { backupCurrentDeployment(sortedServices) } // 【新增】同步最新的 docker-compose.core.yml 到部署服务器 echo "📂 Syncing docker-compose.core.yml to deploy host..." sh "scp -o StrictHostKeyChecking=no -i ${env.SSH_KEY} docker-compose.core.yml root@${env.DEPLOY_HOST}:${env.DEPLOY_PATH}/" try { // 串行部署(保证依赖关系) sortedServices.each { service -> deployServiceWithTimeout(service) } echo "🚀 All services deployed successfully!" } catch (Exception e) { // 【优化7】部署失败时自动回滚 if (env.ROLLBACK_ENABLED == 'true') { echo "❌ Deployment failed: ${e.message}" echo "🔄 Initiating automatic rollback..." rollbackDeployment(sortedServices) } throw e } recordStageMetrics('Deploy', stageStartTime) } } } stage('Final Health Check') { when { allOf { expression { env.SERVICES_TO_BUILD != '' } branch 'master' } } steps { script { def stageStartTime = System.currentTimeMillis() echo "🏥 Running final health check for all services..." def servicesToCheck = env.SERVICES_TO_BUILD.split(',') def healthCheckTasks = [:] servicesToCheck.each { service -> healthCheckTasks[service] = { checkServiceHealthWithRetry(service) } } try { parallel healthCheckTasks echo "✅ All services are healthy!" } catch (Exception e) { // 【优化7】健康检查失败时回滚 if (env.ROLLBACK_ENABLED == 'true') { echo "❌ Health check failed: ${e.message}" echo "🔄 Initiating automatic rollback..." rollbackDeployment(servicesToCheck) } throw e } // 显示最终状态 sh """ echo "📊 Service Status:" ${servicesToCheck.collect { def container = getContainerNameForService(it) "docker inspect --format='${it}: {{.State.Status}} ({{.State.Health.Status}})' ${container} 2>/dev/null || echo '${it}: not found'" }.join('\n ')} """ recordStageMetrics('Final Health Check', stageStartTime) } } } } post { success { script { // 【优化6】计算总耗时并生成性能报告 def totalDuration = System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong() generatePerformanceReport(totalDuration) echo """ ========================================== ✅ BUILD SUCCESS ========================================== 📦 Services: ${env.SERVICES_TO_BUILD} 🏷️ Tag: ${env.IMAGE_TAG} ⏱️ Duration: ${formatDuration(totalDuration)} ========================================== """ } } failure { script { def totalDuration = System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong() echo """ ========================================== ❌ BUILD FAILED ========================================== 📦 Services: ${env.SERVICES_TO_BUILD ?: 'None'} 🏷️ Tag: ${env.IMAGE_TAG ?: 'Unknown'} 🔖 Rollback Tag: ${env.PREVIOUS_IMAGE_TAG ?: 'N/A'} ⏱️ Duration: ${formatDuration(totalDuration)} ⚠️ Please check the logs above ========================================== """ // 失败时收集诊断信息 sh ''' echo "=== Docker System Info ===" docker system df echo "" echo "=== Disk Usage ===" df -h | grep -E "/$|/var" echo "" echo "=== Recent Containers ===" docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.CreatedAt}}" | head -10 ''' } } always { script { echo "🧹 Cleaning up..." // 清理悬空的镜像(但保留带标签的镜像用于回滚) if (env.KEEP_PREVIOUS_IMAGES == 'true') { sh "docker image prune -f --filter 'dangling=true' || true" } else { sh "docker image prune -f || true" } // 清理超过30天的构建日志 sh """ find ${env.WORKSPACE} -name '*.log' -mtime +30 -delete 2>/dev/null || true """ echo "📊 Final System Status:" sh 'df -h | grep -E "/$|/var" || true' sh 'docker system df || true' // 【优化6】保存性能指标到文件 archivePerformanceMetrics() } } } } // ============================================ // 辅助函数 // ============================================ // 获取变更的文件列表(对比上次成功构建,支持多 commit) def getChangedFiles() { def changedFiles = sh( script: ''' # 优先使用上次成功构建的 commit if [ -n "$GIT_PREVIOUS_SUCCESSFUL_COMMIT" ]; then PREV="$GIT_PREVIOUS_SUCCESSFUL_COMMIT" else # 回退到 origin/master(如果有) PREV=$(git rev-parse origin/master 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || echo "") fi if [ -z "$PREV" ]; then echo "all" else git diff --name-only $PREV HEAD fi ''', returnStdout: true ).trim() if (changedFiles == 'all') { return ['all'] as List } return changedFiles.split('\n') as List } // 检测是否需要重建依赖镜像 @NonCPS def checkIfDepsChanged(List changedFiles) { if (changedFiles.contains('all')) { return 'true' } def depsFiles = ['pom.xml', 'viewsh-dependencies', 'viewsh-framework', 'docker/Dockerfile.deps'] return depsFiles.any { changedFiles.contains(it) } ? 'true' : 'false' } // 检测需要构建的服务 def detectServicesToBuild(List changedFiles) { if (changedFiles.contains('all')) { return env.CORE_SERVICES } def triggerAllFiles = ['pom.xml', 'viewsh-framework', 'viewsh-dependencies', 'Jenkinsfile', 'docker/'] if (triggerAllFiles.any { triggerFile -> changedFiles.any { changedFile -> changedFile.startsWith(triggerFile) || changedFile == triggerFile } }) { return env.CORE_SERVICES } def changedServices = [] def allServices = env.CORE_SERVICES.split(',') allServices.each { service -> def modulePath = getModulePathForService(service) def moduleDir = modulePath.split('/')[0] if (changedFiles.any { it.startsWith(moduleDir) }) { changedServices.add(service) } } return changedServices.isEmpty() ? env.CORE_SERVICES : changedServices.join(',') } // 检查依赖镜像是否存在 def depsImageExists() { def result = sh( script: "docker manifest inspect ${env.DEPS_IMAGE} > /dev/null 2>&1", returnStatus: true ) return result == 0 } // 【优化2】动态检测系统资源并设置并行度 def detectSystemResources() { try { // 获取 CPU 核心数 def cpuCores = sh( script: 'nproc 2>/dev/null || echo "2"', returnStdout: true ).trim() as int // 获取可用内存(GB) def availableMemory = sh( script: 'free -g | grep Mem | awk \'{print $7}\'', returnStdout: true ).trim() as int // 动态计算并行度 // 规则:每个构建任务需要至少 2GB 内存和 1 个 CPU 核心 def maxParallelByMemory = Math.max(1, (availableMemory / 2) as int) def maxParallelByCpu = Math.max(1, cpuCores - 1) // 保留一个核心给系统 env.MAX_PARALLEL_BUILDS = Math.min(maxParallelByMemory, maxParallelByCpu).toString() echo """ ======================================== 📊 System Resources Detected: ======================================== CPU Cores: ${cpuCores} Available Memory: ${availableMemory} GB Max Parallel Builds: ${env.MAX_PARALLEL_BUILDS} ======================================== """ } catch (Exception e) { echo "⚠️ Failed to detect system resources: ${e.message}" echo "Using default parallelism: 2" env.MAX_PARALLEL_BUILDS = '2' } } // 【优化6】记录阶段性能指标 def recordStageMetrics(String stageName, long startTime) { def duration = System.currentTimeMillis() - startTime def durationStr = formatDuration(duration) if (!env.STAGE_METRICS) { env.STAGE_METRICS = "" } env.STAGE_METRICS += "${stageName}:${duration}|" echo "⏱️ Stage '${stageName}' completed in ${durationStr}" } // 【优化6】格式化时长 @NonCPS def formatDuration(long milliseconds) { def seconds = (milliseconds / 1000) as int def minutes = (seconds / 60) as int def hours = (minutes / 60) as int if (hours > 0) { return "${hours}h ${minutes % 60}m ${seconds % 60}s" } else if (minutes > 0) { return "${minutes}m ${seconds % 60}s" } else { return "${seconds}s" } } // 【优化6】生成性能报告 def generatePerformanceReport(long totalDuration) { echo """ ========================================== 📊 PERFORMANCE REPORT ========================================== Total Duration: ${formatDuration(totalDuration)} """ if (env.STAGE_METRICS) { echo "Stage Breakdown:" env.STAGE_METRICS.split('\\|').each { metric -> if (metric) { def parts = metric.split(':') if (parts.size() == 2) { def stageName = parts[0] def duration = parts[1] as long def percentage = (duration * 100 / totalDuration) as int echo " - ${stageName.padRight(25)}: ${formatDuration(duration).padRight(10)} (${percentage}%)" } } } } echo "==========================================" } // 【优化6】归档性能指标 def archivePerformanceMetrics() { try { def metricsFile = "${env.WORKSPACE}/build-metrics-${env.BUILD_NUMBER}.json" def metricsData = [ buildNumber: env.BUILD_NUMBER, timestamp: new Date().format('yyyy-MM-dd HH:mm:ss'), branch: env.BRANCH_NAME, imageTag: env.IMAGE_TAG, servicesToBuild: env.SERVICES_TO_BUILD, totalDuration: System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong(), stages: [:] ] if (env.STAGE_METRICS) { env.STAGE_METRICS.split('\\|').each { metric -> if (metric) { def parts = metric.split(':') if (parts.size() == 2) { metricsData.stages[parts[0]] = parts[1] as long } } } } // writeJSON file: metricsFile, json: metricsData writeFile file: metricsFile, text: groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(metricsData)) archiveArtifacts artifacts: "build-metrics-${env.BUILD_NUMBER}.json", allowEmptyArchive: true echo "✅ Performance metrics archived: ${metricsFile}" } catch (Exception e) { echo "⚠️ Failed to archive performance metrics: ${e.message}" } } // 【优化7】获取上一次成功部署的镜像标签 def getPreviousImageTag() { try { // 从部署主机获取当前运行的镜像标签 def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}" def previousTag = sh( script: """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' cd ${env.DEPLOY_PATH} docker compose -f docker-compose.core.yml images --format json | \ jq -r ".[0].Tag" | head -1 ' 2>/dev/null || echo "latest" """, returnStdout: true ).trim() return previousTag ?: 'latest' } catch (Exception e) { echo "⚠️ Failed to get previous image tag: ${e.message}" return 'latest' } } // 【优化7】备份当前部署 def backupCurrentDeployment(def services) { echo "💾 Backing up current deployment state..." def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}" try { sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' cd ${env.DEPLOY_PATH} # 保存当前 docker-compose 配置 cp docker-compose.core.yml docker-compose.core.yml.backup-${env.BUILD_NUMBER} # 记录当前运行的镜像 docker compose -f docker-compose.core.yml images > deployment-state-${env.BUILD_NUMBER}.txt echo "✅ Backup completed: deployment-state-${env.BUILD_NUMBER}.txt" ' """ echo "✅ Current deployment backed up successfully" } catch (Exception e) { echo "⚠️ Failed to backup current deployment: ${e.message}" } } // 【优化7】回滚部署 def rollbackDeployment(def services) { echo """ ========================================== 🔄 INITIATING ROLLBACK ========================================== Rolling back to: ${env.PREVIOUS_IMAGE_TAG} Services: ${services.join(', ')} ========================================== """ def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}" try { services.each { service -> echo "🔄 Rolling back ${service}..." sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' cd ${env.DEPLOY_PATH} # 设置回滚镜像标签 export IMAGE_TAG=${env.PREVIOUS_IMAGE_TAG} # 拉取旧版本镜像 docker compose -f docker-compose.core.yml pull ${service} # 重启服务 docker compose -f docker-compose.core.yml up -d ${service} echo "✅ ${service} rolled back to ${env.PREVIOUS_IMAGE_TAG}" ' """ // 等待服务启动 sleep 5 } echo """ ========================================== ✅ ROLLBACK COMPLETED ========================================== All services have been rolled back to: ${env.PREVIOUS_IMAGE_TAG} ========================================== """ } catch (Exception e) { echo """ ========================================== ❌ ROLLBACK FAILED ========================================== Error: ${e.message} Manual intervention required! ========================================== """ throw e } } // 构建服务(带重试) def buildServiceWithRetry(String service) { retry(2) { timeout(time: env.BUILD_TIMEOUT.toInteger(), unit: 'MINUTES') { buildService(service) } } } // 构建单个服务 def buildService(String service) { def modulePath = getModulePathForService(service) def buildStartTime = System.currentTimeMillis() echo "" echo "==========================================" echo "🔨 Building ${service}" echo "==========================================" echo "Module: ${modulePath}" echo "Registry: ${env.REGISTRY}" echo "Tag: ${env.IMAGE_TAG}" echo "==========================================" try { // 【优化1】使用 Maven 缓存卷 sh """ set -e set -x # 构建镜像(使用 Maven 缓存) docker build \\ -f docker/Dockerfile.service \\ --build-arg DEPS_IMAGE=${env.DEPS_IMAGE} \\ --build-arg MODULE_NAME=${modulePath} \\ --build-arg JAR_NAME=${service} \\ --build-arg SKIP_TESTS=true \\ --build-arg MAVEN_OPTS="${env.MAVEN_OPTS}" \\ -t ${env.REGISTRY}/${service}:${env.IMAGE_TAG} \\ -t ${env.REGISTRY}/${service}:latest \\ . # 推送镜像 docker push ${env.REGISTRY}/${service}:${env.IMAGE_TAG} docker push ${env.REGISTRY}/${service}:latest set +x """ def buildDuration = System.currentTimeMillis() - buildStartTime echo "✅ ${service} built and pushed successfully in ${formatDuration(buildDuration)}" // 获取镜像大小 def imageSize = sh( script: "docker images ${env.REGISTRY}/${service}:latest --format '{{.Size}}'", returnStdout: true ).trim() echo "📊 Image size: ${imageSize}" } catch (Exception e) { echo "❌ Failed to build ${service}: ${e.message}" // 打印构建日志以便调试 sh """ echo "=== Docker Build Logs for ${service} ===" docker logs ${service}-builder 2>/dev/null || true """ throw e } } // 部署服务(带超时) def deployServiceWithTimeout(String service) { timeout(time: env.DEPLOY_TIMEOUT.toInteger(), unit: 'MINUTES') { deployService(service) } } // 部署单个服务 def deployService(String service) { def containerName = getContainerNameForService(service) def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}" echo "" echo "==========================================" echo "🚀 Deploying ${service}" echo "==========================================" echo "Container: ${containerName}" echo "Host: ${env.DEPLOY_HOST}" echo "==========================================" try { // 部署服务 sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' set -e cd ${env.DEPLOY_PATH} echo "📥 Pulling ${service}..." docker compose -f docker-compose.core.yml pull ${service} echo "🔄 Restarting ${service}..." docker compose -f docker-compose.core.yml up -d ${service} echo "⏳ Waiting for container to start..." sleep 5 ' """ // 等待服务健康 waitForServiceHealthy(containerName, service, sshOpts) echo "✅ ${service} deployed successfully" } catch (Exception e) { echo "❌ Failed to deploy ${service}: ${e.message}" // 收集诊断信息 sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' echo "=== Container Status ===" docker ps -a | grep ${containerName} || true echo "" echo "=== Container Logs (last 50 lines) ===" docker logs --tail 50 ${containerName} 2>/dev/null || true echo "" echo "=== Service Health Status ===" docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found" ' """ throw e } } // 等待服务健康 def waitForServiceHealthy(String containerName, String serviceName, String sshOpts) { // iot-gateway 是轻量化网关,不需要健康检查,只检查容器是否运行 if (serviceName == 'viewsh-module-iot-gateway') { echo "⏳ Waiting for ${serviceName} to start (lightweight gateway, no health check)..." sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' set -e sleep 5 if docker ps | grep -q ${containerName}; then echo "✅ ${serviceName} is running" else echo "❌ ${serviceName} is not running" docker logs --tail 50 ${containerName} 2>/dev/null || true exit 1 fi ' """ return } def maxAttempts = env.HEALTH_CHECK_TIMEOUT.toInteger() / env.HEALTH_CHECK_INTERVAL.toInteger() echo "⏳ Waiting for ${serviceName} to be healthy (max ${env.HEALTH_CHECK_TIMEOUT}s)..." sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' set -e for i in \$(seq 1 ${maxAttempts}); do STATUS=\$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "starting") case "\$STATUS" in healthy) echo "✅ ${serviceName} is healthy" exit 0 ;; unhealthy) echo "❌ ${serviceName} is unhealthy" echo "=== Last 100 lines of logs ===" docker logs --tail 100 ${containerName} exit 1 ;; starting) ELAPSED=\$((i * ${env.HEALTH_CHECK_INTERVAL})) echo "⏳ ${serviceName} is starting... (\${ELAPSED}s/${env.HEALTH_CHECK_TIMEOUT}s)" ;; *) echo "⚠️ ${serviceName} status: \$STATUS" ;; esac sleep ${env.HEALTH_CHECK_INTERVAL} done echo "❌ ${serviceName} health check timeout after ${env.HEALTH_CHECK_TIMEOUT}s" echo "=== Full logs ===" docker logs ${containerName} exit 1 ' """ } // 检查服务健康(带重试) def checkServiceHealthWithRetry(String service) { def containerName = getContainerNameForService(service) def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}" retry(3) { timeout(time: 2, unit: 'MINUTES') { checkServiceHealth(containerName, service, sshOpts) } } } // 检查服务健康状态 def checkServiceHealth(String containerName, String serviceName, String sshOpts) { // iot-gateway 是轻量化网关,不需要健康检查,只检查容器是否运行 if (serviceName == 'viewsh-module-iot-gateway') { sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' if docker ps | grep -q ${containerName}; then echo "✅ ${serviceName} is running (lightweight gateway)" else echo "❌ ${serviceName} is not running" docker logs --tail 50 ${containerName} 2>/dev/null || true exit 1 fi ' """ return } sh """ ssh ${sshOpts} root@${env.DEPLOY_HOST} ' STATUS=\$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found") case "\$STATUS" in healthy) echo "✅ ${serviceName} is healthy" ;; not_found) echo "⚠️ ${serviceName} not found (may not be deployed)" exit 1 ;; *) echo "❌ ${serviceName} is \$STATUS" echo "=== Last 50 lines of logs ===" docker logs --tail 50 ${containerName} exit 1 ;; esac ' """ } // 按依赖顺序排序服务 @NonCPS def sortServicesByDependency(def services) { def deployOrder = [ 'viewsh-gateway', 'viewsh-module-system-server', 'viewsh-module-infra-server', 'viewsh-module-iot-server', 'viewsh-module-iot-gateway', 'viewsh-module-ops-server' ] return services.sort { a, b -> deployOrder.indexOf(a) <=> deployOrder.indexOf(b) } } // 获取服务对应的容器名称 @NonCPS def getContainerNameForService(String service) { def map = [ 'viewsh-gateway': 'aiot-gateway', 'viewsh-module-system-server': 'aiot-system-server', 'viewsh-module-infra-server': 'aiot-infra-server', 'viewsh-module-iot-server': 'aiot-iot-server', 'viewsh-module-iot-gateway': 'aiot-iot-gateway', 'viewsh-module-ops-server': 'aiot-ops-server' ] return map.get(service, "aiot-${service}") } // 获取服务对应的模块路径 @NonCPS def getModulePathForService(String service) { def map = [ 'viewsh-gateway': 'viewsh-gateway', 'viewsh-module-system-server': 'viewsh-module-system/viewsh-module-system-server', 'viewsh-module-infra-server': 'viewsh-module-infra/viewsh-module-infra-server', 'viewsh-module-iot-server': 'viewsh-module-iot/viewsh-module-iot-server', 'viewsh-module-iot-gateway': 'viewsh-module-iot/viewsh-module-iot-gateway', 'viewsh-module-ops-server': 'viewsh-module-ops/viewsh-module-ops-server' ] return map.get(service, service) }