aiot-platform-cloud/Jenkinsfile

// ============================================
// AIOT Platform - Jenkins Pipeline (Optimized Edition)
// ============================================

pipeline {
    agent any

    options {
        buildDiscarder(logRotator(
            numToKeepStr: '10',
            artifactNumToKeepStr: '5'
        ))
        disableConcurrentBuilds()
        timeout(time: 90, unit: 'MINUTES')
        timestamps()
        retry(1)
    }

    environment {
        // 镜像仓库配置（Infra 服务器内网地址，Prod 服务器可通过内网拉取）
        REGISTRY = '172.17.16.7:5000'
        DEPS_IMAGE = "${REGISTRY}/aiot-deps:latest"

        // 服务配置
        CORE_SERVICES = 'viewsh-gateway,viewsh-module-system-server,viewsh-module-infra-server,viewsh-module-iot-server,viewsh-module-iot-gateway,viewsh-module-ops-server'

        // 部署配置（Prod 服务器内网地址）
        DEPLOY_HOST = '172.17.16.14'
        DEPLOY_PATH = '/opt/aiot-platform-cloud'
        SSH_KEY = '/var/jenkins_home/.ssh/id_rsa'

        // 性能配置 - 将动态调整
        BUILD_TIMEOUT = 45
        DEPLOY_TIMEOUT = 10
        HEALTH_CHECK_TIMEOUT = 180
        HEALTH_CHECK_INTERVAL = 10

        // 【优化1】Maven 缓存配置
        MAVEN_CACHE_VOLUME = 'jenkins-maven-cache'
        MAVEN_OPTS = '-Dmaven.repo.local=/var/jenkins_home/.m2/repository'

        // 【优化7】回滚配置
        ROLLBACK_ENABLED = 'true'
        KEEP_PREVIOUS_IMAGES = 'true'
    }

    stages {
        stage('Initialize') {
            steps {
                script {
                    // 【优化6】记录开始时间
                    env.PIPELINE_START_TIME = System.currentTimeMillis()
                    
                    echo "=========================================="
                    echo "  AIOT Platform - CI/CD Pipeline (Optimized)"
                    echo "=========================================="
                    echo "Branch: ${env.BRANCH_NAME}"
                    echo "Build: #${env.BUILD_NUMBER}"
                    echo "Workspace: ${env.WORKSPACE}"
                    echo "Start Time: ${new Date()}"
                    echo "=========================================="
                    
                    // 【优化2】动态检测系统资源
                    detectSystemResources()
                }
            }
        }

        stage('Checkout') {
            steps {
                script {
                    def stageStartTime = System.currentTimeMillis()
                    
                    retry(3) {
                        checkout scm
                    }
                    
                    def shortCommit = sh(
                        script: 'git rev-parse --short HEAD',
                        returnStdout: true
                    ).trim()

                    // 清理分支名：将斜杠替换为连字符，确保 Docker 标签格式有效
                    def sanitizedBranchName = env.BRANCH_NAME.replaceAll('/', '-')
                    env.IMAGE_TAG = "${sanitizedBranchName}-${env.BUILD_NUMBER}-${shortCommit}"
                    env.PREVIOUS_IMAGE_TAG = getPreviousImageTag()
                    env.GIT_COMMIT_MSG = sh(
                        script: 'git log -1 --pretty=%B',
                        returnStdout: true
                    ).trim()

                    echo "📦 Commit: ${shortCommit}"
                    echo "📝 Message: ${env.GIT_COMMIT_MSG}"
                    echo "🏷️  Current Tag: ${env.IMAGE_TAG}"
                    echo "🔖 Previous Tag: ${env.PREVIOUS_IMAGE_TAG}"
                    
                    // 【优化6】记录阶段耗时
                    recordStageMetrics('Checkout', stageStartTime)
                }
            }
        }

        stage('Detect Changes') {
            steps {
                script {
                    def stageStartTime = System.currentTimeMillis()
                    
                    def changedFiles = getChangedFiles()
                    echo "📝 Changed files: ${changedFiles.size()} files"

                    env.SERVICES_TO_BUILD = detectServicesToBuild(changedFiles)
                    env.DEPS_CHANGED = checkIfDepsChanged(changedFiles)
                    
                    // Check if deps image exists
                    if (env.DEPS_CHANGED == 'false') {
                        def depsExists = sh(
                            script: "docker manifest inspect ${env.DEPS_IMAGE} > /dev/null 2>&1",
                            returnStatus: true
                        )
                        if (depsExists != 0) {
                            env.DEPS_CHANGED = 'true'
                            echo "📦 Deps image not found, will rebuild"
                        }
                    }

                    if (env.SERVICES_TO_BUILD.isEmpty()) {
                        echo "⏭️  No changes detected, skipping build"
                        currentBuild.result = 'SUCCESS'
                        return
                    }

                    echo "🔄 Services to build: ${env.SERVICES_TO_BUILD}"
                    echo "📦 Deps changed: ${env.DEPS_CHANGED}"

                    env.SERVICES_TO_BUILD.split(',').each { service ->
                        def module = getModulePathForService(service)
                        echo "   - ${service} (${module})"
                    }
                    
                    recordStageMetrics('Detect Changes', stageStartTime)
                }
            }
        }

        stage('Pre-build Check') {
            when {
                expression { env.SERVICES_TO_BUILD != '' }
            }
            steps {
                script {
                    def stageStartTime = System.currentTimeMillis()
                    
                    echo "🔍 Running pre-build checks..."

                    // 检查 Docker
                    sh "docker version >/dev/null 2>&1 || { echo '❌ Docker not available'; exit 1; }"

                    // 【优化1】检查并创建 Maven 缓存卷
                    sh """
                        if ! docker volume inspect ${env.MAVEN_CACHE_VOLUME} >/dev/null 2>&1; then
                            echo "📦 Creating Maven cache volume: ${env.MAVEN_CACHE_VOLUME}"
                            docker volume create ${env.MAVEN_CACHE_VOLUME}
                        else
                            echo "✅ Maven cache volume exists: ${env.MAVEN_CACHE_VOLUME}"
                        fi
                    """

                    // 检查磁盘空间
                    def diskUsage = sh(
                        script: "df ${env.WORKSPACE} | tail -1 | awk '{print \$5}' | sed 's/%//'",
                        returnStdout: true
                    ).trim() as int

                    if (diskUsage > 80) {
                        echo "⚠️  Disk usage is ${diskUsage}%, cleaning up..."
                        sh "docker system prune -f --volumes || true"
                    }

                    // 检查镜像仓库连接
                    sh """
                        curl -f ${env.REGISTRY}/v2/ >/dev/null 2>&1 || \
                        { echo '⚠️  Registry not accessible, will continue...'; }
                    """

                    echo "✅ Pre-build checks passed"
                    recordStageMetrics('Pre-build Check', stageStartTime)
                }
            }
        }

        stage('Build Dependencies Image') {
            when {
                expression {
                    env.SERVICES_TO_BUILD != '' &&
                    env.DEPS_CHANGED == 'true'
                }
            }
            steps {
                script {
                    def stageStartTime = System.currentTimeMillis()
                    
                    echo "📦 Building dependencies base image with Maven cache..."
                    timeout(time: 15, unit: 'MINUTES') {
                        // 【优化1】使用 Maven 缓存卷加速依赖下载
                        sh """
                            set -e
                            echo "Building ${env.DEPS_IMAGE} with cache..."
                            docker build \
                                -f docker/Dockerfile.deps \
                                -t ${env.DEPS_IMAGE} \
                                --build-arg BUILDKIT_INLINE_CACHE=1 \
                                --build-arg MAVEN_OPTS="${env.MAVEN_OPTS}" \
                                .

                            docker push ${env.DEPS_IMAGE}
                            echo "✅ Dependencies image built and pushed"
                        """
                    }
                    
                    recordStageMetrics('Build Dependencies Image', stageStartTime)
                }
            }
        }

        stage('Build Services') {
            when {
                expression { env.SERVICES_TO_BUILD != '' }
            }
            steps {
                script {
                    def stageStartTime = System.currentTimeMillis()
                    
                    def servicesToBuild = env.SERVICES_TO_BUILD.split(',')
                    echo "🔨 Building ${servicesToBuild.size()} services (parallelism: ${env.MAX_PARALLEL_BUILDS})"

                    // 【优化2】动态并行构建
                    def buildTasks = [:]
                    servicesToBuild.each { service ->
                        buildTasks[service] = {
                            buildServiceWithRetry(service)
                        }
                    }

                    parallel buildTasks

                    echo "✅ All services built successfully"

                    // 显示构建后的镜像信息
                    sh """
                        echo "📊 Built images:"
                        ${env.SERVICES_TO_BUILD.split(',').collect { "docker images ${env.REGISTRY}/${it} --format '  {{.Repository}}:{{.Tag}} - {{.Size}}'" }.join('\n                        ')}
                    """
                    
                    recordStageMetrics('Build Services', stageStartTime)
                }
            }
        }

        stage('Deploy') {
            when {
                allOf {
                    expression { env.SERVICES_TO_BUILD != '' }
                    branch 'master'
                }
            }
            steps {
                script {
                    def stageStartTime = System.currentTimeMillis()
                    
                    def servicesToDeploy = env.SERVICES_TO_BUILD.split(',')
                    def sortedServices = sortServicesByDependency(servicesToDeploy)

                    echo "🚀 Deploying ${sortedServices.size()} services in dependency order"
                    sortedServices.eachWithIndex { service, index ->
                        echo "   ${index + 1}. ${service}"
                    }

                    // 【优化7】部署前备份当前镜像标签
                    if (env.ROLLBACK_ENABLED == 'true') {
                        backupCurrentDeployment(sortedServices)
                    }

                    // 【新增】同步最新的 docker-compose.core.yml 到部署服务器
                    echo "📂 Syncing docker-compose.core.yml to deploy host..."
                    sh "scp -o StrictHostKeyChecking=no -i ${env.SSH_KEY} docker-compose.core.yml root@${env.DEPLOY_HOST}:${env.DEPLOY_PATH}/"

                    try {
                        // 串行部署（保证依赖关系）
                        sortedServices.each { service ->
                            deployServiceWithTimeout(service)
                        }

                        echo "🚀 All services deployed successfully!"
                        
                    } catch (Exception e) {
                        // 【优化7】部署失败时自动回滚
                        if (env.ROLLBACK_ENABLED == 'true') {
                            echo "❌ Deployment failed: ${e.message}"
                            echo "🔄 Initiating automatic rollback..."
                            rollbackDeployment(sortedServices)
                        }
                        throw e
                    }
                    
                    recordStageMetrics('Deploy', stageStartTime)
                }
            }
        }

        stage('Final Health Check') {
            when {
                allOf {
                    expression { env.SERVICES_TO_BUILD != '' }
                    branch 'master'
                }
            }
            steps {
                script {
                    def stageStartTime = System.currentTimeMillis()
                    
                    echo "🏥 Running final health check for all services..."

                    def servicesToCheck = env.SERVICES_TO_BUILD.split(',')
                    def healthCheckTasks = [:]

                    servicesToCheck.each { service ->
                        healthCheckTasks[service] = {
                            checkServiceHealthWithRetry(service)
                        }
                    }

                    try {
                        parallel healthCheckTasks
                        echo "✅ All services are healthy!"
                    } catch (Exception e) {
                        // 【优化7】健康检查失败时回滚
                        if (env.ROLLBACK_ENABLED == 'true') {
                            echo "❌ Health check failed: ${e.message}"
                            echo "🔄 Initiating automatic rollback..."
                            rollbackDeployment(servicesToCheck)
                        }
                        throw e
                    }

                    // 显示最终状态
                    sh """
                        echo "📊 Service Status:"
                        ${servicesToCheck.collect {
                            def container = getContainerNameForService(it)
                            "docker inspect --format='${it}: {{.State.Status}} ({{.State.Health.Status}})' ${container} 2>/dev/null || echo '${it}: not found'"
                        }.join('\n                        ')}
                    """
                    
                    recordStageMetrics('Final Health Check', stageStartTime)
                }
            }
        }
    }

    post {
        success {
            script {
                // 【优化6】计算总耗时并生成性能报告
                def totalDuration = System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong()
                generatePerformanceReport(totalDuration)
                
                echo """
                ==========================================
                ✅ BUILD SUCCESS
                ==========================================
                📦 Services: ${env.SERVICES_TO_BUILD}
                🏷️  Tag: ${env.IMAGE_TAG}
                ⏱️  Duration: ${formatDuration(totalDuration)}
                ==========================================
                """
            }
        }
        failure {
            script {
                def totalDuration = System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong()
                
                echo """
                ==========================================
                ❌ BUILD FAILED
                ==========================================
                📦 Services: ${env.SERVICES_TO_BUILD ?: 'None'}
                🏷️  Tag: ${env.IMAGE_TAG ?: 'Unknown'}
                🔖 Rollback Tag: ${env.PREVIOUS_IMAGE_TAG ?: 'N/A'}
                ⏱️  Duration: ${formatDuration(totalDuration)}
                ⚠️  Please check the logs above
                ==========================================
                """

                // 失败时收集诊断信息
                sh '''
                    echo "=== Docker System Info ==="
                    docker system df
                    echo ""
                    echo "=== Disk Usage ==="
                    df -h | grep -E "/$|/var"
                    echo ""
                    echo "=== Recent Containers ==="
                    docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.CreatedAt}}" | head -10
                '''
            }
        }
        always {
            script {
                echo "🧹 Cleaning up..."

                // 清理悬空的镜像（但保留带标签的镜像用于回滚）
                if (env.KEEP_PREVIOUS_IMAGES == 'true') {
                    sh "docker image prune -f --filter 'dangling=true' || true"
                } else {
                    sh "docker image prune -f || true"
                }

                // 清理超过30天的构建日志
                sh """
                    find ${env.WORKSPACE} -name '*.log' -mtime +30 -delete 2>/dev/null || true
                """

                echo "📊 Final System Status:"
                sh 'df -h | grep -E "/$|/var" || true'
                sh 'docker system df || true'
                
                // 【优化6】保存性能指标到文件
                archivePerformanceMetrics()
            }
        }
    }
}

// ============================================
// 辅助函数
// ============================================

// 获取变更的文件列表（对比上次成功构建，支持多 commit）
def getChangedFiles() {
    def changedFiles = sh(
        script: '''
            # 优先使用上次成功构建的 commit
            if [ -n "$GIT_PREVIOUS_SUCCESSFUL_COMMIT" ]; then
                PREV="$GIT_PREVIOUS_SUCCESSFUL_COMMIT"
            else
                # 回退到 origin/master（如果有）
                PREV=$(git rev-parse origin/master 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || echo "")
            fi

            if [ -z "$PREV" ]; then
                echo "all"
            else
                git diff --name-only $PREV HEAD
            fi
        ''',
        returnStdout: true
    ).trim()

    if (changedFiles == 'all') {
        return ['all'] as List<String>
    }

    return changedFiles.split('\n') as List<String>
}

// 检测是否需要重建依赖镜像
@NonCPS
def checkIfDepsChanged(List<String> changedFiles) {
    if (changedFiles.contains('all')) {
        return 'true'
    }

    def depsFiles = ['pom.xml', 'viewsh-dependencies', 'viewsh-framework', 'docker/Dockerfile.deps']
    return depsFiles.any { changedFiles.contains(it) } ? 'true' : 'false'
}

// 检测需要构建的服务
def detectServicesToBuild(List<String> changedFiles) {
    if (changedFiles.contains('all')) {
        return env.CORE_SERVICES
    }

    def triggerAllFiles = ['pom.xml', 'viewsh-framework', 'viewsh-dependencies', 'Jenkinsfile', 'docker/']
    if (triggerAllFiles.any { triggerFile ->
        changedFiles.any { changedFile ->
            changedFile.startsWith(triggerFile) || changedFile == triggerFile
        }
    }) {
        return env.CORE_SERVICES
    }

    def changedServices = []
    def allServices = env.CORE_SERVICES.split(',')

    allServices.each { service ->
        def modulePath = getModulePathForService(service)
        def moduleDir = modulePath.split('/')[0]

        if (changedFiles.any { it.startsWith(moduleDir) }) {
            changedServices.add(service)
        }
    }

    return changedServices.isEmpty() ? env.CORE_SERVICES : changedServices.join(',')
}

// 检查依赖镜像是否存在
def depsImageExists() {
    def result = sh(
        script: "docker manifest inspect ${env.DEPS_IMAGE} > /dev/null 2>&1",
        returnStatus: true
    )
    return result == 0
}

// 【优化2】动态检测系统资源并设置并行度
def detectSystemResources() {
    try {
        // 获取 CPU 核心数
        def cpuCores = sh(
            script: 'nproc 2>/dev/null || echo "2"',
            returnStdout: true
        ).trim() as int

        // 获取可用内存（GB）
        def availableMemory = sh(
            script: 'free -g | grep Mem | awk \'{print $7}\'',
            returnStdout: true
        ).trim() as int

        // 动态计算并行度
        // 规则：每个构建任务需要至少 2GB 内存和 1 个 CPU 核心
        def maxParallelByMemory = Math.max(1, (availableMemory / 2) as int)
        def maxParallelByCpu = Math.max(1, cpuCores - 1)  // 保留一个核心给系统
        
        env.MAX_PARALLEL_BUILDS = Math.min(maxParallelByMemory, maxParallelByCpu).toString()

        echo """
        ========================================
        📊 System Resources Detected:
        ========================================
        CPU Cores: ${cpuCores}
        Available Memory: ${availableMemory} GB
        Max Parallel Builds: ${env.MAX_PARALLEL_BUILDS}
        ========================================
        """
    } catch (Exception e) {
        echo "⚠️  Failed to detect system resources: ${e.message}"
        echo "Using default parallelism: 2"
        env.MAX_PARALLEL_BUILDS = '2'
    }
}

// 【优化6】记录阶段性能指标
def recordStageMetrics(String stageName, long startTime) {
    def duration = System.currentTimeMillis() - startTime
    def durationStr = formatDuration(duration)
    
    if (!env.STAGE_METRICS) {
        env.STAGE_METRICS = ""
    }
    
    env.STAGE_METRICS += "${stageName}:${duration}|"
    echo "⏱️  Stage '${stageName}' completed in ${durationStr}"
}

// 【优化6】格式化时长
@NonCPS
def formatDuration(long milliseconds) {
    def seconds = (milliseconds / 1000) as int
    def minutes = (seconds / 60) as int
    def hours = (minutes / 60) as int
    
    if (hours > 0) {
        return "${hours}h ${minutes % 60}m ${seconds % 60}s"
    } else if (minutes > 0) {
        return "${minutes}m ${seconds % 60}s"
    } else {
        return "${seconds}s"
    }
}

// 【优化6】生成性能报告
def generatePerformanceReport(long totalDuration) {
    echo """
    ==========================================
    📊 PERFORMANCE REPORT
    ==========================================
    Total Duration: ${formatDuration(totalDuration)}
    """
    
    if (env.STAGE_METRICS) {
        echo "Stage Breakdown:"
        env.STAGE_METRICS.split('\\|').each { metric ->
            if (metric) {
                def parts = metric.split(':')
                if (parts.size() == 2) {
                    def stageName = parts[0]
                    def duration = parts[1] as long
                    def percentage = (duration * 100 / totalDuration) as int
                    echo "  - ${stageName.padRight(25)}: ${formatDuration(duration).padRight(10)} (${percentage}%)"
                }
            }
        }
    }
    
    echo "=========================================="
}

// 【优化6】归档性能指标
def archivePerformanceMetrics() {
    try {
        def metricsFile = "${env.WORKSPACE}/build-metrics-${env.BUILD_NUMBER}.json"
        def metricsData = [
            buildNumber: env.BUILD_NUMBER,
            timestamp: new Date().format('yyyy-MM-dd HH:mm:ss'),
            branch: env.BRANCH_NAME,
            imageTag: env.IMAGE_TAG,
            servicesToBuild: env.SERVICES_TO_BUILD,
            totalDuration: System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong(),
            stages: [:]
        ]
        
        if (env.STAGE_METRICS) {
            env.STAGE_METRICS.split('\\|').each { metric ->
                if (metric) {
                    def parts = metric.split(':')
                    if (parts.size() == 2) {
                        metricsData.stages[parts[0]] = parts[1] as long
                    }
                }
            }
        }
        
        // writeJSON file: metricsFile, json: metricsData
        writeFile file: metricsFile, text: groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(metricsData))
        archiveArtifacts artifacts: "build-metrics-${env.BUILD_NUMBER}.json", allowEmptyArchive: true
        
        echo "✅ Performance metrics archived: ${metricsFile}"
    } catch (Exception e) {
        echo "⚠️  Failed to archive performance metrics: ${e.message}"
    }
}

// 【优化7】获取上一次成功部署的镜像标签
def getPreviousImageTag() {
    try {
        // 从部署主机获取当前运行的镜像标签
        def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"
        def previousTag = sh(
            script: """
                ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                    cd ${env.DEPLOY_PATH}
                    docker compose -f docker-compose.core.yml images --format json | \
                    jq -r ".[0].Tag" | head -1
                ' 2>/dev/null || echo "latest"
            """,
            returnStdout: true
        ).trim()
        
        return previousTag ?: 'latest'
    } catch (Exception e) {
        echo "⚠️  Failed to get previous image tag: ${e.message}"
        return 'latest'
    }
}

// 【优化7】备份当前部署
def backupCurrentDeployment(def services) {
    echo "💾 Backing up current deployment state..."
    
    def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"
    
    try {
        sh """
            ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                cd ${env.DEPLOY_PATH}
                
                # 保存当前 docker-compose 配置
                cp docker-compose.core.yml docker-compose.core.yml.backup-${env.BUILD_NUMBER}
                
                # 记录当前运行的镜像
                docker compose -f docker-compose.core.yml images > deployment-state-${env.BUILD_NUMBER}.txt
                
                echo "✅ Backup completed: deployment-state-${env.BUILD_NUMBER}.txt"
            '
        """
        
        echo "✅ Current deployment backed up successfully"
    } catch (Exception e) {
        echo "⚠️  Failed to backup current deployment: ${e.message}"
    }
}

// 【优化7】回滚部署
def rollbackDeployment(def services) {
    echo """
    ==========================================
    🔄 INITIATING ROLLBACK
    ==========================================
    Rolling back to: ${env.PREVIOUS_IMAGE_TAG}
    Services: ${services.join(', ')}
    ==========================================
    """
    
    def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"
    
    try {
        services.each { service ->
            echo "🔄 Rolling back ${service}..."
            
            sh """
                ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                    cd ${env.DEPLOY_PATH}
                    
                    # 设置回滚镜像标签
                    export IMAGE_TAG=${env.PREVIOUS_IMAGE_TAG}
                    
                    # 拉取旧版本镜像
                    docker compose -f docker-compose.core.yml pull ${service}
                    
                    # 重启服务
                    docker compose -f docker-compose.core.yml up -d ${service}
                    
                    echo "✅ ${service} rolled back to ${env.PREVIOUS_IMAGE_TAG}"
                '
            """
            
            // 等待服务启动
            sleep 5
        }
        
        echo """
        ==========================================
        ✅ ROLLBACK COMPLETED
        ==========================================
        All services have been rolled back to: ${env.PREVIOUS_IMAGE_TAG}
        ==========================================
        """
        
    } catch (Exception e) {
        echo """
        ==========================================
        ❌ ROLLBACK FAILED
        ==========================================
        Error: ${e.message}
        Manual intervention required!
        ==========================================
        """
        throw e
    }
}

// 构建服务（带重试）
def buildServiceWithRetry(String service) {
    retry(2) {
        timeout(time: env.BUILD_TIMEOUT.toInteger(), unit: 'MINUTES') {
            buildService(service)
        }
    }
}

// 构建单个服务
def buildService(String service) {
    def modulePath = getModulePathForService(service)
    def buildStartTime = System.currentTimeMillis()

    echo ""
    echo "=========================================="
    echo "🔨 Building ${service}"
    echo "=========================================="
    echo "Module: ${modulePath}"
    echo "Registry: ${env.REGISTRY}"
    echo "Tag: ${env.IMAGE_TAG}"
    echo "=========================================="

    try {
        // 【优化1】使用 Maven 缓存卷
        sh """
            set -e
            set -x

            # 构建镜像（使用 Maven 缓存）
            docker build \\
                -f docker/Dockerfile.service \\
                --build-arg DEPS_IMAGE=${env.DEPS_IMAGE} \\
                --build-arg MODULE_NAME=${modulePath} \\
                --build-arg JAR_NAME=${service} \\
                --build-arg SKIP_TESTS=true \\
                --build-arg MAVEN_OPTS="${env.MAVEN_OPTS}" \\
                -t ${env.REGISTRY}/${service}:${env.IMAGE_TAG} \\
                -t ${env.REGISTRY}/${service}:latest \\
                .

            # 推送镜像
            docker push ${env.REGISTRY}/${service}:${env.IMAGE_TAG}
            docker push ${env.REGISTRY}/${service}:latest

            set +x
        """

        def buildDuration = System.currentTimeMillis() - buildStartTime
        echo "✅ ${service} built and pushed successfully in ${formatDuration(buildDuration)}"

        // 获取镜像大小
        def imageSize = sh(
            script: "docker images ${env.REGISTRY}/${service}:latest --format '{{.Size}}'",
            returnStdout: true
        ).trim()

        echo "📊 Image size: ${imageSize}"

    } catch (Exception e) {
        echo "❌ Failed to build ${service}: ${e.message}"

        // 打印构建日志以便调试
        sh """
            echo "=== Docker Build Logs for ${service} ==="
            docker logs ${service}-builder 2>/dev/null || true
        """

        throw e
    }
}

// 部署服务（带超时）
def deployServiceWithTimeout(String service) {
    timeout(time: env.DEPLOY_TIMEOUT.toInteger(), unit: 'MINUTES') {
        deployService(service)
    }
}

// 部署单个服务
def deployService(String service) {
    def containerName = getContainerNameForService(service)
    def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"

    echo ""
    echo "=========================================="
    echo "🚀 Deploying ${service}"
    echo "=========================================="
    echo "Container: ${containerName}"
    echo "Host: ${env.DEPLOY_HOST}"
    echo "=========================================="

    try {
        // 部署服务
        sh """
            ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                set -e
                cd ${env.DEPLOY_PATH}

                echo "📥 Pulling ${service}..."
                docker compose -f docker-compose.core.yml pull ${service}

                echo "🔄 Restarting ${service}..."
                docker compose -f docker-compose.core.yml up -d ${service}

                echo "⏳ Waiting for container to start..."
                sleep 5
            '
        """

        // 等待服务健康
        waitForServiceHealthy(containerName, service, sshOpts)

        echo "✅ ${service} deployed successfully"

    } catch (Exception e) {
        echo "❌ Failed to deploy ${service}: ${e.message}"

        // 收集诊断信息
        sh """
            ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                echo "=== Container Status ==="
                docker ps -a | grep ${containerName} || true

                echo ""
                echo "=== Container Logs (last 50 lines) ==="
                docker logs --tail 50 ${containerName} 2>/dev/null || true

                echo ""
                echo "=== Service Health Status ==="
                docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found"
            '
        """

        throw e
    }
}

// 等待服务健康
def waitForServiceHealthy(String containerName, String serviceName, String sshOpts) {
    // iot-gateway 是轻量化网关，不需要健康检查，只检查容器是否运行
    if (serviceName == 'viewsh-module-iot-gateway') {
        echo "⏳ Waiting for ${serviceName} to start (lightweight gateway, no health check)..."
        sh """
            ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                set -e
                sleep 5
                if docker ps | grep -q ${containerName}; then
                    echo "✅ ${serviceName} is running"
                else
                    echo "❌ ${serviceName} is not running"
                    docker logs --tail 50 ${containerName} 2>/dev/null || true
                    exit 1
                fi
            '
        """
        return
    }

    def maxAttempts = env.HEALTH_CHECK_TIMEOUT.toInteger() / env.HEALTH_CHECK_INTERVAL.toInteger()

    echo "⏳ Waiting for ${serviceName} to be healthy (max ${env.HEALTH_CHECK_TIMEOUT}s)..."

    sh """
        ssh ${sshOpts} root@${env.DEPLOY_HOST} '
            set -e

            for i in \$(seq 1 ${maxAttempts}); do
                STATUS=\$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "starting")

                case "\$STATUS" in
                    healthy)
                        echo "✅ ${serviceName} is healthy"
                        exit 0
                        ;;
                    unhealthy)
                        echo "❌ ${serviceName} is unhealthy"
                        echo "=== Last 100 lines of logs ==="
                        docker logs --tail 100 ${containerName}
                        exit 1
                        ;;
                    starting)
                        ELAPSED=\$((i * ${env.HEALTH_CHECK_INTERVAL}))
                        echo "⏳ ${serviceName} is starting... (\${ELAPSED}s/${env.HEALTH_CHECK_TIMEOUT}s)"
                        ;;
                    *)
                        echo "⚠️  ${serviceName} status: \$STATUS"
                        ;;
                esac

                sleep ${env.HEALTH_CHECK_INTERVAL}
            done

            echo "❌ ${serviceName} health check timeout after ${env.HEALTH_CHECK_TIMEOUT}s"
            echo "=== Full logs ==="
            docker logs ${containerName}
            exit 1
        '
    """
}

// 检查服务健康（带重试）
def checkServiceHealthWithRetry(String service) {
    def containerName = getContainerNameForService(service)
    def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"

    retry(3) {
        timeout(time: 2, unit: 'MINUTES') {
            checkServiceHealth(containerName, service, sshOpts)
        }
    }
}

// 检查服务健康状态
def checkServiceHealth(String containerName, String serviceName, String sshOpts) {
    // iot-gateway 是轻量化网关，不需要健康检查，只检查容器是否运行
    if (serviceName == 'viewsh-module-iot-gateway') {
        sh """
            ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                if docker ps | grep -q ${containerName}; then
                    echo "✅ ${serviceName} is running (lightweight gateway)"
                else
                    echo "❌ ${serviceName} is not running"
                    docker logs --tail 50 ${containerName} 2>/dev/null || true
                    exit 1
                fi
            '
        """
        return
    }

    sh """
        ssh ${sshOpts} root@${env.DEPLOY_HOST} '
            STATUS=\$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found")

            case "\$STATUS" in
                healthy)
                    echo "✅ ${serviceName} is healthy"
                    ;;
                not_found)
                    echo "⚠️  ${serviceName} not found (may not be deployed)"
                    exit 1
                    ;;
                *)
                    echo "❌ ${serviceName} is \$STATUS"
                    echo "=== Last 50 lines of logs ==="
                    docker logs --tail 50 ${containerName}
                    exit 1
                    ;;
            esac
        '
    """
}

// 按依赖顺序排序服务
@NonCPS
def sortServicesByDependency(def services) {
    def deployOrder = [
        'viewsh-gateway',
        'viewsh-module-system-server',
        'viewsh-module-infra-server',
        'viewsh-module-iot-server',
        'viewsh-module-iot-gateway',
        'viewsh-module-ops-server'
    ]

    return services.sort { a, b ->
        deployOrder.indexOf(a) <=> deployOrder.indexOf(b)
    }
}

// 获取服务对应的容器名称
@NonCPS
def getContainerNameForService(String service) {
    def map = [
        'viewsh-gateway': 'aiot-gateway',
        'viewsh-module-system-server': 'aiot-system-server',
        'viewsh-module-infra-server': 'aiot-infra-server',
        'viewsh-module-iot-server': 'aiot-iot-server',
        'viewsh-module-iot-gateway': 'aiot-iot-gateway',
        'viewsh-module-ops-server': 'aiot-ops-server'
    ]
    return map.get(service, "aiot-${service}")
}

// 获取服务对应的模块路径
@NonCPS
def getModulePathForService(String service) {
    def map = [
        'viewsh-gateway': 'viewsh-gateway',
        'viewsh-module-system-server': 'viewsh-module-system/viewsh-module-system-server',
        'viewsh-module-infra-server': 'viewsh-module-infra/viewsh-module-infra-server',
        'viewsh-module-iot-server': 'viewsh-module-iot/viewsh-module-iot-server',
        'viewsh-module-iot-gateway': 'viewsh-module-iot/viewsh-module-iot-gateway',
        'viewsh-module-ops-server': 'viewsh-module-ops/viewsh-module-ops-server'
    ]
    return map.get(service, service)
}