aiot-platform-cloud/Jenkinsfile

// ============================================
// AIOT Platform - Jenkins Pipeline (Enterprise Edition)
// 优化版本：错误处理 + 性能优化 + 完善日志
// ============================================

pipeline {
    agent any

    options {
        buildDiscarder(logRotator(
            numToKeepStr: '10',
            artifactNumToKeepStr: '5'
        ))
        disableConcurrentBuilds()
        timeout(time: 90, unit: 'MINUTES')
        timestamps()
        retry(1)  // 失败自动重试1次
    }

    environment {
        // 镜像仓库配置
        REGISTRY = 'localhost:5000'
        // 注意：IMAGE_TAG 将在 Checkout 阶段动态设置
        // IMAGE_TAG = "${BRANCH_NAME}-${BUILD_NUMBER}-${GIT_COMMIT}"
        DEPS_IMAGE = "${REGISTRY}/aiot-deps:latest"

        // 服务配置
        CORE_SERVICES = 'viewsh-gateway,viewsh-module-system-server,viewsh-module-infra-server,viewsh-module-iot-server,viewsh-module-iot-gateway'

        // 部署配置
        DEPLOY_HOST = '172.19.0.1'
        DEPLOY_PATH = '/opt/aiot-platform-cloud'
        SSH_KEY = '/var/jenkins_home/.ssh/id_rsa'

        // 性能配置
        MAX_PARALLEL_BUILDS = 2
        BUILD_TIMEOUT = 45  // 单个服务构建超时（分钟）
        DEPLOY_TIMEOUT = 10  // 单个服务部署超时（分钟）
        HEALTH_CHECK_TIMEOUT = 180  // 健康检查总超时（秒）
        HEALTH_CHECK_INTERVAL = 10   // 健康检查间隔（秒）
    }

    stages {
        stage('Initialize') {
            steps {
                script {
                    echo "=========================================="
                    echo "  AIOT Platform - CI/CD Pipeline"
                    echo "=========================================="
                    echo "Branch: ${env.BRANCH_NAME}"
                    echo "Build: #${env.BUILD_NUMBER}"
                    echo "Workspace: ${env.WORKSPACE}"
                    echo "=========================================="
                }
            }
        }

        stage('Checkout') {
            steps {
                retry(3) {
                    checkout scm
                }
                script {
                    // 动态设置环境变量（避免在 environment 块中使用 env 变量）
                    def shortCommit = sh(
                        script: 'git rev-parse --short HEAD',
                        returnStdout: true
                    ).trim()

                    env.IMAGE_TAG = "${env.BRANCH_NAME}-${env.BUILD_NUMBER}-${shortCommit}"
                    env.GIT_COMMIT_MSG = sh(
                        script: 'git log -1 --pretty=%B',
                        returnStdout: true
                    ).trim()

                    echo "📦 Commit: ${shortCommit}"
                    echo "📝 Message: ${env.GIT_COMMIT_MSG}"
                    echo "🏷️  Image Tag: ${env.IMAGE_TAG}"
                }
            }
        }

        stage('Detect Changes') {
            steps {
                script {
                    // 获取变更文件（只执行一次 git diff）
                    def changedFiles = getChangedFiles()
                    echo "📝 Changed files: ${changedFiles.size()} files"

                    // 判断需要构建的服务
                    env.SERVICES_TO_BUILD = detectServicesToBuild(changedFiles)
                    env.DEPS_CHANGED = checkIfDepsChanged(changedFiles)

                    if (env.SERVICES_TO_BUILD.isEmpty()) {
                        echo "⏭️  No changes detected, skipping build"
                        currentBuild.result = 'SUCCESS'
                        return  // 直接跳过后续阶段
                    }

                    echo "🔄 Services to build: ${env.SERVICES_TO_BUILD}"
                    echo "📦 Deps changed: ${env.DEPS_CHANGED}"

                    // 显示变更的服务
                    env.SERVICES_TO_BUILD.split(',').each { service ->
                        def module = getModulePathForService(service)
                        echo "   - ${service} (${module})"
                    }
                }
            }
        }

        stage('Pre-build Check') {
            when {
                expression { env.SERVICES_TO_BUILD != '' }
            }
            steps {
                script {
                    echo "🔍 Running pre-build checks..."

                    // 检查 Docker 是否可用
                    sh "docker version >/dev/null 2>&1 || { echo '❌ Docker not available'; exit 1; }"

                    // 检查磁盘空间
                    def diskUsage = sh(
                        script: "df ${env.WORKSPACE} | tail -1 | awk '{print \$5}' | sed 's/%//'",
                        returnStdout: true
                    ).trim() as int

                    if (diskUsage > 80) {
                        echo "⚠️  Disk usage is ${diskUsage}%, cleaning up..."
                        sh "docker system prune -f --volumes || true"
                    }

                    // 检查镜像仓库连接
                    sh """
                        curl -f ${env.REGISTRY}/v2/ >/dev/null 2>&1 || \
                        { echo '⚠️  Registry not accessible, will continue...'; }
                    """

                    echo "✅ Pre-build checks passed"
                }
            }
        }

        stage('Build Dependencies Image') {
            when {
                expression {
                    env.SERVICES_TO_BUILD != '' &&
                    (env.DEPS_CHANGED == 'true' || !depsImageExists())
                }
            }
            steps {
                script {
                    echo "📦 Building dependencies base image..."
                    timeout(time: 15, unit: 'MINUTES') {
                        sh """
                            set -e
                            echo "Building ${env.DEPS_IMAGE}..."
                            docker build \
                                -f docker/Dockerfile.deps \
                                -t ${env.DEPS_IMAGE} \
                                --build-arg BUILDKIT_INLINE_CACHE=1 \
                                .

                            docker push ${env.DEPS_IMAGE}
                            echo "✅ Dependencies image built and pushed"
                        """
                    }
                }
            }
        }

        stage('Build Services') {
            when {
                expression { env.SERVICES_TO_BUILD != '' }
            }
            steps {
                script {
                    def servicesToBuild = env.SERVICES_TO_BUILD.split(',')
                    echo "🔨 Building ${servicesToBuild.size()} services (parallelism: ${MAX_PARALLEL_BUILDS})"

                    // 分批并行构建
                    def buildTasks = [:]
                    def batchSize = env.MAX_PARALLEL_BUILDS.toInteger()

                    servicesToBuild.each { service ->
                        buildTasks[service] = {
                            buildServiceWithRetry(service)
                        }
                    }

                    // 限制并发数
                    parallel buildTasks

                    echo "✅ All services built successfully"

                    // 显示构建后的镜像信息
                    sh """
                        echo "📊 Built images:"
                        ${env.SERVICES_TO_BUILD.split(',').collect { "docker images ${env.REGISTRY}/${it} --format '  {{.Repository}}:{{.Tag}} - {{.Size}}'" }.join('\n                        ')}
                    """
                }
            }
        }

        stage('Deploy') {
            when {
                allOf {
                    expression { env.SERVICES_TO_BUILD != '' }
                    branch 'master'
                }
            }
            steps {
                script {
                    def servicesToDeploy = env.SERVICES_TO_BUILD.split(',')

                    // 按依赖顺序排序
                    def sortedServices = sortServicesByDependency(servicesToDeploy)

                    echo "🚀 Deploying ${sortedServices.size()} services in dependency order"
                    sortedServices.eachWithIndex { service, index ->
                        echo "   ${index + 1}. ${service}"
                    }

                    // 串行部署（保证依赖关系）
                    sortedServices.each { service ->
                        deployServiceWithTimeout(service)
                    }

                    echo "🚀 All services deployed successfully!"
                }
            }
        }

        stage('Final Health Check') {
            when {
                allOf {
                    expression { env.SERVICES_TO_BUILD != '' }
                    branch 'master'
                }
            }
            steps {
                script {
                    echo "🏥 Running final health check for all services..."

                    def servicesToCheck = env.SERVICES_TO_BUILD.split(',')
                    def healthCheckTasks = [:]

                    servicesToCheck.each { service ->
                        healthCheckTasks[service] = {
                            checkServiceHealthWithRetry(service)
                        }
                    }

                    // 并行健康检查
                    parallel healthCheckTasks

                    echo "✅ All services are healthy!"

                    // 显示最终状态
                    sh """
                        echo "📊 Service Status:"
                        ${servicesToCheck.collect {
                            def container = getContainerNameForService(it)
                            "docker inspect --format='${it}: {{.State.Status}} ({{.State.Health.Status}})' ${container} 2>/dev/null || echo '${it}: not found'"
                        }.join('\n                        ')}
                    """
                }
            }
        }
    }

    post {
        success {
            script {
                echo """
                ==========================================
                ✅ BUILD SUCCESS
                ==========================================
                📦 Services: ${env.SERVICES_TO_BUILD}
                🏷️  Tag: ${env.IMAGE_TAG}
                ⏱️  Duration: ${currentBuild.durationString}
                ==========================================
                """
            }
        }
        failure {
            script {
                echo """
                ==========================================
                ❌ BUILD FAILED
                ==========================================
                📦 Services: ${env.SERVICES_TO_BUILD ?: 'None'}
                🏷️  Tag: ${env.IMAGE_TAG ?: 'Unknown'}
                ⚠️  Please check the logs above
                ==========================================
                """

                // 失败时收集诊断信息
                sh '''
                    echo "=== Docker System Info ==="
                    docker system df
                    echo ""
                    echo "=== Disk Usage ==="
                    df -h | grep -E "/$|/var"
                    echo ""
                    echo "=== Recent Containers ==="
                    docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.CreatedAt}}" | head -10
                '''
            }
        }
        always {
            script {
                echo "🧹 Cleaning up..."

                // 清理悬空的镜像
                sh "docker image prune -f || true"

                // 清理超过30天的构建日志
                sh """
                    find ${env.WORKSPACE} -name '*.log' -mtime +30 -delete 2>/dev/null || true
                """

                echo "📊 Final System Status:"
                sh 'df -h | grep -E "/$|/var" || true'
                sh 'docker system df || true'
            }
        }
    }
}

// ============================================
// 辅助函数
// ============================================

// 获取变更的文件列表
@NonCPS
def getChangedFiles() {
    def changedFiles = sh(
        script: '''
            PREV=$(git rev-parse HEAD~1 2>/dev/null || echo "")
            [ -z "$PREV" ] && echo "all" || git diff --name-only $PREV HEAD
        ''',
        returnStdout: true
    ).trim()

    if (changedFiles == 'all') {
        return ['all'] as List<String>
    }

    return changedFiles.split('\n') as List<String>
}

// 检测是否需要重建依赖镜像
@NonCPS
def checkIfDepsChanged(List<String> changedFiles) {
    if (changedFiles.contains('all')) {
        return 'true'
    }

    def depsFiles = ['pom.xml', 'viewsh-dependencies', 'viewsh-framework', 'docker/Dockerfile.deps']
    return depsFiles.any { changedFiles.contains(it) } ? 'true' : 'false'
}

// 检测需要构建的服务
@NonCPS
def detectServicesToBuild(List<String> changedFiles) {
    // 如果是第一次构建或强制全量构建
    if (changedFiles.contains('all')) {
        return env.CORE_SERVICES
    }

    // 检查是否触发了全量构建
    def triggerAllFiles = ['pom.xml', 'viewsh-framework', 'viewsh-dependencies', 'Jenkinsfile', 'docker/']
    if (triggerAllFiles.any { triggerFile ->
        changedFiles.any { changedFile ->
            changedFile.startsWith(triggerFile) || changedFile == triggerFile
        }
    }) {
        return env.CORE_SERVICES
    }

    // 检测变更的模块
    def changedServices = []
    def allServices = env.CORE_SERVICES.split(',')

    allServices.each { service ->
        def modulePath = getModulePathForService(service)
        def moduleDir = modulePath.split('/')[0]

        if (changedFiles.any { it.startsWith(moduleDir) }) {
            changedServices.add(service)
        }
    }

    return changedServices.isEmpty() ? env.CORE_SERVICES : changedServices.join(',')
}

// 检查依赖镜像是否存在
@NonCPS
def depsImageExists() {
    def result = sh(
        script: "docker manifest inspect ${env.DEPS_IMAGE} > /dev/null 2>&1",
        returnStatus: true
    )
    return result == 0
}

// 构建服务（带重试）
def buildServiceWithRetry(String service) {
    retry(2) {
        timeout(time: env.BUILD_TIMEOUT.toInteger(), unit: 'MINUTES') {
            buildService(service)
        }
    }
}

// 构建单个服务
def buildService(String service) {
    def modulePath = getModulePathForService(service)

    echo ""
    echo "=========================================="
    echo "🔨 Building ${service}"
    echo "=========================================="
    echo "Module: ${modulePath}"
    echo "Registry: ${env.REGISTRY}"
    echo "Tag: ${env.IMAGE_TAG}"
    echo "=========================================="

    try {
        sh """
            set -e
            set -x

            # 构建镜像
            docker build \\
                -f docker/Dockerfile.service \\
                --build-arg DEPS_IMAGE=${env.DEPS_IMAGE} \\
                --build-arg MODULE_NAME=${modulePath} \\
                --build-arg JAR_NAME=${service} \\
                --build-arg SKIP_TESTS=true \\
                -t ${env.REGISTRY}/${service}:${env.IMAGE_TAG} \\
                -t ${env.REGISTRY}/${service}:latest \\
                .

            # 推送镜像
            docker push ${env.REGISTRY}/${service}:${env.IMAGE_TAG}
            docker push ${env.REGISTRY}/${service}:latest

            set +x
        """

        echo "✅ ${service} built and pushed successfully"

        // 获取镜像大小
        def imageSize = sh(
            script: "docker images ${env.REGISTRY}/${service}:latest --format '{{.Size}}'",
            returnStdout: true
        ).trim()

        echo "📊 Image size: ${imageSize}"

    } catch (Exception e) {
        echo "❌ Failed to build ${service}: ${e.message}"

        // 打印构建日志以便调试
        sh """
            echo "=== Docker Build Logs for ${service} ==="
            docker logs ${service}-builder 2>/dev/null || true
        """

        throw e
    }
}

// 部署服务（带超时）
def deployServiceWithTimeout(String service) {
    timeout(time: env.DEPLOY_TIMEOUT.toInteger(), unit: 'MINUTES') {
        deployService(service)
    }
}

// 部署单个服务
def deployService(String service) {
    def containerName = getContainerNameForService(service)
    def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"

    echo ""
    echo "=========================================="
    echo "🚀 Deploying ${service}"
    echo "=========================================="
    echo "Container: ${containerName}"
    echo "Host: ${env.DEPLOY_HOST}"
    echo "=========================================="

    try {
        // 部署服务
        sh """
            ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                set -e
                cd ${env.DEPLOY_PATH}

                echo "📥 Pulling ${service}..."
                docker compose -f docker-compose.core.yml pull ${service}

                echo "🔄 Restarting ${service}..."
                docker compose -f docker-compose.core.yml up -d ${service}

                echo "⏳ Waiting for container to start..."
                sleep 5
            '
        """

        // 等待服务健康
        waitForServiceHealthy(containerName, service, sshOpts)

        echo "✅ ${service} deployed successfully"

    } catch (Exception e) {
        echo "❌ Failed to deploy ${service}: ${e.message}"

        // 收集诊断信息
        sh """
            ssh ${sshOpts} root@${env.DEPLOY_HOST} '
                echo "=== Container Status ==="
                docker ps -a | grep ${containerName} || true

                echo ""
                echo "=== Container Logs (last 50 lines) ==="
                docker logs --tail 50 ${containerName} 2>/dev/null || true

                echo ""
                echo "=== Service Health Status ==="
                docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found"
            '
        """

        throw e
    }
}

// 等待服务健康
def waitForServiceHealthy(String containerName, String serviceName, String sshOpts) {
    def maxAttempts = env.HEALTH_CHECK_TIMEOUT.toInteger() / env.HEALTH_CHECK_INTERVAL.toInteger()

    echo "⏳ Waiting for ${serviceName} to be healthy (max ${env.HEALTH_CHECK_TIMEOUT}s)..."

    sh """
        ssh ${sshOpts} root@${env.DEPLOY_HOST} '
            set -e

            for i in $(seq 1 ${maxAttempts}); do
                STATUS=$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "starting")

                case "$STATUS" in
                    healthy)
                        echo "✅ ${serviceName} is healthy"
                        exit 0
                        ;;
                    unhealthy)
                        echo "❌ ${serviceName} is unhealthy"
                        echo "=== Last 100 lines of logs ==="
                        docker logs --tail 100 ${containerName}
                        exit 1
                        ;;
                    starting)
                        ELAPSED=$((i * ${env.HEALTH_CHECK_INTERVAL}))
                        echo "⏳ ${serviceName} is starting... (\${ELAPSED}s/${env.HEALTH_CHECK_TIMEOUT}s)"
                        ;;
                    *)
                        echo "⚠️  ${serviceName} status: \$STATUS"
                        ;;
                esac

                sleep ${env.HEALTH_CHECK_INTERVAL}
            done

            echo "❌ ${serviceName} health check timeout after ${env.HEALTH_CHECK_TIMEOUT}s"
            echo "=== Full logs ==="
            docker logs ${containerName}
            exit 1
        '
    """
}

// 检查服务健康（带重试）
def checkServiceHealthWithRetry(String service) {
    def containerName = getContainerNameForService(service)
    def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"

    retry(3) {
        timeout(time: 2, unit: 'MINUTES') {
            checkServiceHealth(containerName, service, sshOpts)
        }
    }
}

// 检查服务健康状态
def checkServiceHealth(String containerName, String serviceName, String sshOpts) {
    sh """
        ssh ${sshOpts} root@${env.DEPLOY_HOST} '
            STATUS=$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found")

            case "$STATUS" in
                healthy)
                    echo "✅ ${serviceName} is healthy"
                    ;;
                not_found)
                    echo "⚠️  ${serviceName} not found (may not be deployed)"
                    exit 1
                    ;;
                *)
                    echo "❌ ${serviceName} is \$STATUS"
                    echo "=== Last 50 lines of logs ==="
                    docker logs --tail 50 ${containerName}
                    exit 1
                    ;;
            esac
        '
    """
}

// 按依赖顺序排序服务
@NonCPS
def sortServicesByDependency(def services) {
    def deployOrder = [
        'viewsh-gateway',
        'viewsh-module-system-server',
        'viewsh-module-infra-server',
        'viewsh-module-iot-server',
        'viewsh-module-iot-gateway'
    ]

    return services.sort { a, b ->
        deployOrder.indexOf(a) <=> deployOrder.indexOf(b)
    }
}

// 获取服务对应的容器名称
@NonCPS
def getContainerNameForService(String service) {
    def map = [
        'viewsh-gateway': 'aiot-gateway',
        'viewsh-module-system-server': 'aiot-system-server',
        'viewsh-module-infra-server': 'aiot-infra-server',
        'viewsh-module-iot-server': 'aiot-iot-server',
        'viewsh-module-iot-gateway': 'aiot-iot-gateway'
    ]
    return map.get(service, "aiot-${service}")
}

// 获取服务对应的模块路径
@NonCPS
def getModulePathForService(String service) {
    def map = [
        'viewsh-gateway': 'viewsh-gateway',
        'viewsh-module-system-server': 'viewsh-module-system/viewsh-module-system-server',
        'viewsh-module-infra-server': 'viewsh-module-infra/viewsh-module-infra-server',
        'viewsh-module-iot-server': 'viewsh-module-iot/viewsh-module-iot-server',
        'viewsh-module-iot-gateway': 'viewsh-module-iot/viewsh-module-iot-gateway'
    ]
    return map.get(service, service)
}