lzh
e4d07a5306
原 getChangedFiles() 只对比 HEAD~1 和 HEAD，导致一次 push 多个 commits 时只会检测最新一个 commit 的变更。 修改为使用 GIT_PREVIOUS_SUCCESSFUL_COMMIT（Jenkins 内置变量，上次 成功构建的 commit）作为基准，确保所有变更文件都能被正确检测。 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1052 lines
37 KiB
Groovy
1052 lines
37 KiB
Groovy
// ============================================
|
||
// AIOT Platform - Jenkins Pipeline (Optimized Edition)
|
||
// ============================================
|
||
|
||
pipeline {
|
||
agent any
|
||
|
||
options {
|
||
buildDiscarder(logRotator(
|
||
numToKeepStr: '10',
|
||
artifactNumToKeepStr: '5'
|
||
))
|
||
disableConcurrentBuilds()
|
||
timeout(time: 90, unit: 'MINUTES')
|
||
timestamps()
|
||
retry(1)
|
||
}
|
||
|
||
environment {
|
||
// 镜像仓库配置
|
||
REGISTRY = 'localhost:5000'
|
||
DEPS_IMAGE = "${REGISTRY}/aiot-deps:latest"
|
||
|
||
// 服务配置
|
||
CORE_SERVICES = 'viewsh-gateway,viewsh-module-system-server,viewsh-module-infra-server,viewsh-module-iot-server,viewsh-module-iot-gateway,viewsh-module-ops-server'
|
||
|
||
// 部署配置
|
||
DEPLOY_HOST = '172.19.0.1'
|
||
DEPLOY_PATH = '/opt/aiot-platform-cloud'
|
||
SSH_KEY = '/var/jenkins_home/.ssh/id_rsa'
|
||
|
||
// 性能配置 - 将动态调整
|
||
BUILD_TIMEOUT = 45
|
||
DEPLOY_TIMEOUT = 10
|
||
HEALTH_CHECK_TIMEOUT = 180
|
||
HEALTH_CHECK_INTERVAL = 10
|
||
|
||
// 【优化1】Maven 缓存配置
|
||
MAVEN_CACHE_VOLUME = 'jenkins-maven-cache'
|
||
MAVEN_OPTS = '-Dmaven.repo.local=/var/jenkins_home/.m2/repository'
|
||
|
||
// 【优化7】回滚配置
|
||
ROLLBACK_ENABLED = 'true'
|
||
KEEP_PREVIOUS_IMAGES = 'true'
|
||
}
|
||
|
||
stages {
|
||
stage('Initialize') {
|
||
steps {
|
||
script {
|
||
// 【优化6】记录开始时间
|
||
env.PIPELINE_START_TIME = System.currentTimeMillis()
|
||
|
||
echo "=========================================="
|
||
echo " AIOT Platform - CI/CD Pipeline (Optimized)"
|
||
echo "=========================================="
|
||
echo "Branch: ${env.BRANCH_NAME}"
|
||
echo "Build: #${env.BUILD_NUMBER}"
|
||
echo "Workspace: ${env.WORKSPACE}"
|
||
echo "Start Time: ${new Date()}"
|
||
echo "=========================================="
|
||
|
||
// 【优化2】动态检测系统资源
|
||
detectSystemResources()
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Checkout') {
|
||
steps {
|
||
script {
|
||
def stageStartTime = System.currentTimeMillis()
|
||
|
||
retry(3) {
|
||
checkout scm
|
||
}
|
||
|
||
def shortCommit = sh(
|
||
script: 'git rev-parse --short HEAD',
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
// 清理分支名：将斜杠替换为连字符，确保 Docker 标签格式有效
|
||
def sanitizedBranchName = env.BRANCH_NAME.replaceAll('/', '-')
|
||
env.IMAGE_TAG = "${sanitizedBranchName}-${env.BUILD_NUMBER}-${shortCommit}"
|
||
env.PREVIOUS_IMAGE_TAG = getPreviousImageTag()
|
||
env.GIT_COMMIT_MSG = sh(
|
||
script: 'git log -1 --pretty=%B',
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
echo "📦 Commit: ${shortCommit}"
|
||
echo "📝 Message: ${env.GIT_COMMIT_MSG}"
|
||
echo "🏷️ Current Tag: ${env.IMAGE_TAG}"
|
||
echo "🔖 Previous Tag: ${env.PREVIOUS_IMAGE_TAG}"
|
||
|
||
// 【优化6】记录阶段耗时
|
||
recordStageMetrics('Checkout', stageStartTime)
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Detect Changes') {
|
||
steps {
|
||
script {
|
||
def stageStartTime = System.currentTimeMillis()
|
||
|
||
def changedFiles = getChangedFiles()
|
||
echo "📝 Changed files: ${changedFiles.size()} files"
|
||
|
||
env.SERVICES_TO_BUILD = detectServicesToBuild(changedFiles)
|
||
env.DEPS_CHANGED = checkIfDepsChanged(changedFiles)
|
||
|
||
// Check if deps image exists
|
||
if (env.DEPS_CHANGED == 'false') {
|
||
def depsExists = sh(
|
||
script: "docker manifest inspect ${env.DEPS_IMAGE} > /dev/null 2>&1",
|
||
returnStatus: true
|
||
)
|
||
if (depsExists != 0) {
|
||
env.DEPS_CHANGED = 'true'
|
||
echo "📦 Deps image not found, will rebuild"
|
||
}
|
||
}
|
||
|
||
if (env.SERVICES_TO_BUILD.isEmpty()) {
|
||
echo "⏭️ No changes detected, skipping build"
|
||
currentBuild.result = 'SUCCESS'
|
||
return
|
||
}
|
||
|
||
echo "🔄 Services to build: ${env.SERVICES_TO_BUILD}"
|
||
echo "📦 Deps changed: ${env.DEPS_CHANGED}"
|
||
|
||
env.SERVICES_TO_BUILD.split(',').each { service ->
|
||
def module = getModulePathForService(service)
|
||
echo " - ${service} (${module})"
|
||
}
|
||
|
||
recordStageMetrics('Detect Changes', stageStartTime)
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Pre-build Check') {
|
||
when {
|
||
expression { env.SERVICES_TO_BUILD != '' }
|
||
}
|
||
steps {
|
||
script {
|
||
def stageStartTime = System.currentTimeMillis()
|
||
|
||
echo "🔍 Running pre-build checks..."
|
||
|
||
// 检查 Docker
|
||
sh "docker version >/dev/null 2>&1 || { echo '❌ Docker not available'; exit 1; }"
|
||
|
||
// 【优化1】检查并创建 Maven 缓存卷
|
||
sh """
|
||
if ! docker volume inspect ${env.MAVEN_CACHE_VOLUME} >/dev/null 2>&1; then
|
||
echo "📦 Creating Maven cache volume: ${env.MAVEN_CACHE_VOLUME}"
|
||
docker volume create ${env.MAVEN_CACHE_VOLUME}
|
||
else
|
||
echo "✅ Maven cache volume exists: ${env.MAVEN_CACHE_VOLUME}"
|
||
fi
|
||
"""
|
||
|
||
// 检查磁盘空间
|
||
def diskUsage = sh(
|
||
script: "df ${env.WORKSPACE} | tail -1 | awk '{print \$5}' | sed 's/%//'",
|
||
returnStdout: true
|
||
).trim() as int
|
||
|
||
if (diskUsage > 80) {
|
||
echo "⚠️ Disk usage is ${diskUsage}%, cleaning up..."
|
||
sh "docker system prune -f --volumes || true"
|
||
}
|
||
|
||
// 检查镜像仓库连接
|
||
sh """
|
||
curl -f ${env.REGISTRY}/v2/ >/dev/null 2>&1 || \
|
||
{ echo '⚠️ Registry not accessible, will continue...'; }
|
||
"""
|
||
|
||
echo "✅ Pre-build checks passed"
|
||
recordStageMetrics('Pre-build Check', stageStartTime)
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Build Dependencies Image') {
|
||
when {
|
||
expression {
|
||
env.SERVICES_TO_BUILD != '' &&
|
||
env.DEPS_CHANGED == 'true'
|
||
}
|
||
}
|
||
steps {
|
||
script {
|
||
def stageStartTime = System.currentTimeMillis()
|
||
|
||
echo "📦 Building dependencies base image with Maven cache..."
|
||
timeout(time: 15, unit: 'MINUTES') {
|
||
// 【优化1】使用 Maven 缓存卷加速依赖下载
|
||
sh """
|
||
set -e
|
||
echo "Building ${env.DEPS_IMAGE} with cache..."
|
||
docker build \
|
||
-f docker/Dockerfile.deps \
|
||
-t ${env.DEPS_IMAGE} \
|
||
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
||
--build-arg MAVEN_OPTS="${env.MAVEN_OPTS}" \
|
||
.
|
||
|
||
docker push ${env.DEPS_IMAGE}
|
||
echo "✅ Dependencies image built and pushed"
|
||
"""
|
||
}
|
||
|
||
recordStageMetrics('Build Dependencies Image', stageStartTime)
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Build Services') {
|
||
when {
|
||
expression { env.SERVICES_TO_BUILD != '' }
|
||
}
|
||
steps {
|
||
script {
|
||
def stageStartTime = System.currentTimeMillis()
|
||
|
||
def servicesToBuild = env.SERVICES_TO_BUILD.split(',')
|
||
echo "🔨 Building ${servicesToBuild.size()} services (parallelism: ${env.MAX_PARALLEL_BUILDS})"
|
||
|
||
// 【优化2】动态并行构建
|
||
def buildTasks = [:]
|
||
servicesToBuild.each { service ->
|
||
buildTasks[service] = {
|
||
buildServiceWithRetry(service)
|
||
}
|
||
}
|
||
|
||
parallel buildTasks
|
||
|
||
echo "✅ All services built successfully"
|
||
|
||
// 显示构建后的镜像信息
|
||
sh """
|
||
echo "📊 Built images:"
|
||
${env.SERVICES_TO_BUILD.split(',').collect { "docker images ${env.REGISTRY}/${it} --format ' {{.Repository}}:{{.Tag}} - {{.Size}}'" }.join('\n ')}
|
||
"""
|
||
|
||
recordStageMetrics('Build Services', stageStartTime)
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Deploy') {
|
||
when {
|
||
allOf {
|
||
expression { env.SERVICES_TO_BUILD != '' }
|
||
branch 'master'
|
||
}
|
||
}
|
||
steps {
|
||
script {
|
||
def stageStartTime = System.currentTimeMillis()
|
||
|
||
def servicesToDeploy = env.SERVICES_TO_BUILD.split(',')
|
||
def sortedServices = sortServicesByDependency(servicesToDeploy)
|
||
|
||
echo "🚀 Deploying ${sortedServices.size()} services in dependency order"
|
||
sortedServices.eachWithIndex { service, index ->
|
||
echo " ${index + 1}. ${service}"
|
||
}
|
||
|
||
// 【优化7】部署前备份当前镜像标签
|
||
if (env.ROLLBACK_ENABLED == 'true') {
|
||
backupCurrentDeployment(sortedServices)
|
||
}
|
||
|
||
// 【新增】同步最新的 docker-compose.core.yml 到部署服务器
|
||
echo "📂 Syncing docker-compose.core.yml to deploy host..."
|
||
sh "scp -o StrictHostKeyChecking=no -i ${env.SSH_KEY} docker-compose.core.yml root@${env.DEPLOY_HOST}:${env.DEPLOY_PATH}/"
|
||
|
||
try {
|
||
// 串行部署（保证依赖关系）
|
||
sortedServices.each { service ->
|
||
deployServiceWithTimeout(service)
|
||
}
|
||
|
||
echo "🚀 All services deployed successfully!"
|
||
|
||
} catch (Exception e) {
|
||
// 【优化7】部署失败时自动回滚
|
||
if (env.ROLLBACK_ENABLED == 'true') {
|
||
echo "❌ Deployment failed: ${e.message}"
|
||
echo "🔄 Initiating automatic rollback..."
|
||
rollbackDeployment(sortedServices)
|
||
}
|
||
throw e
|
||
}
|
||
|
||
recordStageMetrics('Deploy', stageStartTime)
|
||
}
|
||
}
|
||
}
|
||
|
||
stage('Final Health Check') {
|
||
when {
|
||
allOf {
|
||
expression { env.SERVICES_TO_BUILD != '' }
|
||
branch 'master'
|
||
}
|
||
}
|
||
steps {
|
||
script {
|
||
def stageStartTime = System.currentTimeMillis()
|
||
|
||
echo "🏥 Running final health check for all services..."
|
||
|
||
def servicesToCheck = env.SERVICES_TO_BUILD.split(',')
|
||
def healthCheckTasks = [:]
|
||
|
||
servicesToCheck.each { service ->
|
||
healthCheckTasks[service] = {
|
||
checkServiceHealthWithRetry(service)
|
||
}
|
||
}
|
||
|
||
try {
|
||
parallel healthCheckTasks
|
||
echo "✅ All services are healthy!"
|
||
} catch (Exception e) {
|
||
// 【优化7】健康检查失败时回滚
|
||
if (env.ROLLBACK_ENABLED == 'true') {
|
||
echo "❌ Health check failed: ${e.message}"
|
||
echo "🔄 Initiating automatic rollback..."
|
||
rollbackDeployment(servicesToCheck)
|
||
}
|
||
throw e
|
||
}
|
||
|
||
// 显示最终状态
|
||
sh """
|
||
echo "📊 Service Status:"
|
||
${servicesToCheck.collect {
|
||
def container = getContainerNameForService(it)
|
||
"docker inspect --format='${it}: {{.State.Status}} ({{.State.Health.Status}})' ${container} 2>/dev/null || echo '${it}: not found'"
|
||
}.join('\n ')}
|
||
"""
|
||
|
||
recordStageMetrics('Final Health Check', stageStartTime)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
post {
|
||
success {
|
||
script {
|
||
// 【优化6】计算总耗时并生成性能报告
|
||
def totalDuration = System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong()
|
||
generatePerformanceReport(totalDuration)
|
||
|
||
echo """
|
||
==========================================
|
||
✅ BUILD SUCCESS
|
||
==========================================
|
||
📦 Services: ${env.SERVICES_TO_BUILD}
|
||
🏷️ Tag: ${env.IMAGE_TAG}
|
||
⏱️ Duration: ${formatDuration(totalDuration)}
|
||
==========================================
|
||
"""
|
||
}
|
||
}
|
||
failure {
|
||
script {
|
||
def totalDuration = System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong()
|
||
|
||
echo """
|
||
==========================================
|
||
❌ BUILD FAILED
|
||
==========================================
|
||
📦 Services: ${env.SERVICES_TO_BUILD ?: 'None'}
|
||
🏷️ Tag: ${env.IMAGE_TAG ?: 'Unknown'}
|
||
🔖 Rollback Tag: ${env.PREVIOUS_IMAGE_TAG ?: 'N/A'}
|
||
⏱️ Duration: ${formatDuration(totalDuration)}
|
||
⚠️ Please check the logs above
|
||
==========================================
|
||
"""
|
||
|
||
// 失败时收集诊断信息
|
||
sh '''
|
||
echo "=== Docker System Info ==="
|
||
docker system df
|
||
echo ""
|
||
echo "=== Disk Usage ==="
|
||
df -h | grep -E "/$|/var"
|
||
echo ""
|
||
echo "=== Recent Containers ==="
|
||
docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.CreatedAt}}" | head -10
|
||
'''
|
||
}
|
||
}
|
||
always {
|
||
script {
|
||
echo "🧹 Cleaning up..."
|
||
|
||
// 清理悬空的镜像（但保留带标签的镜像用于回滚）
|
||
if (env.KEEP_PREVIOUS_IMAGES == 'true') {
|
||
sh "docker image prune -f --filter 'dangling=true' || true"
|
||
} else {
|
||
sh "docker image prune -f || true"
|
||
}
|
||
|
||
// 清理超过30天的构建日志
|
||
sh """
|
||
find ${env.WORKSPACE} -name '*.log' -mtime +30 -delete 2>/dev/null || true
|
||
"""
|
||
|
||
echo "📊 Final System Status:"
|
||
sh 'df -h | grep -E "/$|/var" || true'
|
||
sh 'docker system df || true'
|
||
|
||
// 【优化6】保存性能指标到文件
|
||
archivePerformanceMetrics()
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ============================================
|
||
// 辅助函数
|
||
// ============================================
|
||
|
||
// 获取变更的文件列表（对比上次成功构建，支持多 commit）
|
||
def getChangedFiles() {
|
||
def changedFiles = sh(
|
||
script: '''
|
||
# 优先使用上次成功构建的 commit
|
||
if [ -n "$GIT_PREVIOUS_SUCCESSFUL_COMMIT" ]; then
|
||
PREV="$GIT_PREVIOUS_SUCCESSFUL_COMMIT"
|
||
else
|
||
# 回退到 origin/master（如果有）
|
||
PREV=$(git rev-parse origin/master 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || echo "")
|
||
fi
|
||
|
||
if [ -z "$PREV" ]; then
|
||
echo "all"
|
||
else
|
||
git diff --name-only $PREV HEAD
|
||
fi
|
||
''',
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
if (changedFiles == 'all') {
|
||
return ['all'] as List<String>
|
||
}
|
||
|
||
return changedFiles.split('\n') as List<String>
|
||
}
|
||
|
||
// 检测是否需要重建依赖镜像
|
||
@NonCPS
|
||
def checkIfDepsChanged(List<String> changedFiles) {
|
||
if (changedFiles.contains('all')) {
|
||
return 'true'
|
||
}
|
||
|
||
def depsFiles = ['pom.xml', 'viewsh-dependencies', 'viewsh-framework', 'docker/Dockerfile.deps']
|
||
return depsFiles.any { changedFiles.contains(it) } ? 'true' : 'false'
|
||
}
|
||
|
||
// 检测需要构建的服务
|
||
def detectServicesToBuild(List<String> changedFiles) {
|
||
if (changedFiles.contains('all')) {
|
||
return env.CORE_SERVICES
|
||
}
|
||
|
||
def triggerAllFiles = ['pom.xml', 'viewsh-framework', 'viewsh-dependencies', 'Jenkinsfile', 'docker/']
|
||
if (triggerAllFiles.any { triggerFile ->
|
||
changedFiles.any { changedFile ->
|
||
changedFile.startsWith(triggerFile) || changedFile == triggerFile
|
||
}
|
||
}) {
|
||
return env.CORE_SERVICES
|
||
}
|
||
|
||
def changedServices = []
|
||
def allServices = env.CORE_SERVICES.split(',')
|
||
|
||
allServices.each { service ->
|
||
def modulePath = getModulePathForService(service)
|
||
def moduleDir = modulePath.split('/')[0]
|
||
|
||
if (changedFiles.any { it.startsWith(moduleDir) }) {
|
||
changedServices.add(service)
|
||
}
|
||
}
|
||
|
||
return changedServices.isEmpty() ? env.CORE_SERVICES : changedServices.join(',')
|
||
}
|
||
|
||
// 检查依赖镜像是否存在
|
||
def depsImageExists() {
|
||
def result = sh(
|
||
script: "docker manifest inspect ${env.DEPS_IMAGE} > /dev/null 2>&1",
|
||
returnStatus: true
|
||
)
|
||
return result == 0
|
||
}
|
||
|
||
// 【优化2】动态检测系统资源并设置并行度
|
||
def detectSystemResources() {
|
||
try {
|
||
// 获取 CPU 核心数
|
||
def cpuCores = sh(
|
||
script: 'nproc 2>/dev/null || echo "2"',
|
||
returnStdout: true
|
||
).trim() as int
|
||
|
||
// 获取可用内存（GB）
|
||
def availableMemory = sh(
|
||
script: 'free -g | grep Mem | awk \'{print $7}\'',
|
||
returnStdout: true
|
||
).trim() as int
|
||
|
||
// 动态计算并行度
|
||
// 规则：每个构建任务需要至少 2GB 内存和 1 个 CPU 核心
|
||
def maxParallelByMemory = Math.max(1, (availableMemory / 2) as int)
|
||
def maxParallelByCpu = Math.max(1, cpuCores - 1) // 保留一个核心给系统
|
||
|
||
env.MAX_PARALLEL_BUILDS = Math.min(maxParallelByMemory, maxParallelByCpu).toString()
|
||
|
||
echo """
|
||
========================================
|
||
📊 System Resources Detected:
|
||
========================================
|
||
CPU Cores: ${cpuCores}
|
||
Available Memory: ${availableMemory} GB
|
||
Max Parallel Builds: ${env.MAX_PARALLEL_BUILDS}
|
||
========================================
|
||
"""
|
||
} catch (Exception e) {
|
||
echo "⚠️ Failed to detect system resources: ${e.message}"
|
||
echo "Using default parallelism: 2"
|
||
env.MAX_PARALLEL_BUILDS = '2'
|
||
}
|
||
}
|
||
|
||
// 【优化6】记录阶段性能指标
|
||
def recordStageMetrics(String stageName, long startTime) {
|
||
def duration = System.currentTimeMillis() - startTime
|
||
def durationStr = formatDuration(duration)
|
||
|
||
if (!env.STAGE_METRICS) {
|
||
env.STAGE_METRICS = ""
|
||
}
|
||
|
||
env.STAGE_METRICS += "${stageName}:${duration}|"
|
||
echo "⏱️ Stage '${stageName}' completed in ${durationStr}"
|
||
}
|
||
|
||
// 【优化6】格式化时长
|
||
@NonCPS
|
||
def formatDuration(long milliseconds) {
|
||
def seconds = (milliseconds / 1000) as int
|
||
def minutes = (seconds / 60) as int
|
||
def hours = (minutes / 60) as int
|
||
|
||
if (hours > 0) {
|
||
return "${hours}h ${minutes % 60}m ${seconds % 60}s"
|
||
} else if (minutes > 0) {
|
||
return "${minutes}m ${seconds % 60}s"
|
||
} else {
|
||
return "${seconds}s"
|
||
}
|
||
}
|
||
|
||
// 【优化6】生成性能报告
|
||
def generatePerformanceReport(long totalDuration) {
|
||
echo """
|
||
==========================================
|
||
📊 PERFORMANCE REPORT
|
||
==========================================
|
||
Total Duration: ${formatDuration(totalDuration)}
|
||
"""
|
||
|
||
if (env.STAGE_METRICS) {
|
||
echo "Stage Breakdown:"
|
||
env.STAGE_METRICS.split('\\|').each { metric ->
|
||
if (metric) {
|
||
def parts = metric.split(':')
|
||
if (parts.size() == 2) {
|
||
def stageName = parts[0]
|
||
def duration = parts[1] as long
|
||
def percentage = (duration * 100 / totalDuration) as int
|
||
echo " - ${stageName.padRight(25)}: ${formatDuration(duration).padRight(10)} (${percentage}%)"
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
echo "=========================================="
|
||
}
|
||
|
||
// 【优化6】归档性能指标
|
||
def archivePerformanceMetrics() {
|
||
try {
|
||
def metricsFile = "${env.WORKSPACE}/build-metrics-${env.BUILD_NUMBER}.json"
|
||
def metricsData = [
|
||
buildNumber: env.BUILD_NUMBER,
|
||
timestamp: new Date().format('yyyy-MM-dd HH:mm:ss'),
|
||
branch: env.BRANCH_NAME,
|
||
imageTag: env.IMAGE_TAG,
|
||
servicesToBuild: env.SERVICES_TO_BUILD,
|
||
totalDuration: System.currentTimeMillis() - env.PIPELINE_START_TIME.toLong(),
|
||
stages: [:]
|
||
]
|
||
|
||
if (env.STAGE_METRICS) {
|
||
env.STAGE_METRICS.split('\\|').each { metric ->
|
||
if (metric) {
|
||
def parts = metric.split(':')
|
||
if (parts.size() == 2) {
|
||
metricsData.stages[parts[0]] = parts[1] as long
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// writeJSON file: metricsFile, json: metricsData
|
||
writeFile file: metricsFile, text: groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(metricsData))
|
||
archiveArtifacts artifacts: "build-metrics-${env.BUILD_NUMBER}.json", allowEmptyArchive: true
|
||
|
||
echo "✅ Performance metrics archived: ${metricsFile}"
|
||
} catch (Exception e) {
|
||
echo "⚠️ Failed to archive performance metrics: ${e.message}"
|
||
}
|
||
}
|
||
|
||
// 【优化7】获取上一次成功部署的镜像标签
|
||
def getPreviousImageTag() {
|
||
try {
|
||
// 从部署主机获取当前运行的镜像标签
|
||
def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"
|
||
def previousTag = sh(
|
||
script: """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
cd ${env.DEPLOY_PATH}
|
||
docker compose -f docker-compose.core.yml images --format json | \
|
||
jq -r ".[0].Tag" | head -1
|
||
' 2>/dev/null || echo "latest"
|
||
""",
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
return previousTag ?: 'latest'
|
||
} catch (Exception e) {
|
||
echo "⚠️ Failed to get previous image tag: ${e.message}"
|
||
return 'latest'
|
||
}
|
||
}
|
||
|
||
// 【优化7】备份当前部署
|
||
def backupCurrentDeployment(def services) {
|
||
echo "💾 Backing up current deployment state..."
|
||
|
||
def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"
|
||
|
||
try {
|
||
sh """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
cd ${env.DEPLOY_PATH}
|
||
|
||
# 保存当前 docker-compose 配置
|
||
cp docker-compose.core.yml docker-compose.core.yml.backup-${env.BUILD_NUMBER}
|
||
|
||
# 记录当前运行的镜像
|
||
docker compose -f docker-compose.core.yml images > deployment-state-${env.BUILD_NUMBER}.txt
|
||
|
||
echo "✅ Backup completed: deployment-state-${env.BUILD_NUMBER}.txt"
|
||
'
|
||
"""
|
||
|
||
echo "✅ Current deployment backed up successfully"
|
||
} catch (Exception e) {
|
||
echo "⚠️ Failed to backup current deployment: ${e.message}"
|
||
}
|
||
}
|
||
|
||
// 【优化7】回滚部署
|
||
def rollbackDeployment(def services) {
|
||
echo """
|
||
==========================================
|
||
🔄 INITIATING ROLLBACK
|
||
==========================================
|
||
Rolling back to: ${env.PREVIOUS_IMAGE_TAG}
|
||
Services: ${services.join(', ')}
|
||
==========================================
|
||
"""
|
||
|
||
def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"
|
||
|
||
try {
|
||
services.each { service ->
|
||
echo "🔄 Rolling back ${service}..."
|
||
|
||
sh """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
cd ${env.DEPLOY_PATH}
|
||
|
||
# 设置回滚镜像标签
|
||
export IMAGE_TAG=${env.PREVIOUS_IMAGE_TAG}
|
||
|
||
# 拉取旧版本镜像
|
||
docker compose -f docker-compose.core.yml pull ${service}
|
||
|
||
# 重启服务
|
||
docker compose -f docker-compose.core.yml up -d ${service}
|
||
|
||
echo "✅ ${service} rolled back to ${env.PREVIOUS_IMAGE_TAG}"
|
||
'
|
||
"""
|
||
|
||
// 等待服务启动
|
||
sleep 5
|
||
}
|
||
|
||
echo """
|
||
==========================================
|
||
✅ ROLLBACK COMPLETED
|
||
==========================================
|
||
All services have been rolled back to: ${env.PREVIOUS_IMAGE_TAG}
|
||
==========================================
|
||
"""
|
||
|
||
} catch (Exception e) {
|
||
echo """
|
||
==========================================
|
||
❌ ROLLBACK FAILED
|
||
==========================================
|
||
Error: ${e.message}
|
||
Manual intervention required!
|
||
==========================================
|
||
"""
|
||
throw e
|
||
}
|
||
}
|
||
|
||
// 构建服务（带重试）
|
||
def buildServiceWithRetry(String service) {
|
||
retry(2) {
|
||
timeout(time: env.BUILD_TIMEOUT.toInteger(), unit: 'MINUTES') {
|
||
buildService(service)
|
||
}
|
||
}
|
||
}
|
||
|
||
// 构建单个服务
|
||
def buildService(String service) {
|
||
def modulePath = getModulePathForService(service)
|
||
def buildStartTime = System.currentTimeMillis()
|
||
|
||
echo ""
|
||
echo "=========================================="
|
||
echo "🔨 Building ${service}"
|
||
echo "=========================================="
|
||
echo "Module: ${modulePath}"
|
||
echo "Registry: ${env.REGISTRY}"
|
||
echo "Tag: ${env.IMAGE_TAG}"
|
||
echo "=========================================="
|
||
|
||
try {
|
||
// 【优化1】使用 Maven 缓存卷
|
||
sh """
|
||
set -e
|
||
set -x
|
||
|
||
# 构建镜像（使用 Maven 缓存）
|
||
docker build \\
|
||
-f docker/Dockerfile.service \\
|
||
--build-arg DEPS_IMAGE=${env.DEPS_IMAGE} \\
|
||
--build-arg MODULE_NAME=${modulePath} \\
|
||
--build-arg JAR_NAME=${service} \\
|
||
--build-arg SKIP_TESTS=true \\
|
||
--build-arg MAVEN_OPTS="${env.MAVEN_OPTS}" \\
|
||
-t ${env.REGISTRY}/${service}:${env.IMAGE_TAG} \\
|
||
-t ${env.REGISTRY}/${service}:latest \\
|
||
.
|
||
|
||
# 推送镜像
|
||
docker push ${env.REGISTRY}/${service}:${env.IMAGE_TAG}
|
||
docker push ${env.REGISTRY}/${service}:latest
|
||
|
||
set +x
|
||
"""
|
||
|
||
def buildDuration = System.currentTimeMillis() - buildStartTime
|
||
echo "✅ ${service} built and pushed successfully in ${formatDuration(buildDuration)}"
|
||
|
||
// 获取镜像大小
|
||
def imageSize = sh(
|
||
script: "docker images ${env.REGISTRY}/${service}:latest --format '{{.Size}}'",
|
||
returnStdout: true
|
||
).trim()
|
||
|
||
echo "📊 Image size: ${imageSize}"
|
||
|
||
} catch (Exception e) {
|
||
echo "❌ Failed to build ${service}: ${e.message}"
|
||
|
||
// 打印构建日志以便调试
|
||
sh """
|
||
echo "=== Docker Build Logs for ${service} ==="
|
||
docker logs ${service}-builder 2>/dev/null || true
|
||
"""
|
||
|
||
throw e
|
||
}
|
||
}
|
||
|
||
// 部署服务（带超时）
|
||
def deployServiceWithTimeout(String service) {
|
||
timeout(time: env.DEPLOY_TIMEOUT.toInteger(), unit: 'MINUTES') {
|
||
deployService(service)
|
||
}
|
||
}
|
||
|
||
// 部署单个服务
|
||
def deployService(String service) {
|
||
def containerName = getContainerNameForService(service)
|
||
def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"
|
||
|
||
echo ""
|
||
echo "=========================================="
|
||
echo "🚀 Deploying ${service}"
|
||
echo "=========================================="
|
||
echo "Container: ${containerName}"
|
||
echo "Host: ${env.DEPLOY_HOST}"
|
||
echo "=========================================="
|
||
|
||
try {
|
||
// 部署服务
|
||
sh """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
set -e
|
||
cd ${env.DEPLOY_PATH}
|
||
|
||
echo "📥 Pulling ${service}..."
|
||
docker compose -f docker-compose.core.yml pull ${service}
|
||
|
||
echo "🔄 Restarting ${service}..."
|
||
docker compose -f docker-compose.core.yml up -d ${service}
|
||
|
||
echo "⏳ Waiting for container to start..."
|
||
sleep 5
|
||
'
|
||
"""
|
||
|
||
// 等待服务健康
|
||
waitForServiceHealthy(containerName, service, sshOpts)
|
||
|
||
echo "✅ ${service} deployed successfully"
|
||
|
||
} catch (Exception e) {
|
||
echo "❌ Failed to deploy ${service}: ${e.message}"
|
||
|
||
// 收集诊断信息
|
||
sh """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
echo "=== Container Status ==="
|
||
docker ps -a | grep ${containerName} || true
|
||
|
||
echo ""
|
||
echo "=== Container Logs (last 50 lines) ==="
|
||
docker logs --tail 50 ${containerName} 2>/dev/null || true
|
||
|
||
echo ""
|
||
echo "=== Service Health Status ==="
|
||
docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found"
|
||
'
|
||
"""
|
||
|
||
throw e
|
||
}
|
||
}
|
||
|
||
// 等待服务健康
|
||
def waitForServiceHealthy(String containerName, String serviceName, String sshOpts) {
|
||
// iot-gateway 是轻量化网关，不需要健康检查，只检查容器是否运行
|
||
if (serviceName == 'viewsh-module-iot-gateway') {
|
||
echo "⏳ Waiting for ${serviceName} to start (lightweight gateway, no health check)..."
|
||
sh """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
set -e
|
||
sleep 5
|
||
if docker ps | grep -q ${containerName}; then
|
||
echo "✅ ${serviceName} is running"
|
||
else
|
||
echo "❌ ${serviceName} is not running"
|
||
docker logs --tail 50 ${containerName} 2>/dev/null || true
|
||
exit 1
|
||
fi
|
||
'
|
||
"""
|
||
return
|
||
}
|
||
|
||
def maxAttempts = env.HEALTH_CHECK_TIMEOUT.toInteger() / env.HEALTH_CHECK_INTERVAL.toInteger()
|
||
|
||
echo "⏳ Waiting for ${serviceName} to be healthy (max ${env.HEALTH_CHECK_TIMEOUT}s)..."
|
||
|
||
sh """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
set -e
|
||
|
||
for i in \$(seq 1 ${maxAttempts}); do
|
||
STATUS=\$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "starting")
|
||
|
||
case "\$STATUS" in
|
||
healthy)
|
||
echo "✅ ${serviceName} is healthy"
|
||
exit 0
|
||
;;
|
||
unhealthy)
|
||
echo "❌ ${serviceName} is unhealthy"
|
||
echo "=== Last 100 lines of logs ==="
|
||
docker logs --tail 100 ${containerName}
|
||
exit 1
|
||
;;
|
||
starting)
|
||
ELAPSED=\$((i * ${env.HEALTH_CHECK_INTERVAL}))
|
||
echo "⏳ ${serviceName} is starting... (\${ELAPSED}s/${env.HEALTH_CHECK_TIMEOUT}s)"
|
||
;;
|
||
*)
|
||
echo "⚠️ ${serviceName} status: \$STATUS"
|
||
;;
|
||
esac
|
||
|
||
sleep ${env.HEALTH_CHECK_INTERVAL}
|
||
done
|
||
|
||
echo "❌ ${serviceName} health check timeout after ${env.HEALTH_CHECK_TIMEOUT}s"
|
||
echo "=== Full logs ==="
|
||
docker logs ${containerName}
|
||
exit 1
|
||
'
|
||
"""
|
||
}
|
||
|
||
// 检查服务健康（带重试）
|
||
def checkServiceHealthWithRetry(String service) {
|
||
def containerName = getContainerNameForService(service)
|
||
def sshOpts = "-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${env.SSH_KEY}"
|
||
|
||
retry(3) {
|
||
timeout(time: 2, unit: 'MINUTES') {
|
||
checkServiceHealth(containerName, service, sshOpts)
|
||
}
|
||
}
|
||
}
|
||
|
||
// 检查服务健康状态
|
||
def checkServiceHealth(String containerName, String serviceName, String sshOpts) {
|
||
// iot-gateway 是轻量化网关，不需要健康检查，只检查容器是否运行
|
||
if (serviceName == 'viewsh-module-iot-gateway') {
|
||
sh """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
if docker ps | grep -q ${containerName}; then
|
||
echo "✅ ${serviceName} is running (lightweight gateway)"
|
||
else
|
||
echo "❌ ${serviceName} is not running"
|
||
docker logs --tail 50 ${containerName} 2>/dev/null || true
|
||
exit 1
|
||
fi
|
||
'
|
||
"""
|
||
return
|
||
}
|
||
|
||
sh """
|
||
ssh ${sshOpts} root@${env.DEPLOY_HOST} '
|
||
STATUS=\$(docker inspect --format="{{.State.Health.Status}}" ${containerName} 2>/dev/null || echo "not_found")
|
||
|
||
case "\$STATUS" in
|
||
healthy)
|
||
echo "✅ ${serviceName} is healthy"
|
||
;;
|
||
not_found)
|
||
echo "⚠️ ${serviceName} not found (may not be deployed)"
|
||
exit 1
|
||
;;
|
||
*)
|
||
echo "❌ ${serviceName} is \$STATUS"
|
||
echo "=== Last 50 lines of logs ==="
|
||
docker logs --tail 50 ${containerName}
|
||
exit 1
|
||
;;
|
||
esac
|
||
'
|
||
"""
|
||
}
|
||
|
||
// 按依赖顺序排序服务
|
||
@NonCPS
|
||
def sortServicesByDependency(def services) {
|
||
def deployOrder = [
|
||
'viewsh-gateway',
|
||
'viewsh-module-system-server',
|
||
'viewsh-module-infra-server',
|
||
'viewsh-module-iot-server',
|
||
'viewsh-module-iot-gateway',
|
||
'viewsh-module-ops-server'
|
||
]
|
||
|
||
return services.sort { a, b ->
|
||
deployOrder.indexOf(a) <=> deployOrder.indexOf(b)
|
||
}
|
||
}
|
||
|
||
// 获取服务对应的容器名称
|
||
@NonCPS
|
||
def getContainerNameForService(String service) {
|
||
def map = [
|
||
'viewsh-gateway': 'aiot-gateway',
|
||
'viewsh-module-system-server': 'aiot-system-server',
|
||
'viewsh-module-infra-server': 'aiot-infra-server',
|
||
'viewsh-module-iot-server': 'aiot-iot-server',
|
||
'viewsh-module-iot-gateway': 'aiot-iot-gateway',
|
||
'viewsh-module-ops-server': 'aiot-ops-server'
|
||
]
|
||
return map.get(service, "aiot-${service}")
|
||
}
|
||
|
||
// 获取服务对应的模块路径
|
||
@NonCPS
|
||
def getModulePathForService(String service) {
|
||
def map = [
|
||
'viewsh-gateway': 'viewsh-gateway',
|
||
'viewsh-module-system-server': 'viewsh-module-system/viewsh-module-system-server',
|
||
'viewsh-module-infra-server': 'viewsh-module-infra/viewsh-module-infra-server',
|
||
'viewsh-module-iot-server': 'viewsh-module-iot/viewsh-module-iot-server',
|
||
'viewsh-module-iot-gateway': 'viewsh-module-iot/viewsh-module-iot-gateway',
|
||
'viewsh-module-ops-server': 'viewsh-module-ops/viewsh-module-ops-server'
|
||
]
|
||
return map.get(service, service)
|
||
}
|