build-flow.api/Jenkinsfile
xiangning caf3c17fce fix(cicd): push_retry 加 timeout 60s 包裹,自动处理跨地域 push 死锁
问题:跨地域推 Harbor 时 docker push 推大 layer TCP 死锁,
命令 hang 住不退出,push_retry 永远等不到重试触发。

修复:
- 每次 docker push 用 timeout 60 包裹,超 60s 强制杀掉 CLI(断点续传)
- 重试次数 10→15,给更多断点续传机会
- 被 timeout 杀掉后返回非零,自动进入下一次重试循环
2026-07-17 15:42:36 +08:00

125 lines
5.6 KiB
Groovy
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// BuildFlow buildflow-api CI/CD
// 链路:①拉取代码 → ②编译打包(docker build) → 推送 Harbor → ③部署(cd-target docker compose) → HealthCheck
// 自动触发:Gitea push → Generic Webhook Trigger(token=buildflow-api)
// Gitea webhook 指向:http://jenkins:8080/generic-webhook-trigger/invoke?token=buildflow-api
// 部署:docker compose;api 生产密钥通过 volume 挂载 appsettings.Secrets.json 注入(不进镜像)
// api 运行依赖:PostgreSQL(5库) / Redis / MongoDB(im_system) / RabbitMQ / S3(RustFS) / 本地LLM(1234) / KKFileView(8012)
pipeline {
agent any
triggers {
// Gitea push 触发:Generic Webhook Trigger 按 token 路由到本 job
// token 通过 URL ?token=xxx 传入,与 Gitea webhook URL 的 token 参数匹配
GenericTrigger(
token: 'buildflow-api',
causeString: 'Gitea push 触发',
printContributedVariables: false,
printPostContent: false,
regexpFilterText: '',
regexpFilterExpression: ''
)
}
environment {
HARBOR = "43.134.235.58:58080"
IMAGE = "${HARBOR}/buildflow/buildflow-api"
TAG = "${env.BUILD_NUMBER}"
// Jenkins 容器内无 ssh 别名,直接用 IP;ubuntu 用户 sudo 免密(NOPASSWD)
DEPLOY_HOST = "ubuntu@110.42.240.10"
APP_PORT = "5667"
// cd-target 上 compose 编排目录 + 本服务名
DEPLOY_DIR = "/home/ubuntu/deploy"
SERVICE = "buildflow-api"
}
options {
timeout(time: 30, unit: "MINUTES")
buildDiscarder(logRotator(numToKeepStr: "10"))
}
stages {
stage("① 拉取代码") {
steps {
checkout scm
sh "git log --oneline -3"
}
}
stage("② 编译打包") {
steps {
sh "docker build -t ${IMAGE}:${TAG} -t ${IMAGE}:latest ."
sh "docker images | grep ${IMAGE} | head -2"
}
}
stage("推送 Harbor") {
steps {
withCredentials([usernamePassword(credentialsId: "harbor-credentials", usernameVariable: "HUSER", passwordVariable: "HPASS")]) {
sh '''
# 跨地域 push 大 blob 易断流 + Harbor auth session 会过期 + 网络死锁
# 每次 push 前重新 login,用 timeout 包裹防 hang 死,断点续传(已传层跳过)
push_retry() {
local img=$1
for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
echo $HPASS | docker login $HARBOR -u $HUSER --password-stdin >/dev/null 2>&1
# timeout 60s:单次 push 超过 60s 强制杀掉(防 TCP 死锁 hang 住)
# 被杀后返回非零 → 重试;断点续传已传 layer 跳过,每次推进一部分
timeout 60 docker push "$img" && return 0
echo "push $img 第 $i 次中断/超时,5秒后重试(重新登录+断点续传)..."
sleep 5
done
return 1
}
push_retry $IMAGE:$TAG
push_retry $IMAGE:latest
docker logout $HARBOR
'''
}
}
}
stage("③ 部署") {
steps {
withCredentials([usernamePassword(credentialsId: "harbor-credentials", usernameVariable: "HUSER", passwordVariable: "HPASS")]) {
sh """
ssh -o StrictHostKeyChecking=accept-new ${DEPLOY_HOST} "
cd ${DEPLOY_DIR}
echo '${HPASS}' | sudo docker login ${HARBOR} -u ${HUSER} --password-stdin
sudo docker compose pull ${SERVICE}
sudo docker compose up -d --no-deps ${SERVICE}
"
"""
}
echo "部署完成"
}
}
stage("HealthCheck") {
steps {
script {
// .NET 启动较慢(连 PG/Redis/Mongo),轮询最多 60s
def healthy = false
for (int i = 1; i <= 12; i++) {
sleep 5
try {
sh """
ssh -o StrictHostKeyChecking=accept-new ${DEPLOY_HOST} 'wget -q --spider http://127.0.0.1:${APP_PORT}/health && echo HEALTHCHECK_OK'
"""
healthy = true
echo "=== HealthCheck OK: api http://110.42.240.10:${APP_PORT} ==="
break
} catch (e) {
echo "HealthCheck 第 ${i}/12 次探测未就绪,5s 后重试..."
}
}
if (!healthy) {
// 不再静默吞失败:标记 unstable,流水线变黄便于发现
unstable("HealthCheck 60s 内未通过,查日志:ssh cd-target 'sudo docker logs ${SERVICE} --tail 50'")
}
}
}
}
}
post {
success { echo '=== buildflow-api CI/CD SUCCESS ===' }
failure { echo '=== buildflow-api CI/CD FAILED ===' }
always {
sh "docker rmi ${IMAGE}:${TAG} ${IMAGE}:latest 2>/dev/null || true"
deleteDir()
}
}
}