本文目录导读：
- Kubernetes + Jenkins 实现金丝雀发布
- Nginx + Jenkins 实现金丝雀发布
- 金丝雀发布的自动化框架
- 最佳实践建议
- 完整示例：Kubernetes + Jenkins 金丝雀发布
- 关键要点
在Jenkins中实现金丝雀发布（Canary Release）策略，通常需要结合流水线（Pipeline）、负载均衡（如Nginx、Kubernetes）以及监控/回滚机制，以下是几种常见的实现方式：
Kubernetes + Jenkins 实现金丝雀发布
使用原生Kubernetes Deployment + Service
// Jenkinsfile
pipeline {
agent any
environment {
DOCKER_REGISTRY = 'your-registry.com'
APP_NAME = 'my-app'
NAMESPACE = 'production'
}
stages {
stage('Build & Push') {
steps {
script {
docker.build("${DOCKER_REGISTRY}/${APP_NAME}:${BUILD_NUMBER}")
docker.push("${DOCKER_REGISTRY}/${APP_NAME}:${BUILD_NUMBER}")
}
}
}
stage('Canary Deploy') {
steps {
script {
// 1. 部署金丝雀版本（20%流量）
sh """
kubectl set image deployment/${APP_NAME}-stable \
${APP_NAME}=${DOCKER_REGISTRY}/${APP_NAME}:${BUILD_NUMBER} \
-n ${NAMESPACE} --record
# 创建金丝雀deployment（副本数为stable的20%）
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
name: ${APP_NAME}-canary
namespace: ${NAMESPACE}
spec:
replicas: 2
selector:
matchLabels:
app: ${APP_NAME}
track: canary
template:
metadata:
labels:
app: ${APP_NAME}
track: canary
spec:
containers:
- name: ${APP_NAME}
image: ${DOCKER_REGISTRY}/${APP_NAME}:${BUILD_NUMBER}
EOF
"""
}
}
}
stage('Monitor & Promote') {
steps {
script {
// 2. 等待金丝雀版本正常运行
sh """
kubectl rollout status deployment/${APP_NAME}-canary \
-n ${NAMESPACE} --timeout=5m
"""
// 3. 验证金丝雀健康状态（可集成监控指标）
def healthy = checkCanaryHealth() // 自定义函数
if (healthy) {
echo "金丝雀测试通过，开始全量发布"
// 4. 更新稳定版本到最新
sh """
kubectl set image deployment/${APP_NAME}-stable \
${APP_NAME}=${DOCKER_REGISTRY}/${APP_NAME}:${BUILD_NUMBER} \
-n ${NAMESPACE}
kubectl rollout status deployment/${APP_NAME}-stable \
-n ${NAMESPACE} --timeout=10m
"""
// 5. 删除金丝雀版本
sh "kubectl delete deployment ${APP_NAME}-canary -n ${NAMESPACE}"
} else {
error "金丝雀测试失败，触发回滚"
}
}
}
}
}
post {
failure {
// 回滚金丝雀
sh "kubectl delete deployment ${APP_NAME}-canary -n ${NAMESPACE} --ignore-not-found"
emailext subject: "金丝雀发布失败",
body: "请检查并回滚稳定版本到上一个版本",
to: "team@company.com"
}
}
}
使用Service Mesh（Istio）实现精细流量控制
// 使用Istio的VirtualService实现请求级别的金丝雀
stage('Canary with Istio') {
steps {
script {
// 1. 部署金丝雀版本
sh """
kubectl apply -f canary-deployment.yaml
"""
// 2. 配置路由规则（5%流量到金丝雀）
sh """
cat <<EOF | kubectl apply -f -
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: ${APP_NAME}
namespace: ${NAMESPACE}
spec:
hosts:
- ${APP_NAME}
http:
- route:
- destination:
host: ${APP_NAME}
subset: stable
weight: 95
- destination:
host: ${APP_NAME}
subset: canary
weight: 5
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
name: ${APP_NAME}
namespace: ${NAMESPACE}
spec:
host: ${APP_NAME}
subsets:
- name: stable
labels:
version: stable
- name: canary
labels:
version: canary
EOF
"""
// 3. 监控指标（错误率、延迟）
def metrics = getIstioMetrics() // 通过Prometheus查询
if (metrics.errorRate < 0.01 && metrics.latency.p99 < 200) {
// 逐步增加流量比例
for (int weight = 10; weight <= 100; weight += 20) {
updateCanaryWeight(weight)
sleep(60) // 观察一段时间
metrics = getIstioMetrics()
if (!isHealthy(metrics)) {
rollbackCanary()
}
}
}
}
}
}
Nginx + Jenkins 实现金丝雀发布
使用Nginx权重分流
stage('Canary with Nginx') {
steps {
script {
// 1. 部署金丝雀版本到新服务器组
sh """
ansible-playbook -i canary.hosts deploy-app.yml \
-e "version=${BUILD_NUMBER}"
"""
// 2. 更新Nginx配置（5%流量到金丝雀）
def nginxConfig = """
upstream app_stable {
server stable1.internal:8080 weight=95;
server stable2.internal:8080 weight=95;
}
upstream app_canary {
server canary.internal:8080 weight=5;
}
server {
listen 80;
location / {
# 基于Cookie或Header分流
if (\$http_x_canary = "true") {
proxy_pass http://app_canary;
break;
}
proxy_pass http://app_stable;
}
}
"""
// 更合理的方式：使用split_clients模块
sh """
cat > /etc/nginx/conf.d/canary.conf << 'EOF'
split_clients "\${remote_addr}\${http_user_agent}" \$app_backend {
5% app_canary;
* app_stable;
}
server {
listen 80;
location / {
proxy_pass http://\$app_backend;
}
}
EOF
nginx -s reload
"""
// 3. 监控并逐步增加流量
def success = monitorAndScale()
if (success) {
// 100%流量到新版本
updateNginxWeight(100)
}
}
}
}
金丝雀发布的自动化框架
创建一个可复用的Jenkins库
// vars/canaryDeploy.groovy
def call(Map config) {
def namespace = config.namespace ?: 'default'
def appName = config.appName
def imageTag = config.imageTag
def canaryWeight = config.canaryWeight ?: 20
pipeline {
agent any
parameters {
string(name: 'CANARY_WEIGHT', defaultValue: "${canaryWeight}",
description: '金丝雀流量比例')
choice(name: 'ACTION', choices: ['deploy', 'promote', 'rollback'],
description: '操作类型')
}
stages {
stage('Deploy Canary') {
when { expression { params.ACTION == 'deploy' } }
steps {
deployCanaryVersion(appName, imageTag, namespace)
}
}
stage('Monitor Health') {
steps {
script {
def timeoutMinutes = config.monitorTimeout ?: 10
timeout(time: timeoutMinutes, unit: 'MINUTES') {
waitForHealthCheck(namespace, appName)
}
}
}
}
stage('Promote or Rollback') {
steps {
script {
if (isCanaryHealthy()) {
promoteCanary(namespace, appName)
} else {
rollbackCanary(namespace, appName)
}
}
}
}
}
}
}
最佳实践建议
安全措施
- 渐进式流量调整：从1%开始，逐步增加到5%、20%、50%、100%
- 自动回滚阈值：设置错误率>1%或延迟增加>20%时自动回滚
- 时间段限制：金丝雀发布仅在业务低峰期执行
监控指标
def getHealthMetrics(namespace, appName) {
// 使用Prometheus查询
def query = """
rate(request_count{namespace='${namespace}', app='${appName}',
status=~'5..'}[5m]) /
rate(request_count{namespace='${namespace}', app='${appName}'}[5m])
"""
def result = prometheusQuery(query)
return [
errorRate: result.data.result[0].value[1],
latencyP99: getLatencyMetric(namespace, appName, 0.99)
]
}
def isHealthy(metrics) {
return metrics.errorRate < 0.01 && metrics.latencyP99 < 500
}
通知与审批
stage('Approval Gate') {
steps {
script {
// 关键流量比例需要人工审批
if (currentWeight > 50) {
input message: "是否继续增加到 ${nextWeight}%?",
ok: '继续',
submitterParameter: 'APPROVER',
parameters: [text(name: 'COMMENTS', description: '备注')]
}
}
}
}
完整示例：Kubernetes + Jenkins 金丝雀发布
pipeline {
agent any
parameters {
string(name: 'VERSION', defaultValue: 'latest', description: '应用版本')
choice(name: 'CANARY_RATIO', choices: ['5%', '10%', '20%', '50%'],
description: '金丝雀流量比例')
booleanParam(name: 'AUTO_PROMOTE', defaultValue: true,
description: '成功后自动全量发布')
}
environment {
KUBECONFIG = credentials('kubeconfig')
REGISTRY = 'registry.example.com'
APP = 'my-web-app'
NS = 'production'
}
stages {
stage('验证版本') {
steps {
script {
sh "docker pull ${REGISTRY}/${APP}:${VERSION}"
}
}
}
stage('部署金丝雀') {
steps {
script {
def weight = params.CANARY_RATIO.replace('%', '')
sh """
kubectl apply -f k8s/canary/
kubectl set image deployment/${APP}-canary \
${APP}=${REGISTRY}/${APP}:${VERSION} \
-n ${NS}
cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ${APP}
annotations:
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-weight: "${weight}"
spec:
rules:
- host: app.example.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: ${APP}-canary
port:
number: 80
EOF
"""
}
}
}
stage('监控验证') {
parallel {
stage('错误率检查') {
steps {
script {
timeout(time: 30, unit: 'MINUTES') {
waitUntil {
def metrics = getErrorRate()
metrics < 1.0 // 错误率小于1%
}
}
}
}
}
stage('性能检查') {
steps {
script {
timeout(time: 30, unit: 'MINUTES') {
waitUntil {
def latency = getLatency()
latency < 200 // 延迟小于200ms
}
}
}
}
}
}
}
stage('全量发布') {
when {
expression {
params.AUTO_PROMOTE &&
!currentBuild.previousBuild?.result == 'FAILURE'
}
}
steps {
script {
sh """
# 更新稳定版本
kubectl set image deployment/${APP}-stable \
${APP}=${REGISTRY}/${APP}:${VERSION} -n ${NS}
# 移除金丝雀Ingress
kubectl delete ingress ${APP} -n ${NS}
# 清理金丝雀资源
kubectl delete deployment ${APP}-canary -n ${NS}
"""
}
}
}
}
}
关键要点
-
选择合适的分流机制：
- K8s Service：基于标签选择器
- Nginx：基于权重或Cookie
- Istio：支持请求头、权重等精细控制
-
自动化程度：
- 小比例（<10%）：全自动
- 中等比例（10-50%）：自动+告警
- 大比例（>50%）：需要人工审批
-
回滚策略：
- 自动回滚：监控指标异常时
- 手动回滚：保留金丝雀版本，流量切回稳定版
-
可观测性：
- 集成Prometheus/Grafana
- 自定义告警规则
- 日志聚合分析
这种实现方式可以确保发布的可靠性,同时将风险降到最低，根据具体的基础设施和业务需求，可以选择最适合的方案组合。
