云计算（七）：运维与 DevOps 实践

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

name: deploy
on:
  push:
    branches: [main]

permissions:
  id-token: write          # 使用 OIDC 实现云平台身份认证
  contents: read
  packages: write

env:
  REGISTRY: ghcr.io
  IMAGE:    ghcr.io/${{ github.repository }}

jobs:
  build-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: 设置 Go 环境
        uses: actions/setup-go@v5
        with: { go-version-file: go.mod }

      - name: 代码静态检查（Lint）
        uses: golangci/golangci-lint-action@v4

      - name: 单元测试
        run: go test -race -coverprofile=coverage.out ./...

      - name: 覆盖率门禁
        run: |
          pct=$(go tool cover -func=coverage.out | tail -1 | awk '{print $3}' | tr -d '%')
          echo "覆盖率：${pct}%"
          if (( $(echo "$pct < 80" | bc -l) )); then
            echo "::error::覆盖率 ${pct}% 低于 80% 门槛"
            exit 1
          fi          

      - name: 构建并推送镜像
        run: |
          echo "${{ secrets.GITHUB_TOKEN }}" | docker login $REGISTRY -u ${{ github.actor }} --password-stdin
          docker build -t $IMAGE:${{ github.sha }} -t $IMAGE:latest .
          docker push $IMAGE --all-tags          

  security-scan:
    needs: build-test
    runs-on: ubuntu-latest
    steps:
      - name: Trivy 镜像安全扫描
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.IMAGE }}:${{ github.sha }}
          severity: CRITICAL,HIGH
          exit-code: 1

      - name: SAST（源码审计）—— 使用 Semgrep
        uses: returntocorp/semgrep-action@v1
        with:
          config: p/owasp-top-ten

  deploy-staging:
    needs: security-scan
    runs-on: ubuntu-latest
    environment: staging
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::111122223333:role/deploy-staging
          aws-region: us-east-1

      - name: 部署至 ECS Staging 环境
        run: |
          aws ecs update-service --cluster staging --service web --force-new-deployment          

      - name: 等待服务稳定
        run: aws ecs wait services-stable --cluster staging --services web

  smoke-test:
    needs: deploy-staging
    runs-on: ubuntu-latest
    steps:
      - name: 健康检查
        run: |
          for i in {1..10}; do
            status=$(curl -s -o /dev/null -w '%{http_code}' https://staging.example.com/healthz)
            if [ "$status" = "200" ]; then echo "Healthy"; exit 0; fi
            sleep 5
          done
          echo "::error::Staging 环境健康检查在 50 秒后失败"
          exit 1          

      - name: API 合约测试
        run: |
          npm ci
          npx newman run tests/postman/smoke.json --environment tests/postman/staging.json          

  deploy-prod:
    needs: smoke-test
    runs-on: ubuntu-latest
    environment: production       # 需人工审批
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::111122223333:role/deploy-prod
          aws-region: us-east-1

      - name: 金丝雀发布（10% 流量）
        run: |
          aws ecs update-service --cluster prod --service web \
            --deployment-configuration '{
              "deploymentCircuitBreaker": {"enable": true, "rollback": true},
              "maximumPercent": 200,
              "minimumHealthyPercent": 100
            }' \
            --force-new-deployment          

      - name: 验证金丝雀 SLO（5 分钟）
        run: |
          sleep 300
          # 通过 CloudWatch 检查 error_rate；若超过 1%，自动回滚          

1
2
3
4

terraform init      # 下载 Provider 插件，配置远程后端
terraform plan      # 预览：即将创建 / 修改 / 销毁的资源清单
terraform apply     # 执行变更，使真实环境与代码保持一致
terraform destroy   # 彻底销毁全部资源（仅限非生产环境！）

1
2
3

terraform plan -out=tfplan -input=false
terraform show -no-color tfplan > plan.txt
terraform apply -input=false tfplan

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# backend.tf
terraform {
  backend "s3" {
    bucket         = "company-terraform-state"
    key            = "services/web/prod/terraform.tfstate"
    region         = "us-east-1"
    dynamodb_table = "terraform-locks"
    encrypt        = true
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

# modules/ecs-service/main.tf
variable "name"          { type = string }
variable "environment"   { type = string }
variable "image"         { type = string }
variable "cpu"           { type = number, default = 256 }
variable "memory"        { type = number, default = 512 }
variable "desired_count" { type = number, default = 2 }
variable "health_path"   { type = string, default = "/healthz" }

resource "aws_ecs_task_definition" "this" {
  family                   = "${var.name}-${var.environment}"
  network_mode             = "awsvpc"
  requires_compatibilities = ["FARGATE"]
  cpu                      = var.cpu
  memory                   = var.memory
  execution_role_arn       = aws_iam_role.execution.arn
  task_role_arn            = aws_iam_role.task.arn

  container_definitions = jsonencode([{
    name      = var.name
    image     = var.image
    essential = true
    portMappings = [{ containerPort = 8080, protocol = "tcp" }]
    healthCheck = {
      command     = ["CMD-SHELL", "curl -f http://localhost:8080${var.health_path} || exit 1"]
      interval    = 15
      timeout     = 5
      retries     = 3
      startPeriod = 60
    }
  }])
}

resource "aws_ecs_service" "this" {
  name            = "${var.name}-${var.environment}"
  cluster         = data.aws_ecs_cluster.main.id
  task_definition = aws_ecs_task_definition.this.arn
  desired_count   = var.desired_count
  launch_type     = "FARGATE"

  deployment_circuit_breaker {
    enable   = true
    rollback = true
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

#!/bin/bash
# drift-check.sh -- 通过 cron 或调度流水线每日执行
set -euo pipefail

SERVICES="web api worker"
ENVS="staging prod"

for svc in $SERVICES; do
  for env in $ENVS; do
    dir="services/${svc}/${env}"
    cd "$dir"
    terraform init -backend=true -input=false > /dev/null
    if ! terraform plan -detailed-exitcode -input=false > /dev/null 2>&1; then
      echo "DRIFT DETECTED in ${dir}"
      terraform plan -no-color -input=false > "/tmp/drift-${svc}-${env}.txt"
      curl -X POST "$SLACK_WEBHOOK" -d "{\"text\":\"Drift detected in ${dir}.\"}"
    fi
    cd - > /dev/null
  done
done

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alerts/*.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

scrape_configs:
  - job_name: "kubernetes-pods"
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # 仅拉取带有 annotation prometheus.io/scrape=true 的 Pod
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      # 将 annotation prometheus.io/path 映射为 metrics 路径
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      # 将 annotation prometheus.io/port 与 Pod IP 组合成地址：${pod_ip}:${port}
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_ip]
        action: replace
        target_label: __address__
        regex: (.+);(.+)
        replacement: $2:$1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

package main

import (
    "net/http"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    httpDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "http_request_duration_seconds",
            Help:    "Duration of HTTP requests.",
            Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
        },
        []string{"method", "route", "status"},
    )
    httpTotal = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total number of HTTP requests.",
        },
        []string{"method", "route", "status"},
    )
)

func init() {
    prometheus.MustRegister(httpDuration, httpTotal)
}

func instrumentHandler(route string, next http.HandlerFunc) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        rw := &responseWriter{ResponseWriter: w, statusCode: 200}
        next(rw, r)
        duration := time.Since(start).Seconds()
        status := http.StatusText(rw.statusCode)
        httpDuration.WithLabelValues(r.Method, route, status).Observe(duration)
        httpTotal.WithLabelValues(r.Method, route, status).Inc()
    }
}

func main() {
    http.Handle("/metrics", promhttp.Handler())
    http.HandleFunc("/api/orders", instrumentHandler("/api/orders", handleOrders))
    http.ListenAndServe(":8080", nil)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

# alerts/slo.yml
groups:
  - name: slo-burn-rate
    rules:
      # 多时间窗口、多燃烧速率告警（源自 Google SRE 手册的经典模式）
      # 快速燃烧：1 小时内错误预算燃烧速率达 14.4 倍，且 6 小时内达 6 倍
      - alert: HighErrorBurnRate_Critical
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[1h]))
            / sum(rate(http_requests_total[1h]))
          ) > (14.4 * 0.001)
          and
          (
            sum(rate(http_requests_total{status=~"5.."}[6h]))
            / sum(rate(http_requests_total[6h]))
          ) > (6 * 0.001)          
        for: 2m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "错误预算燃烧过快——预计 < 1 天耗尽"
          runbook: "https://wiki.internal/runbooks/high-error-rate"
          dashboard: "https://grafana.internal/d/slo-overview"

      # 缓慢燃烧：1 天内达 3 倍，且 3 天内达 1 倍
      - alert: HighErrorBurnRate_Warning
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[1d]))
            / sum(rate(http_requests_total[1d]))
          ) > (3 * 0.001)
          and
          (
            sum(rate(http_requests_total{status=~"5.."}[3d]))
            / sum(rate(http_requests_total[3d]))
          ) > (1 * 0.001)          
        for: 15m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "错误预算缓慢燃烧——预计 < 10 天耗尽"
          runbook: "https://wiki.internal/runbooks/high-error-rate"

      - alert: HighLatency_P99
        expr: |
          histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))
          > 2.0          
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.service }} 的 P99 延迟超过 2 秒"
          runbook: "https://wiki.internal/runbooks/high-latency"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

# alertmanager.yml
global:
  resolve_timeout: 5m

route:
  receiver: "default-slack"
  group_by: ["alertname", "service"]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

  routes:
    - match:
        severity: critical
      receiver: "pagerduty-critical"
      repeat_interval: 15m

    - match:
        severity: warning
      receiver: "slack-warnings"
      repeat_interval: 4h

receivers:
  - name: "pagerduty-critical"
    pagerduty_configs:
      - routing_key: "<pagerduty-integration-key>"
        severity: critical

  - name: "slack-warnings"
    slack_configs:
      - api_url: "https://hooks.slack.com/services/xxx"
        channel: "#alerts-warnings"
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'

  - name: "default-slack"
    slack_configs:
      - api_url: "https://hooks.slack.com/services/xxx"
        channel: "#alerts-default"

1
2
3
4
5
6
7
8
9

+-----------+     +------------+     +---------------+     +---------+
| 应用 Pod  | --> | Fluent Bit | --> | Elasticsearch | --> | Kibana  |
| (stdout)  |     | (DaemonSet)|     | (3 节点高可用) |     | (查询界面) |
+-----------+     +------------+     +---------------+     +---------+
                        |
                        v
                  +------------+
                  |   Kafka    |  （可选：用于应对日志洪峰的缓冲层）
                  +------------+

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

import structlog
import uuid

# 配置 structlog 输出 JSON 格式日志
structlog.configure(
    processors=[
        structlog.processors.TimeStamper(fmt="iso"),
        structlog.processors.add_log_level,
        structlog.processors.StackInfoRenderer(),
        structlog.processors.format_exc_info,
        structlog.processors.JSONRenderer(),
    ],
    wrapper_class=structlog.BoundLogger,
)

log = structlog.get_logger()

def handle_request(request):
    request_id = request.headers.get("X-Request-ID", str(uuid.uuid4()))
    logger = log.bind(
        request_id=request_id,
        method=request.method,
        path=request.path,
        user_id=request.user.id if request.user else None,
    )

    logger.info("request_started")
    try:
        result = process(request)
        logger.info("request_completed", status=200, duration_ms=result.duration_ms)
        return result
    except ValidationError as e:
        logger.warning("validation_failed", error=str(e), status=400)
        raise
    except Exception as e:
        logger.error("request_failed", error=str(e), status=500, exc_info=True)
        raise

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

{
  "timestamp": "2024-03-15T14:30:22.123Z",
  "level": "info",
  "event": "request_completed",
  "request_id": "abc-123",
  "method": "POST",
  "path": "/api/orders",
  "user_id": "u_456",
  "status": 200,
  "duration_ms": 42
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# fluent-bit.conf
[SERVICE]
    Flush         5
    Daemon        Off
    Log_Level     info
    Parsers_File  parsers.conf

[INPUT]
    Name              tail
    Path              /var/log/containers/*.log
    Parser            cri
    Tag               kube.*
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10

[FILTER]
    Name                kubernetes
    Match               kube.*
    Kube_URL            https://kubernetes.default.svc:443
    Kube_Tag_Prefix     kube.var.log.containers.
    Merge_Log           On
    Keep_Log            Off
    K8S-Logging.Parser  On

[FILTER]
    Name    modify
    Match   kube.*
    # 在写入索引前脱敏敏感字段
    Remove  password
    Remove  authorization
    Remove  cookie
    Remove  x-api-key

[OUTPUT]
    Name            es
    Match           kube.*
    Host            elasticsearch.logging.svc.cluster.local
    Port            9200
    Logstash_Format On
    Logstash_Prefix k8s-logs
    Retry_Limit     5
    tls             On
    tls.verify      On

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": { "max_size": "50gb", "max_age": "1d" },
          "set_priority": { "priority": 100 }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "shrink": { "number_of_shards": 1 },
          "forcemerge": { "max_num_segments": 1 },
          "set_priority": { "priority": 50 }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "searchable_snapshot": { "snapshot_repository": "s3-logs" },
          "set_priority": { "priority": 0 }
        }
      },
      "delete": { "min_age": "365d", "actions": { "delete": {} } }
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: web-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: web
  minReplicas: 3
  maxReplicas: 50
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60    # 扩容前等待 1 分钟确认趋势
      policies:
        - type: Percent
          value: 100                     # 单次最多扩容 100%（即翻倍）
          periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300   # 缩容前等待 5 分钟冷静期
      policies:
        - type: Percent
          value: 10                      # 每轮最多缩容 10%
          periodSeconds: 60
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
    - type: Pods
      pods:
        metric:
          name: http_requests_per_second
        target:
          type: AverageValue
          averageValue: "1000"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# AWS Auto Scaling 预测策略配置示例
aws autoscaling put-scaling-policy \
  --auto-scaling-group-name web-asg \
  --policy-name predictive-scaling \
  --policy-type PredictiveScaling \
  --predictive-scaling-configuration '{
    "MetricSpecifications": [{
      "TargetValue": 70,
      "PredefinedMetricPairSpecification": {
        "PredefinedMetricType": "ASGCPUUtilization"
      }
    }],
    "Mode": "ForecastAndScale",
    "SchedulingBufferTime": 300
  }'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# shutdown-nonprod.sh —— cron: 0 19 * * 1-5（工作日每晚 7 点）
#!/bin/bash
set -euo pipefail

ENVS=("dev" "staging" "qa")

for env in "${ENVS[@]}"; do
  echo "正在关闭 ${env} 环境的 ECS 服务..."
  for svc in $(aws ecs list-services --cluster "$env" --query 'serviceArns[]' --output text); do
    aws ecs update-service --cluster "$env" --service "$svc" --desired-count 0
  done

  echo "正在停止 ${env} 环境的 RDS 实例..."
  for db in $(aws rds describe-db-instances --query "DBInstances[?TagList[?Key=='Environment'&&Value=='${env}']].DBInstanceIdentifier" --output text); do
    aws rds stop-db-instance --db-instance-identifier "$db" || true
  done
done

echo "非生产环境已关闭，时间：$(date)"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# startup-nonprod.sh —— cron: 0 8 * * 1-5（工作日每天早 8 点）
#!/bin/bash
set -euo pipefail

ENVS=("dev" "staging" "qa")

for env in "${ENVS[@]}"; do
  echo "正在启动 ${env} 环境的 RDS 实例..."
  for db in $(aws rds describe-db-instances --query "DBInstances[?TagList[?Key=='Environment'&&Value=='${env}']].DBInstanceIdentifier" --output text); do
    aws rds start-db-instance --db-instance-identifier "$db" || true
  done

  echo "正在恢复 ${env} 环境的 ECS 服务..."
  for svc in $(aws ecs list-services --cluster "$env" --query 'serviceArns[]' --output text); do
    aws ecs update-service --cluster "$env" --service "$svc" --desired-count 2
  done
done

echo "非生产环境已启动，时间：$(date)"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

{
  "Version": "2012-10-17",
  "Statement": [{
    "Sid": "RequireTags",
    "Effect": "Deny",
    "Action": ["ec2:RunInstances", "rds:CreateDBInstance", "s3:CreateBucket"],
    "Resource": "*",
    "Condition": {
      "Null": {
        "aws:RequestTag/Environment": "true",
        "aws:RequestTag/Team": "true",
        "aws:RequestTag/Service": "true"
      }
    }
  }]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# SLO 定义（以 Prometheus 实践为例）
# 99.9% 可用性 = 0.1% 错误预算 = 每月 43.2 分钟

# 可用性 SLI
- record: sli:availability:ratio_rate5m
  expr: |
    sum(rate(http_requests_total{status!~"5.."}[5m]))
    / sum(rate(http_requests_total[5m]))    

# 延迟 SLI（响应时间 < 300ms 的请求占比）
- record: sli:latency:ratio_rate5m
  expr: |
    sum(rate(http_request_duration_seconds_bucket{le="0.3"}[5m]))
    / sum(rate(http_request_duration_seconds_count[5m]))    

# 剩余错误预算（30 天窗口）
- record: error_budget:remaining
  expr: |
    1 - (
      (1 - avg_over_time(sli:availability:ratio_rate5m[30d]))
      / (1 - 0.999)
    )    

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

## 事故报告：[标题]
**日期**：YYYY-MM-DD  
**持续时间**：X 小时 Y 分钟  
**严重等级**：SEV-1 / SEV-2 / SEV-3  
**撰写人**：[姓名]  
**评审人**：[姓名列表]

### 概述  
一句话说明发生了什么；一句话说明对用户的影响；一句话说明如何恢复。例如：数据库连接池耗尽导致核心 API 错误率飙升，影响约 5% 的用户请求，持续 25 分钟；通过扩容连接池并重启服务完成修复。

### 时间线（所有时间均为 UTC）
- 14:00 — 监控告警触发：错误率 > 5%  
- 14:03 — 值班工程师响应并启动排查  
- 14:15 — 定位根因：数据库连接池耗尽  
- 14:20 — 实施缓解措施：扩大连接池容量、重启服务  
- 14:25 — 错误率回落至基线水平  
- 14:30 — 正式宣告事故结束  

### 根因分析  
连接池初始配置为 50 个连接。两天前上线的新批处理任务，在执行期间平均持有连接达 30 秒（远超预期的 100ms），在业务高峰时段迅速耗尽全部连接。

### 影响范围  
- 高错误率持续 25 分钟（峰值达 12%）  
- 约 3,200 次请求失败  
- 无数据丢失  

### 做得好的地方  
- 监控告警在问题发生后 3 分钟内准确触发  
- 值班工程师具备完整权限，可独立完成诊断与处置  

### 待改进之处  
- 新批处理任务未针对共享数据库开展负载测试  
- 连接池缺乏熔断机制（circuit breaker）  

### 改进项（Action Items）  
| 改进项 | 负责人 | 优先级 | 截止日期 |  
|--------|--------|--------|----------|  
| 在 Grafana 仪表盘中增加连接池关键指标（如活跃连接数、等待队列长度） | Alice | P1 | 2024-02-01 |  
| 对所有批处理任务开展共享资源（尤其是数据库）的压测验证 | Bob | P1 | 2024-02-15 |  
| 为数据库连接池实现熔断机制（例如 Hystrix 或 Resilience4j 集成） | Carol | P2 | 2024-03-01 |  
| 编写《连接池耗尽》标准化应急手册（Runbook），纳入内部知识库 | Dave | P2 | 2024-02-01 |  

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata: { name: web, namespace: argocd }
spec:
  project: default
  source:
    repoURL:        https://github.com/company/k8s-config
    targetRevision: main
    path:           apps/web/overlays/production
  destination:
    server:    https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated:
      prune:    true       # 删掉 git 中移除的资源
      selfHeal: true       # 自动修复手动改动
    retry:
      limit: 5
      backoff: { duration: 30s, maxDuration: 5m, factor: 2 }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

k8s-config/
  apps/
    web/
      base/
        deployment.yaml
        service.yaml
        hpa.yaml
        kustomization.yaml
      overlays/
        staging/
          kustomization.yaml      # 补丁：副本数、资源限制
          ingress-patch.yaml      # 预发布环境域名
        production/
          kustomization.yaml      # 补丁：更高副本数、更多资源
          ingress-patch.yaml      # 生产环境域名
    api/
      base/
        ...
      overlays/
        ...
  platform/
    monitoring/
      prometheus/
      grafana/
    logging/
      fluent-bit/
      elasticsearch/

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# 1. 具体现象是什么？先看仪表盘。
open https://grafana.internal/d/overview

# 2. 是单个服务异常，还是全局性故障？
kubectl get pods -A | grep -v Running
kubectl top nodes
kubectl top pods --sort-by=cpu -A | head -20

# 3. 最近有哪些变更？
kubectl rollout history deployment/web -n production
git log --oneline --since="2 hours ago" -- apps/web/

# 4. 检查资源压力
kubectl describe nodes | grep -A5 "Conditions:"
df -h                           # 磁盘使用率
free -m                         # 内存使用情况
ss -tlnp                        # 监听端口与连接数

# 5. 查看受影响服务的近期日志
kubectl logs -l app=web -n production --tail=100 --since=5m

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# PostgreSQL：当前正在执行的慢查询（>5 秒）
SELECT pid, now() - pg_stat_activity.query_start AS duration,
       query, state, wait_event_type, wait_event
FROM pg_stat_activity
WHERE state != 'idle'
  AND (now() - pg_stat_activity.query_start) > interval '5 seconds'
ORDER BY duration DESC;

# PostgreSQL：按连接状态统计连接数
SELECT state, count(*)
FROM pg_stat_activity
GROUP BY state;

# PostgreSQL：表膨胀检查（死元组超 1 万）
SELECT schemaname, tablename,
       pg_size_pretty(pg_total_relation_size(schemaname || '.' || tablename)) as total_size,
       n_dead_tup,
       n_live_tup,
       ROUND(n_dead_tup * 100.0 / NULLIF(n_live_tup + n_dead_tup, 0), 2) as dead_pct
FROM pg_stat_user_tables
WHERE n_dead_tup > 10000
ORDER BY n_dead_tup DESC;

# MySQL：当前活跃查询
SHOW FULL PROCESSLIST;

# MySQL：InnoDB 引擎状态（锁、死锁、缓冲池）
SHOW ENGINE INNODB STATUS\G

# Redis：内存与键空间统计
redis-cli INFO memory
redis-cli INFO keyspace
redis-cli --bigkeys          # 定位大 Key
redis-cli SLOWLOG GET 10     # 获取最近 10 条慢命令