运维与 DevOps 实践

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

name: deploy
on:
  push:
    branches: [main]

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.11", cache: "pip" }
      - run: pip install -r requirements.txt -r requirements-dev.txt
      - run: pytest --cov=app --cov-report=xml --cov-fail-under=80
      - uses: codecov/codecov-action@v4

  security:
    needs: test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: aquasecurity/trivy-action@master
        with:
          scan-type: fs
          severity: HIGH,CRITICAL
          exit-code: 1

  build:
    needs: [test, security]
    runs-on: ubuntu-latest
    permissions: { id-token: write, contents: read }
    steps:
      - uses: actions/checkout@v4
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::111122223333:role/github-deploy
          aws-region: us-east-1
      - uses: aws-actions/amazon-ecr-login@v2
      - run: |
          docker build -t $ECR/app:${{ github.sha }} .
          docker push $ECR/app:${{ github.sha }}          
        env: { ECR: 111122223333.dkr.ecr.us-east-1.amazonaws.com }

  deploy:
    needs: build
    runs-on: ubuntu-latest
    environment: production         # GitHub 人工审批门
    permissions: { id-token: write, contents: read }
    steps:
      - uses: actions/checkout@v4
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::111122223333:role/github-deploy
          aws-region: us-east-1
      - run: aws ecs update-service --cluster prod --service app
                                    --force-new-deployment
      - run: aws ecs wait services-stable --cluster prod --services app

  verify:
    needs: deploy
    runs-on: ubuntu-latest
    steps:
      - run: ./scripts/check_slo.sh    # 失败时通过重新部署回滚

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

# modules/web-service/main.tf
terraform {
  required_version = ">= 1.5"
  required_providers {
    aws = { source = "hashicorp/aws", version = "~> 5.0" }
  }
}

variable "name"           { type = string }
variable "environment"    { type = string }
variable "instance_type"  { type = string, default = "t3.medium" }
variable "min_size"       { type = number, default = 2 }
variable "max_size"       { type = number, default = 10 }
variable "subnet_ids"     { type = list(string) }
variable "vpc_id"         { type = string }

locals {
  full_name = "${var.environment}-${var.name}"
  tags = {
    Environment = var.environment
    Service     = var.name
    ManagedBy   = "terraform"
  }
}

resource "aws_security_group" "web" {
  name_prefix = "${local.full_name}-"
  vpc_id      = var.vpc_id
  ingress {
    from_port       = 443
    to_port         = 443
    protocol        = "tcp"
    security_groups = [aws_security_group.alb.id]
  }
  egress {
    from_port = 0; to_port = 0; protocol = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
  tags = local.tags
}

resource "aws_launch_template" "web" {
  name_prefix   = "${local.full_name}-"
  image_id      = data.aws_ami.al2023.id
  instance_type = var.instance_type
  vpc_security_group_ids = [aws_security_group.web.id]
  iam_instance_profile { name = aws_iam_instance_profile.web.name }
  metadata_options { http_tokens = "required" }     # 强制 IMDSv2
  tag_specifications {
    resource_type = "instance"
    tags          = local.tags
  }
}

resource "aws_autoscaling_group" "web" {
  name_prefix         = "${local.full_name}-"
  vpc_zone_identifier = var.subnet_ids
  min_size            = var.min_size
  max_size            = var.max_size
  desired_capacity    = var.min_size
  health_check_type   = "ELB"
  health_check_grace_period = 90
  launch_template { id = aws_launch_template.web.id, version = "$Latest" }
  target_group_arns = [aws_lb_target_group.web.arn]
  instance_refresh {
    strategy = "Rolling"
    preferences { min_healthy_percentage = 90 }
  }
  dynamic "tag" {
    for_each = local.tags
    content { key = tag.key, value = tag.value, propagate_at_launch = true }
  }
}

output "asg_name"      { value = aws_autoscaling_group.web.name }
output "target_group"  { value = aws_lb_target_group.web.arn }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

terraform {
  backend "s3" {
    bucket         = "company-tfstate"
    key            = "prod/web-service.tfstate"
    region         = "us-east-1"
    dynamodb_table = "tfstate-lock"   # 提供锁
    encrypt        = true
    kms_key_id     = "alias/terraform-state"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

global:
  scrape_interval:     15s
  evaluation_interval: 15s
  external_labels:
    cluster:     production
    region:      us-east-1

rule_files:
  - "rules/recording.yml"
  - "rules/alerts.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

scrape_configs:
  - job_name: kubernetes-pods
    kubernetes_sd_configs: [{ role: pod }]
    relabel_configs:
      # 只抓取通过注解显式开启的 Pod
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: "true"
      # 尊重自定义路径 / 端口注解
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__,
                        __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
      # 把有用的标签提到时序上
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_label_app]
        target_label: app

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# 节点 CPU 利用率（剔除 idle）
100 - avg by (instance) (
  irate(node_cpu_seconds_total{mode="idle"}[5m])
) * 100

# 每个服务的请求速率
sum by (service) (rate(http_requests_total[5m]))

# 错误率（占总数比例）
sum by (service) (rate(http_requests_total{status=~"5.."}[5m]))
/
sum by (service) (rate(http_requests_total[5m]))

# 来自 histogram 的 P95 延迟
histogram_quantile(0.95,
  sum by (le, service) (rate(http_request_duration_seconds_bucket[5m])))

# Apdex（excellent < 100ms、satisfactory < 500ms）
(
  sum by (service) (rate(http_request_duration_seconds_bucket{le="0.1"}[5m]))
  +
  sum by (service) (rate(http_request_duration_seconds_bucket{le="0.5"}[5m]))
) / 2
/ sum by (service) (rate(http_request_duration_seconds_count[5m]))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

groups:
- name: slo-burn-rate
  rules:
  - alert: ErrorBudgetBurnFast
    # 1 小时窗口内以 14.4 倍正常速率燃烧：约 2 天烧完月度预算
    expr: |
      (
        sum by (service) (rate(http_requests_total{status=~"5.."}[1h]))
        /
        sum by (service) (rate(http_requests_total[1h]))
      ) > 0.014
      and
      (
        sum by (service) (rate(http_requests_total{status=~"5.."}[5m]))
        /
        sum by (service) (rate(http_requests_total[5m]))
      ) > 0.014      
    for: 2m
    labels: { severity: page }
    annotations:
      summary: "{{ $labels.service }} 错误预算 14x 速率燃烧"
      runbook: "https://runbooks/internal/slo-burn"

  - alert: ErrorBudgetBurnSlow
    # 6 小时窗口 6 倍正常速率：约 5 天烧完月度预算
    expr: |
      (
        sum by (service) (rate(http_requests_total{status=~"5.."}[6h]))
        /
        sum by (service) (rate(http_requests_total[6h]))
      ) > 0.006      
    for: 15m
    labels: { severity: ticket }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

import json, logging, sys, time
import contextvars

request_id = contextvars.ContextVar("request_id", default="-")

class JsonFormatter(logging.Formatter):
    def format(self, record: logging.LogRecord) -> str:
        payload = {
            "ts":      time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created)),
            "level":   record.levelname,
            "logger":  record.name,
            "msg":     record.getMessage(),
            "module":  record.module,
            "line":    record.lineno,
            "request_id": request_id.get(),
        }
        if record.exc_info:
            payload["exc"] = self.formatException(record.exc_info)
        # 允许 extra={"user_id": ...} 富化字段
        for k, v in record.__dict__.items():
            if k not in payload and not k.startswith("_") and k not in (
                "msg", "args", "levelname", "levelno", "pathname", "filename",
                "module", "exc_info", "exc_text", "stack_info", "lineno",
                "funcName", "created", "msecs", "relativeCreated", "thread",
                "threadName", "processName", "process", "name", "message",
                "asctime",
            ):
                payload[k] = v
        return json.dumps(payload, default=str)

handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=logging.INFO, handlers=[handler])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata: { name: api-hpa, namespace: production }
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api
  minReplicas: 3
  maxReplicas: 30
  metrics:
    - type: Resource
      resource: { name: cpu, target: { type: Utilization, averageUtilization: 65 } }
    - type: Pods
      pods:
        metric: { name: http_requests_per_second }
        target: { type: AverageValue, averageValue: "100" }
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 0          # 立即响应
      policies:
        - type: Percent, value: 100, periodSeconds: 30
        - type: Pods,    value: 4,   periodSeconds: 30
      selectPolicy: Max
    scaleDown:
      stabilizationWindowSeconds: 300        # 缩容前等 5 分钟
      policies:
        - type: Percent, value: 25, periodSeconds: 60

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

import boto3
from datetime import datetime, timedelta, timezone

ec2, cw = boto3.client("ec2"), boto3.client("cloudwatch")

def idle_instances(threshold_pct: float = 5.0, lookback_days: int = 14) -> list[dict]:
    end   = datetime.now(timezone.utc)
    start = end - timedelta(days=lookback_days)

    out = []
    pages = ec2.get_paginator("describe_instances").paginate(
        Filters=[{"Name": "instance-state-name", "Values": ["running"]}])
    for page in pages:
        for r in page["Reservations"]:
            for i in r["Instances"]:
                m = cw.get_metric_statistics(
                    Namespace="AWS/EC2",
                    MetricName="CPUUtilization",
                    Dimensions=[{"Name": "InstanceId", "Value": i["InstanceId"]}],
                    StartTime=start, EndTime=end,
                    Period=3600, Statistics=["Average"],
                )["Datapoints"]
                if not m:
                    continue
                avg = sum(p["Average"] for p in m) / len(m)
                if avg < threshold_pct:
                    out.append({
                        "id":   i["InstanceId"],
                        "type": i["InstanceType"],
                        "tag_name": next((t["Value"] for t in i.get("Tags", [])
                                          if t["Key"] == "Name"), "-"),
                        "avg_cpu": round(avg, 1),
                    })
    return out

if __name__ == "__main__":
    for inst in idle_instances():
        print(inst)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

groups:
- name: slo
  interval: 30s
  rules:
  - record: slo:requests:rate5m
    expr: sum by (service) (rate(http_requests_total[5m]))

  - record: slo:errors:rate5m
    expr: sum by (service) (rate(http_requests_total{status=~"5.."}[5m]))

  - record: slo:availability:ratio_5m
    expr: 1 - (slo:errors:rate5m / slo:requests:rate5m)

  - record: slo:budget:remaining
    expr: |
      1 - (
        (1 - avg_over_time(slo:availability:ratio_5m[30d]))
        / (1 - 0.999)
      )      

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata: { name: web, namespace: argocd }
spec:
  project: default
  source:
    repoURL:        https://github.com/company/k8s-config
    targetRevision: main
    path:           apps/web/overlays/production
  destination:
    server:    https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated:
      prune:    true       # 删掉 git 中不再存在的资源
      selfHeal: true       # 把人手改的东西自动改回来
    retry:
      limit: 5
      backoff: { duration: 30s, maxDuration: 5m, factor: 2 }