Cloud Computing (7): Cloud Operations and DevOps Practices

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

name: deploy
on:
  push:
    branches: [main]

permissions:
  id-token: write          # OIDC for cloud auth
  contents: read
  packages: write

env:
  REGISTRY: ghcr.io
  IMAGE:    ghcr.io/${{ github.repository }}

jobs:
  build-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Go
        uses: actions/setup-go@v5
        with: { go-version-file: go.mod }

      - name: Lint
        uses: golangci/golangci-lint-action@v4

      - name: Unit tests
        run: go test -race -coverprofile=coverage.out ./...

      - name: Coverage gate
        run: |
          pct=$(go tool cover -func=coverage.out | tail -1 | awk '{print $3}' | tr -d '%')
          echo "Coverage: ${pct}%"
          if (( $(echo "$pct < 80" | bc -l) )); then
            echo "::error::Coverage ${pct}% is below 80% threshold"
            exit 1
          fi          

      - name: Build and push image
        run: |
          echo "${{ secrets.GITHUB_TOKEN }}" | docker login $REGISTRY -u ${{ github.actor }} --password-stdin
          docker build -t $IMAGE:${{ github.sha }} -t $IMAGE:latest .
          docker push $IMAGE --all-tags          

  security-scan:
    needs: build-test
    runs-on: ubuntu-latest
    steps:
      - name: Trivy image scan
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.IMAGE }}:${{ github.sha }}
          severity: CRITICAL,HIGH
          exit-code: 1           # fail the build on critical CVEs

      - name: SAST with Semgrep
        uses: returntocorp/semgrep-action@v1
        with:
          config: p/owasp-top-ten

  deploy-staging:
    needs: security-scan
    runs-on: ubuntu-latest
    environment: staging
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::111122223333:role/deploy-staging
          aws-region: us-east-1

      - name: Deploy to ECS staging
        run: |
          aws ecs update-service \
            --cluster staging \
            --service web \
            --force-new-deployment \
            --task-definition web:$(aws ecs describe-task-definition \
              --task-definition web --query 'taskDefinition.revision')          

      - name: Wait for stable
        run: aws ecs wait services-stable --cluster staging --services web

  smoke-test:
    needs: deploy-staging
    runs-on: ubuntu-latest
    steps:
      - name: Health check
        run: |
          for i in {1..10}; do
            status=$(curl -s -o /dev/null -w '%{http_code}' https://staging.example.com/healthz)
            if [ "$status" = "200" ]; then echo "Healthy"; exit 0; fi
            sleep 5
          done
          echo "::error::Staging health check failed after 50s"
          exit 1          

      - name: API contract tests
        run: |
          npm ci
          npx newman run tests/postman/smoke.json \
            --environment tests/postman/staging.json \
            --reporters cli,junit \
            --reporter-junit-export results.xml          

  deploy-prod:
    needs: smoke-test
    runs-on: ubuntu-latest
    environment: production       # requires manual approval
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::111122223333:role/deploy-prod
          aws-region: us-east-1

      - name: Canary deploy (10% traffic)
        run: |
          aws ecs update-service --cluster prod --service web \
            --deployment-configuration '{
              "deploymentCircuitBreaker": {"enable": true, "rollback": true},
              "maximumPercent": 200,
              "minimumHealthyPercent": 100
            }' \
            --force-new-deployment          

      - name: Verify canary SLOs (5 min)
        run: |
          sleep 300
          error_rate=$(aws cloudwatch get-metric-statistics \
            --namespace Custom/Web --metric-name ErrorRate \
            --start-time $(date -u -d '5 minutes ago' +%Y-%m-%dT%H:%M:%S) \
            --end-time $(date -u +%Y-%m-%dT%H:%M:%S) \
            --period 300 --statistics Average \
            --query 'Datapoints[0].Average' --output text)
          if (( $(echo "$error_rate > 1.0" | bc -l) )); then
            echo "::error::Error rate ${error_rate}% exceeds 1% - triggering rollback"
            aws ecs update-service --cluster prod --service web \
              --task-definition web:$(( $(aws ecs describe-services --cluster prod \
                --services web --query 'services[0].taskDefinition' --output text \
                | grep -o '[0-9]*$') - 1 ))
            exit 1
          fi          

1
2
3
4

terraform init      # download providers, configure backend
terraform plan      # preview: what will be created / changed / destroyed
terraform apply     # make reality match the code
terraform destroy   # tear it all down (non-prod only!)

1
2
3
4
5
6
7
8

# Generate a plan file for CI
terraform plan -out=tfplan -input=false

# Show it in human-readable form for PR comments
terraform show -no-color tfplan > plan.txt

# Apply only the reviewed plan (no re-planning)
terraform apply -input=false tfplan

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# backend.tf
terraform {
  backend "s3" {
    bucket         = "company-terraform-state"
    key            = "services/web/prod/terraform.tfstate"
    region         = "us-east-1"
    dynamodb_table = "terraform-locks"
    encrypt        = true
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

# modules/ecs-service/main.tf
variable "name"          { type = string }
variable "environment"   { type = string }
variable "image"         { type = string }
variable "cpu"           { type = number, default = 256 }
variable "memory"        { type = number, default = 512 }
variable "desired_count" { type = number, default = 2 }
variable "health_path"   { type = string, default = "/healthz" }

resource "aws_ecs_task_definition" "this" {
  family                   = "${var.name}-${var.environment}"
  network_mode             = "awsvpc"
  requires_compatibilities = ["FARGATE"]
  cpu                      = var.cpu
  memory                   = var.memory
  execution_role_arn       = aws_iam_role.execution.arn
  task_role_arn            = aws_iam_role.task.arn

  container_definitions = jsonencode([{
    name      = var.name
    image     = var.image
    essential = true
    portMappings = [{ containerPort = 8080, protocol = "tcp" }]
    logConfiguration = {
      logDriver = "awslogs"
      options = {
        awslogs-group         = aws_cloudwatch_log_group.this.name
        awslogs-region        = data.aws_region.current.name
        awslogs-stream-prefix = var.name
      }
    }
    healthCheck = {
      command     = ["CMD-SHELL", "curl -f http://localhost:8080${var.health_path} || exit 1"]
      interval    = 15
      timeout     = 5
      retries     = 3
      startPeriod = 60
    }
  }])
}

resource "aws_ecs_service" "this" {
  name            = "${var.name}-${var.environment}"
  cluster         = data.aws_ecs_cluster.main.id
  task_definition = aws_ecs_task_definition.this.arn
  desired_count   = var.desired_count
  launch_type     = "FARGATE"

  deployment_circuit_breaker {
    enable   = true
    rollback = true
  }

  network_configuration {
    subnets          = data.aws_subnets.private.ids
    security_groups  = [aws_security_group.service.id]
    assign_public_ip = false
  }

  load_balancer {
    target_group_arn = aws_lb_target_group.this.arn
    container_name   = var.name
    container_port   = 8080
  }
}

output "service_url" {
  value = "https://${aws_lb.this.dns_name}"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#!/bin/bash
# drift-check.sh -- run daily via cron or scheduled pipeline
set -euo pipefail

SERVICES="web api worker"
ENVS="staging prod"

for svc in $SERVICES; do
  for env in $ENVS; do
    dir="services/${svc}/${env}"
    echo "=== Checking ${dir} ==="
    cd "$dir"
    terraform init -backend=true -input=false > /dev/null
    if ! terraform plan -detailed-exitcode -input=false > /dev/null 2>&1; then
      echo "DRIFT DETECTED in ${dir}"
      terraform plan -no-color -input=false > "/tmp/drift-${svc}-${env}.txt"
      # Send to Slack / PagerDuty
      curl -X POST "$SLACK_WEBHOOK" -d "{\"text\":\"Drift detected in ${dir}. See attached plan.\"}"
    else
      echo "No drift in ${dir}"
    fi
    cd - > /dev/null
  done
done

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alerts/*.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

scrape_configs:
  - job_name: "kubernetes-pods"
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # Only scrape pods with annotation prometheus.io/scrape=true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_ip]
        action: replace
        target_label: __address__
        regex: (.+);(.+)
        replacement: $2:$1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

package main

import (
    "net/http"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    httpDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "http_request_duration_seconds",
            Help:    "Duration of HTTP requests.",
            Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
        },
        []string{"method", "route", "status"},
    )
    httpTotal = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total number of HTTP requests.",
        },
        []string{"method", "route", "status"},
    )
)

func init() {
    prometheus.MustRegister(httpDuration, httpTotal)
}

func instrumentHandler(route string, next http.HandlerFunc) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        rw := &responseWriter{ResponseWriter: w, statusCode: 200}
        next(rw, r)
        duration := time.Since(start).Seconds()
        status := http.StatusText(rw.statusCode)
        httpDuration.WithLabelValues(r.Method, route, status).Observe(duration)
        httpTotal.WithLabelValues(r.Method, route, status).Inc()
    }
}

func main() {
    http.Handle("/metrics", promhttp.Handler())
    http.HandleFunc("/api/orders", instrumentHandler("/api/orders", handleOrders))
    http.ListenAndServe(":8080", nil)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

# alerts/slo.yml
groups:
  - name: slo-burn-rate
    rules:
      # Multi-window, multi-burn-rate alerting (Google SRE Workbook pattern)
      # Fast burn: 14.4x in 1h AND 6x in 6h
      - alert: HighErrorBurnRate_Critical
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[1h]))
            / sum(rate(http_requests_total[1h]))
          ) > (14.4 * 0.001)
          and
          (
            sum(rate(http_requests_total{status=~"5.."}[6h]))
            / sum(rate(http_requests_total[6h]))
          ) > (6 * 0.001)          
        for: 2m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "Error budget burning fast -- will exhaust in < 1 day"
          runbook: "https://wiki.internal/runbooks/high-error-rate"
          dashboard: "https://grafana.internal/d/slo-overview"

      # Slow burn: 3x in 1d AND 1x in 3d
      - alert: HighErrorBurnRate_Warning
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[1d]))
            / sum(rate(http_requests_total[1d]))
          ) > (3 * 0.001)
          and
          (
            sum(rate(http_requests_total{status=~"5.."}[3d]))
            / sum(rate(http_requests_total[3d]))
          ) > (1 * 0.001)          
        for: 15m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Error budget burning slowly -- will exhaust in < 10 days"
          runbook: "https://wiki.internal/runbooks/high-error-rate"

      - alert: HighLatency_P99
        expr: |
          histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))
          > 2.0          
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency above 2s for {{ $labels.service }}"
          runbook: "https://wiki.internal/runbooks/high-latency"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

# alertmanager.yml
global:
  resolve_timeout: 5m

route:
  receiver: "default-slack"
  group_by: ["alertname", "service"]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h

  routes:
    - match:
        severity: critical
      receiver: "pagerduty-critical"
      repeat_interval: 15m

    - match:
        severity: warning
      receiver: "slack-warnings"
      repeat_interval: 4h

receivers:
  - name: "pagerduty-critical"
    pagerduty_configs:
      - routing_key: "<pagerduty-integration-key>"
        severity: critical

  - name: "slack-warnings"
    slack_configs:
      - api_url: "https://hooks.slack.com/services/xxx"
        channel: "#alerts-warnings"
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'

  - name: "default-slack"
    slack_configs:
      - api_url: "https://hooks.slack.com/services/xxx"
        channel: "#alerts-default"

1
2
3
4
5
6
7
8
9

+-----------+     +------------+     +---------------+     +---------+
| App pods  | --> | Fluent Bit | --> | Elasticsearch | --> | Kibana  |
| (stdout)  |     | (DaemonSet)|     | (3-node HA)   |     | (query) |
+-----------+     +------------+     +---------------+     +---------+
                        |
                        v
                  +------------+
                  |   Kafka    |  (optional buffer for high volume)
                  +------------+

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

import structlog
import uuid

# Configure structlog for JSON output
structlog.configure(
    processors=[
        structlog.processors.TimeStamper(fmt="iso"),

![Centralised logging pipeline from sources to Elasticsearch and Kibana](https://blog-pic-ck.oss-cn-beijing.aliyuncs.com/posts/en/cloud-computing/operations-devops/fig4_logging_pipeline.png)

        structlog.processors.add_log_level,
        structlog.processors.StackInfoRenderer(),
        structlog.processors.format_exc_info,
        structlog.processors.JSONRenderer(),
    ],
    wrapper_class=structlog.BoundLogger,
)

log = structlog.get_logger()

def handle_request(request):
    request_id = request.headers.get("X-Request-ID", str(uuid.uuid4()))
    logger = log.bind(
        request_id=request_id,
        method=request.method,
        path=request.path,
        user_id=request.user.id if request.user else None,
    )

    logger.info("request_started")
    try:
        result = process(request)
        logger.info("request_completed", status=200, duration_ms=result.duration_ms)
        return result
    except ValidationError as e:
        logger.warning("validation_failed", error=str(e), status=400)
        raise
    except Exception as e:
        logger.error("request_failed", error=str(e), status=500, exc_info=True)
        raise

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

{
  "timestamp": "2024-03-15T14:30:22.123Z",
  "level": "info",
  "event": "request_completed",
  "request_id": "abc-123",
  "method": "POST",
  "path": "/api/orders",
  "user_id": "u_456",
  "status": 200,
  "duration_ms": 42
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# fluent-bit.conf
[SERVICE]
    Flush         5
    Daemon        Off
    Log_Level     info
    Parsers_File  parsers.conf

[INPUT]
    Name              tail
    Path              /var/log/containers/*.log
    Parser            cri
    Tag               kube.*
    Mem_Buf_Limit     50MB
    Skip_Long_Lines   On
    Refresh_Interval  10

[FILTER]
    Name                kubernetes
    Match               kube.*
    Kube_URL            https://kubernetes.default.svc:443
    Kube_Tag_Prefix     kube.var.log.containers.
    Merge_Log           On
    Keep_Log            Off
    K8S-Logging.Parser  On

[FILTER]
    Name    modify
    Match   kube.*
    # Scrub sensitive fields before indexing
    Remove  password
    Remove  authorization
    Remove  cookie
    Remove  x-api-key

[OUTPUT]
    Name            es
    Match           kube.*
    Host            elasticsearch.logging.svc.cluster.local
    Port            9200
    Logstash_Format On
    Logstash_Prefix k8s-logs
    Retry_Limit     5
    tls             On
    tls.verify      On

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": { "max_size": "50gb", "max_age": "1d" },
          "set_priority": { "priority": 100 }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "shrink": { "number_of_shards": 1 },
          "forcemerge": { "max_num_segments": 1 },
          "set_priority": { "priority": 50 }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "searchable_snapshot": { "snapshot_repository": "s3-logs" },
          "set_priority": { "priority": 0 }
        }
      },
      "delete": { "min_age": "365d", "actions": { "delete": {} } }
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: web-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: web
  minReplicas: 3
  maxReplicas: 50
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60    # wait 1 min before scaling up
      policies:
        - type: Percent
          value: 100                     # double at most
          periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300   # wait 5 min before scaling down
      policies:
        - type: Percent
          value: 10                      # remove at most 10% per period
          periodSeconds: 60
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
    - type: Pods
      pods:
        metric:
          name: http_requests_per_second
        target:
          type: AverageValue
          averageValue: "1000"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# AWS Auto Scaling predictive policy
aws autoscaling put-scaling-policy \
  --auto-scaling-group-name web-asg \
  --policy-name predictive-scaling \
  --policy-type PredictiveScaling \
  --predictive-scaling-configuration '{
    "MetricSpecifications": [{
      "TargetValue": 70,
      "PredefinedMetricPairSpecification": {
        "PredefinedMetricType": "ASGCPUUtilization"
      }
    }],
    "Mode": "ForecastAndScale",
    "SchedulingBufferTime": 300
  }'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# shutdown-nonprod.sh -- cron: 0 19 * * 1-5 (7 PM weekdays)
#!/bin/bash
set -euo pipefail

ENVS=("dev" "staging" "qa")

for env in "${ENVS[@]}"; do
  echo "Shutting down ${env} ECS services..."
  for svc in $(aws ecs list-services --cluster "$env" --query 'serviceArns[]' --output text); do
    aws ecs update-service --cluster "$env" --service "$svc" --desired-count 0
  done

  echo "Stopping ${env} RDS instances..."
  for db in $(aws rds describe-db-instances --query "DBInstances[?TagList[?Key=='Environment'&&Value=='${env}']].DBInstanceIdentifier" --output text); do
    aws rds stop-db-instance --db-instance-identifier "$db" || true
  done
done

echo "Non-prod environments shut down at $(date)"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# startup-nonprod.sh -- cron: 0 8 * * 1-5 (8 AM weekdays)
#!/bin/bash
set -euo pipefail

ENVS=("dev" "staging" "qa")

for env in "${ENVS[@]}"; do
  echo "Starting ${env} RDS instances..."
  for db in $(aws rds describe-db-instances --query "DBInstances[?TagList[?Key=='Environment'&&Value=='${env}']].DBInstanceIdentifier" --output text); do
    aws rds start-db-instance --db-instance-identifier "$db" || true
  done

  echo "Restoring ${env} ECS services..."
  for svc in $(aws ecs list-services --cluster "$env" --query 'serviceArns[]' --output text); do
    aws ecs update-service --cluster "$env" --service "$svc" --desired-count 2
  done
done

echo "Non-prod environments started at $(date)"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

{
  "Version": "2012-10-17",
  "Statement": [{
    "Sid": "RequireTags",
    "Effect": "Deny",
    "Action": ["ec2:RunInstances", "rds:CreateDBInstance", "s3:CreateBucket"],
    "Resource": "*",
    "Condition": {
      "Null": {
        "aws:RequestTag/Environment": "true",
        "aws:RequestTag/Team": "true",
        "aws:RequestTag/Service": "true"
      }
    }
  }]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# SLO definition (as applied in Prometheus)
# 99.9% availability = 0.1% error budget = 43.2 min/month

# Availability SLI
- record: sli:availability:ratio_rate5m
  expr: |
    sum(rate(http_requests_total{status!~"5.."}[5m]))
    / sum(rate(http_requests_total[5m]))    

# Latency SLI (proportion of requests < 300ms)
- record: sli:latency:ratio_rate5m
  expr: |
    sum(rate(http_request_duration_seconds_bucket{le="0.3"}[5m]))
    / sum(rate(http_request_duration_seconds_count[5m]))    

# Error budget remaining (30-day window)
- record: error_budget:remaining
  expr: |
    1 - (
      (1 - avg_over_time(sli:availability:ratio_rate5m[30d]))
      / (1 - 0.999)
    )    

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

## Incident Report: [Title]
**Date**: YYYY-MM-DD
**Duration**: X hours Y minutes
**Severity**: SEV-1 / SEV-2 / SEV-3
**Author**: [name]
**Reviewers**: [names]

### Summary
One paragraph: what happened, what was the user impact, how was it resolved.

### Timeline (all times UTC)
- 14:00 - Monitoring alert fires: error rate > 5%
- 14:03 - On-call engineer acknowledges, begins investigation
- 14:15 - Root cause identified: database connection pool exhausted
- 14:20 - Mitigation applied: increased pool size, restarted service
- 14:25 - Error rate returns to baseline
- 14:30 - Incident declared resolved

### Root cause
The connection pool was sized for 50 connections. A new batch job,
deployed two days prior, held connections for 30 seconds instead of
the expected 100ms, exhausting the pool during peak traffic.

### Impact
- 25 minutes of elevated error rates (peak 12%)
- ~3,200 failed requests
- No data loss

### What went well
- Alert fired within 3 minutes of the problem starting
- On-call engineer had the right access to diagnose and mitigate

### What went poorly
- The batch job was not load-tested against the shared database
- The connection pool had no circuit breaker

### Action items
| Action | Owner | Priority | Due date |
|--------|-------|----------|----------|
| Add connection pool metrics to dashboard | Alice | P1 | 2024-02-01 |
| Load-test batch jobs against shared resources | Bob | P1 | 2024-02-15 |
| Implement connection pool circuit breaker | Carol | P2 | 2024-03-01 |
| Add runbook for connection pool exhaustion | Dave | P2 | 2024-02-01 |

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata: { name: web, namespace: argocd }
spec:
  project: default
  source:
    repoURL:        https://github.com/company/k8s-config
    targetRevision: main
    path:           apps/web/overlays/production
  destination:
    server:    https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated:
      prune:    true       # delete resources removed from git
      selfHeal: true       # revert manual changes back to git state
    retry:
      limit: 5
      backoff: { duration: 30s, maxDuration: 5m, factor: 2 }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

k8s-config/
  apps/
    web/
      base/
        deployment.yaml
        service.yaml
        hpa.yaml
        kustomization.yaml
      overlays/
        staging/
          kustomization.yaml      # patches: replica count, resource limits
          ingress-patch.yaml      # staging domain
        production/
          kustomization.yaml      # patches: higher replicas, more resources
          ingress-patch.yaml      # prod domain
    api/
      base/
        ...
      overlays/
        ...
  platform/
    monitoring/
      prometheus/
      grafana/
    logging/
      fluent-bit/
      elasticsearch/

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# 1. What is the symptom? Check the dashboard.
open https://grafana.internal/d/overview

# 2. Is it one service or everything?
kubectl get pods -A | grep -v Running
kubectl top nodes
kubectl top pods --sort-by=cpu -A | head -20

# 3. What changed recently?
kubectl rollout history deployment/web -n production
git log --oneline --since="2 hours ago" -- apps/web/

# 4. Check resource pressure
kubectl describe nodes | grep -A5 "Conditions:"
df -h                           # disk
free -m                         # memory
ss -tlnp                        # open ports / connection counts

# 5. Read the logs for the affected service
kubectl logs -l app=web -n production --tail=100 --since=5m

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# PostgreSQL: slow queries right now
SELECT pid, now() - pg_stat_activity.query_start AS duration,
       query, state, wait_event_type, wait_event
FROM pg_stat_activity
WHERE state != 'idle'
  AND (now() - pg_stat_activity.query_start) > interval '5 seconds'
ORDER BY duration DESC;

# PostgreSQL: connection count by state
SELECT state, count(*)
FROM pg_stat_activity
GROUP BY state;

# PostgreSQL: table bloat check
SELECT schemaname, tablename,
       pg_size_pretty(pg_total_relation_size(schemaname || '.' || tablename)) as total_size,
       n_dead_tup,
       n_live_tup,
       ROUND(n_dead_tup * 100.0 / NULLIF(n_live_tup + n_dead_tup, 0), 2) as dead_pct
FROM pg_stat_user_tables
WHERE n_dead_tup > 10000
ORDER BY n_dead_tup DESC;

# MySQL: current queries
SHOW FULL PROCESSLIST;

# MySQL: InnoDB status (locks, deadlocks, buffer pool)
SHOW ENGINE INNODB STATUS\G

# Redis: memory and key stats
redis-cli INFO memory
redis-cli INFO keyspace
redis-cli --bigkeys          # find large keys
redis-cli SLOWLOG GET 10     # recent slow commands