Kubernetes에서 Flagger 카나리 배포 모니터링하기: Grafana 대시보드 + AlertManager Slack 알림 가이드

Flagger ──(메트릭 기록)──▶ Prometheus
                               │
              ┌────────────────┴─────────────────┐
              ▼                                   ▼
         Grafana                          AlertManager
    (실시간 대시보드)                    (알림 그루핑·라우팅)
                                               │
                                               ▼
                                     Slack (#deployments-alert)
                                    (30초 이내 팀 전체 인지)

Primary (90%) ──▶ 실 사용자 트래픽
Canary  (10%) ──▶ 점진적으로 증가 (stepWeight: 10)
               └▶ 분석 실패 시 0%로 즉시 롤백

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: flagger
  namespace: flagger-system
  labels:
    release: kube-prometheus-stack   # Prometheus Operator의 selector에 맞게 조정
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: flagger
  namespaceSelector:
    matchNames:
      - flagger-system
  endpoints:
    - port: http
      path: /metrics
      interval: 15s

helm upgrade -i flagger-grafana flagger/grafana \
  --create-namespace \
  --namespace=flagger-system \
  --set url=http://prometheus:9090

# 카나리 현재 상태 (3 = 실패/롤백)
flagger_canary_status{name="my-app", namespace="default"}
 
# 현재 트래픽 가중치 (%)
flagger_canary_weight{name="my-app", namespace="default"}
 
# 요청 성공률 (5xx 제외)
sum(rate(http_requests_total{status!~"5.."}[5m]))
  / sum(rate(http_requests_total[5m])) * 100
 
# 카나리 분석 경과 시간 (초)
flagger_canary_duration_seconds{name="my-app"}

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: flagger-alerts
  namespace: flagger-system
  labels:
    release: kube-prometheus-stack   # Prometheus Operator selector에 맞게 조정
spec:
  groups:
    - name: flagger.rules
      rules:
        - alert: CanaryRollback
          expr: flagger_canary_status == 3
          for: 0m
          labels:
            severity: critical
          annotations:
            summary: "카나리 롤백 발생: {{ $labels.name }}"
            description: >
              네임스페이스 {{ $labels.namespace }}의
              {{ $labels.name }} 카나리 분석 실패로 롤백됨
 
        - alert: CanaryProgressing
          expr: flagger_canary_status == 1
          for: 1m
          labels:
            severity: info
          annotations:
            summary: "카나리 배포 진행 중: {{ $labels.name }}"
 
        - alert: CanaryWeightHigh
          expr: flagger_canary_weight > 50
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "카나리 트래픽 50% 초과: {{ $labels.name }} ({{ $value }}%)"

# values.yaml
alertmanager:
  config:
    global:
      resolve_timeout: 5m
    route:
      group_by: ['alertname', 'namespace']
      group_wait: 10s
      group_interval: 10m
      repeat_interval: 1h
      receiver: 'slack-default'
      routes:
        - match:
            alertname: CanaryRollback
            severity: critical
          receiver: 'slack-canary-rollback'
          continue: false
    receivers:
      - name: 'slack-canary-rollback'
        slack_configs:
          - api_url: 'https://hooks.slack.com/services/T.../B.../XXXX'
            channel: '#deployments-alert'
            send_resolved: true
            title: ':rotating_light: 카나리 롤백 발생'
            text: |
              *앱:* {{ .GroupLabels.name }}
              *네임스페이스:* {{ .GroupLabels.namespace }}
              *상태:* {{ .CommonAnnotations.description }}
            color: 'danger'
      - name: 'slack-default'
        slack_configs:
          - api_url: 'https://hooks.slack.com/services/T.../B.../XXXX'
            channel: '#deployments'
            send_resolved: true
            title: '{{ .GroupLabels.alertname }}'
            text: '{{ .CommonAnnotations.summary }}'

helm upgrade kube-prometheus-stack prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  -f values.yaml

kubectl create secret generic alertmanager-kube-prometheus-stack-alertmanager \
  --from-file=alertmanager.yaml=./alertmanager.yaml \
  --namespace monitoring \
  --dry-run=client -o yaml | kubectl apply -f -

kubectl create secret generic slack-webhook-secret \
  --from-literal=address='https://hooks.slack.com/services/T.../B.../XXXX' \
  --namespace flagger-system

apiVersion: notification.toolkit.fluxcd.io/v1beta2
kind: Provider
metadata:
  name: slack-provider
  namespace: flagger-system
spec:
  type: slack
  channel: deployments
  secretRef:
    name: slack-webhook-secret

apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
  name: my-app
  namespace: production
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-app
  analysis:
    interval: 1m
    threshold: 5
    maxWeight: 50
    stepWeight: 10
    metrics:
      - name: request-success-rate
        thresholdRange:
          min: 99
        interval: 1m
      - name: request-duration
        thresholdRange:
          max: 500
        interval: 30s
  alerts:
    - name: "rollback-alert"
      severity: error
      providerRef:
        name: slack-provider
        namespace: flagger-system