Day 8 (alt) — AlertManager 真接入 + PrometheusRule 实战

apiVersion: apps/v1
kind: Deployment
metadata: {name: webhook-mock, namespace: alert-receiver}
spec:
  replicas: 1
  template:
    spec:
      containers:
      - name: c
        image: mendhak/http-https-echo:latest    # 现成 echo server
        ports: [{containerPort: 8080}]
        env:
        - {name: HTTP_PORT, value: '8080'}

apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
  name: bootcamp-routes
  namespace: monitoring
  labels:
    alertmanagerConfig: bootcamp     # ← matchLabel for AM selector
spec:
  route:
    receiver: webhook-mock
    groupBy: [alertname, severity]
    groupWait: 10s
    groupInterval: 30s
    repeatInterval: 1h
    routes:
    - matchers:
      - {name: severity, value: critical}
      receiver: webhook-mock
      groupWait: 0s                  # critical 立即推
    - matchers:
      - {name: alertname, value: Watchdog}
      receiver: webhook-mock
  receivers:
  - name: webhook-mock
    webhookConfigs:
    - url: http://webhook-mock.alert-receiver.svc.cluster.local/alerts
      sendResolved: true              # ← resolve 后也通知一次

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: bootcamp-custom-alerts
  namespace: monitoring
  labels:
    release: kps              # ← kps chart selector 抓这个 label
spec:
  groups:
  - name: bootcamp.cluster
    rules:
    # ① CPU 高
    - alert: NodeCPUHigh
      expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
      for: 2m
      labels: {severity: warning}
      annotations:
        summary: 'Node CPU > 80% on {{ $labels.instance }}'

    # ② Longhorn volume 降级
    - alert: LonghornVolumeDegraded
      expr: longhorn_volume_robustness == 2
      for: 3m
      labels: {severity: warning}

    # ③ PVC > 85%
    - alert: PVCFillingUp
      expr: (kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes) > 0.85
      for: 5m
      labels: {severity: critical}

    # ④ etcd p99 写延迟 > 1s
    - alert: EtcdHighWriteLatency
      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 1
      for: 5m
      labels: {severity: critical}

    # ⑤ apiserver 5xx 率 > 5%
    - alert: ApiserverHighErrorRate
      expr: sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) > 0.05
      for: 5m
      labels: {severity: critical}

receivers:
- name: dingtalk
  webhookConfigs:
  - url: https://oapi.dingtalk.com/robot/send?access_token=XXX
    sendResolved: true

receivers:
- name: slack
  slackConfigs:
  - apiURL: https://hooks.slack.com/services/XXX
    channel: '#alerts'
    sendResolved: true
    title: '{{ .GroupLabels.alertname }}'
    text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'

receivers:
- name: wechat
  wechatConfigs:
  - corpId: XXX
    apiSecret: XXX
    sendResolved: true
    toUser: '@all'