feat(prometheus): add alerting rules and alertmanager config

prometheus-stack/k8s/alertmanager-config.example.yaml

@@ -0,0 +1,38 @@
+route:
+  receiver: telegram
+  group_by:
+    - alertname
+    - namespace
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 12h
+  routes:
+    - receiver: null
+      matchers:
+        - alertname="InfoInhibitor"
+    - receiver: null
+      matchers:
+        - alertname="Watchdog"
+
+inhibit_rules:
+  - source_matchers:
+      - severity="critical"
+    target_matchers:
+      - severity=~"warning|info"
+    equal:
+      - namespace
+  - source_matchers:
+      - severity="warning"
+    target_matchers:
+      - severity="info"
+    equal:
+      - namespace
+
+receivers:
+  - name: telegram
+    telegram_configs:
+      - bot_token: REPLACE_WITH_TELEGRAM_BOT_TOKEN
+        chat_id: REPLACE_WITH_TELEGRAM_CHAT_ID
+        parse_mode: HTML
+        send_resolved: true
+  - name: null

prometheus-stack/k8s/alerts.yaml

@@ -0,0 +1,68 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: homelab-infrastructure
+  namespace: prometheus
+  labels:
+    release: prometheus-stack
+spec:
+  groups:
+    - name: homelab.infrastructure
+      rules:
+        - alert: TargetDown
+          expr: up == 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Prometheus target is down"
+            description: "{{ $labels.job }} target {{ $labels.instance }} has been down for more than 10 minutes."
+
+        - alert: PodCrashLooping
+          expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}[10m]) >= 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Pod is crash looping"
+            description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff."
+
+        - alert: PersistentVolumeClaimFillingUp
+          expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "PVC has less than 15% free space"
+            description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} has less than 15% free space."
+
+        - alert: CPUThrottlingHigh
+          expr: |
+            sum without (id, metrics_path, name, image, endpoint, job, node) (
+              topk by (cluster, namespace, pod, container, instance) (1,
+                increase(container_cpu_cfs_throttled_periods_total{container!="", job="kubelet", metrics_path="/metrics/cadvisor", }[5m])
+              )
+            )
+            / on (cluster, namespace, pod, container, instance) group_left
+            sum without (id, metrics_path, name, image, endpoint, job, node) (
+              topk by (cluster, namespace, pod, container, instance) (1,
+                increase(container_cpu_cfs_periods_total{job="kubelet", metrics_path="/metrics/cadvisor", }[5m])
+              )
+            )
+            > ( 50 / 100 )
+          for: 30m
+          labels:
+            severity: info
+          annotations:
+            summary: "Processes experience elevated CPU throttling"
+            description: "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }} on cluster {{ $labels.cluster }}."
+            runbook_url: "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh"
+
+        - alert: NodeMemoryPressure
+          expr: 100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 90
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Node memory pressure"
+            description: "Node {{ $labels.instance }} has used more than 90% of memory for 15 minutes."

prometheus-stack/k8s/grafana-values.yaml

@@ -14,7 +14,8 @@ grafana:
 
   persistence:
     enabled: true
-    size: 10Gi
+    storageClassName: local-path-retain
+    size: 20Gi
 
   ingress:
     enabled: false
@@ -24,10 +25,12 @@ grafana:
 
 prometheus:
   prometheusSpec:
+    retention: 60d
+    retentionSize: 32GB
     storageSpec:
       volumeClaimTemplate:
         spec:
-          storageClassName: "local-path"
+          storageClassName: "local-path-retain"
           accessModes:
             - ReadWriteOnce
           resources:
@@ -41,12 +44,17 @@ prometheus:
         memory: "2Gi"
 alertmanager:
   alertmanagerSpec:
+    configSecret: alertmanager-config
     storage:
       volumeClaimTemplate:
         spec:
+          storageClassName: local-path-retain
           accessModes:
             - ReadWriteOnce
-
           resources:
             requests:
-              storage: 20Gi
+              storage: 10Gi
+
+defaultRules:
+  disabled:
+    CPUThrottlingHigh: true

prometheus-stack/k8s/traefik-alerts.yaml

@@ -0,0 +1,60 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: traefik
+  namespace: prometheus
+  labels:
+    release: prometheus-stack
+spec:
+  groups:
+    - name: traefik
+      rules:
+        - alert: TraefikDown
+          expr: absent(up{job="traefik"})
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Traefik metrics are unavailable"
+            description: "Prometheus has no Traefik target."
+
+        - alert: TraefikConfigReloadFailed
+          expr: traefik_config_last_reload_success == 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Traefik configuration reload failed"
+            description: "Traefik failed to apply its last configuration reload. Check Traefik logs and the dynamic config sources."
+
+        - alert: TraefikServiceHigh5xxRate
+          expr: |
+            sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) by (service)
+              / sum(rate(traefik_service_requests_total[5m])) by (service) * 100 > 5
+              and sum(rate(traefik_service_requests_total[5m])) by (service) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High 5xx error rate for service {{ $labels.service }}"
+            description: "Service {{ $labels.service }} is returning 5xx errors for more than 5% of requests over the last 5 minutes (current: {{ $value | humanizePercentage }})."
+
+        - alert: TraefikServiceHighLatency
+          expr: |
+            histogram_quantile(0.95,
+              sum(rate(traefik_service_request_duration_seconds_bucket[5m])) by (le, service)) > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High latency for service {{ $labels.service }}"
+            description: "P95 latency of {{ $labels.service }} exceeded 2 seconds over the last 5 minutes (current: {{ $value | humanizeDuration }})."
+
+        - alert: TraefikCertExpiringSoon
+          expr: min(traefik_tls_certs_not_after) - time() < 14 * 24 * 60 * 60
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Traefik TLS certificate expires soon"
+            description: "A TLS certificate managed by Traefik expires in less than 14 days (in {{ $value | humanizeDuration }})."