diff --git a/prometheus-stack/k8s/alertmanager-config.example.yaml b/prometheus-stack/k8s/alertmanager-config.example.yaml new file mode 100644 index 0000000..5143129 --- /dev/null +++ b/prometheus-stack/k8s/alertmanager-config.example.yaml @@ -0,0 +1,38 @@ +route: + receiver: telegram + group_by: + - alertname + - namespace + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + routes: + - receiver: null + matchers: + - alertname="InfoInhibitor" + - receiver: null + matchers: + - alertname="Watchdog" + +inhibit_rules: + - source_matchers: + - severity="critical" + target_matchers: + - severity=~"warning|info" + equal: + - namespace + - source_matchers: + - severity="warning" + target_matchers: + - severity="info" + equal: + - namespace + +receivers: + - name: telegram + telegram_configs: + - bot_token: REPLACE_WITH_TELEGRAM_BOT_TOKEN + chat_id: REPLACE_WITH_TELEGRAM_CHAT_ID + parse_mode: HTML + send_resolved: true + - name: null diff --git a/prometheus-stack/k8s/alerts.yaml b/prometheus-stack/k8s/alerts.yaml new file mode 100644 index 0000000..04a7306 --- /dev/null +++ b/prometheus-stack/k8s/alerts.yaml @@ -0,0 +1,68 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: homelab-infrastructure + namespace: prometheus + labels: + release: prometheus-stack +spec: + groups: + - name: homelab.infrastructure + rules: + - alert: TargetDown + expr: up == 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Prometheus target is down" + description: "{{ $labels.job }} target {{ $labels.instance }} has been down for more than 10 minutes." + + - alert: PodCrashLooping + expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}[10m]) >= 1 + for: 10m + labels: + severity: warning + annotations: + summary: "Pod is crash looping" + description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff." + + - alert: PersistentVolumeClaimFillingUp + expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15 + for: 15m + labels: + severity: warning + annotations: + summary: "PVC has less than 15% free space" + description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} has less than 15% free space." + + - alert: CPUThrottlingHigh + expr: | + sum without (id, metrics_path, name, image, endpoint, job, node) ( + topk by (cluster, namespace, pod, container, instance) (1, + increase(container_cpu_cfs_throttled_periods_total{container!="", job="kubelet", metrics_path="/metrics/cadvisor", }[5m]) + ) + ) + / on (cluster, namespace, pod, container, instance) group_left + sum without (id, metrics_path, name, image, endpoint, job, node) ( + topk by (cluster, namespace, pod, container, instance) (1, + increase(container_cpu_cfs_periods_total{job="kubelet", metrics_path="/metrics/cadvisor", }[5m]) + ) + ) + > ( 50 / 100 ) + for: 30m + labels: + severity: info + annotations: + summary: "Processes experience elevated CPU throttling" + description: "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }} on cluster {{ $labels.cluster }}." + runbook_url: "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh" + + - alert: NodeMemoryPressure + expr: 100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 90 + for: 15m + labels: + severity: warning + annotations: + summary: "Node memory pressure" + description: "Node {{ $labels.instance }} has used more than 90% of memory for 15 minutes." diff --git a/prometheus-stack/k8s/grafana-values.yaml b/prometheus-stack/k8s/grafana-values.yaml index 6e7c8e1..7e7b14c 100644 --- a/prometheus-stack/k8s/grafana-values.yaml +++ b/prometheus-stack/k8s/grafana-values.yaml @@ -14,7 +14,8 @@ grafana: persistence: enabled: true - size: 10Gi + storageClassName: local-path-retain + size: 20Gi ingress: enabled: false @@ -24,10 +25,12 @@ grafana: prometheus: prometheusSpec: + retention: 60d + retentionSize: 32GB storageSpec: volumeClaimTemplate: spec: - storageClassName: "local-path" + storageClassName: "local-path-retain" accessModes: - ReadWriteOnce resources: @@ -41,12 +44,17 @@ prometheus: memory: "2Gi" alertmanager: alertmanagerSpec: + configSecret: alertmanager-config storage: volumeClaimTemplate: spec: + storageClassName: local-path-retain accessModes: - ReadWriteOnce - resources: requests: - storage: 20Gi + storage: 10Gi + +defaultRules: + disabled: + CPUThrottlingHigh: true diff --git a/prometheus-stack/k8s/traefik-alerts.yaml b/prometheus-stack/k8s/traefik-alerts.yaml new file mode 100644 index 0000000..8f15d92 --- /dev/null +++ b/prometheus-stack/k8s/traefik-alerts.yaml @@ -0,0 +1,60 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: traefik + namespace: prometheus + labels: + release: prometheus-stack +spec: + groups: + - name: traefik + rules: + - alert: TraefikDown + expr: absent(up{job="traefik"}) + for: 10m + labels: + severity: critical + annotations: + summary: "Traefik metrics are unavailable" + description: "Prometheus has no Traefik target." + + - alert: TraefikConfigReloadFailed + expr: traefik_config_last_reload_success == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Traefik configuration reload failed" + description: "Traefik failed to apply its last configuration reload. Check Traefik logs and the dynamic config sources." + + - alert: TraefikServiceHigh5xxRate + expr: | + sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) by (service) + / sum(rate(traefik_service_requests_total[5m])) by (service) * 100 > 5 + and sum(rate(traefik_service_requests_total[5m])) by (service) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "High 5xx error rate for service {{ $labels.service }}" + description: "Service {{ $labels.service }} is returning 5xx errors for more than 5% of requests over the last 5 minutes (current: {{ $value | humanizePercentage }})." + + - alert: TraefikServiceHighLatency + expr: | + histogram_quantile(0.95, + sum(rate(traefik_service_request_duration_seconds_bucket[5m])) by (le, service)) > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "High latency for service {{ $labels.service }}" + description: "P95 latency of {{ $labels.service }} exceeded 2 seconds over the last 5 minutes (current: {{ $value | humanizeDuration }})." + + - alert: TraefikCertExpiringSoon + expr: min(traefik_tls_certs_not_after) - time() < 14 * 24 * 60 * 60 + for: 10m + labels: + severity: warning + annotations: + summary: "Traefik TLS certificate expires soon" + description: "A TLS certificate managed by Traefik expires in less than 14 days (in {{ $value | humanizeDuration }})." \ No newline at end of file