feat(alerts): cover OOM kills, restart loops and evictions

The OOMKilled container behind the immich crash loop was invisible: PodCrashLooping only fires once kubelet has already given up and started the backoff.
This commit is contained in:
forust committed 2026-09-28 17:01:36 +02:00
1 parent f5b2f89f38
commit 74adf38d63
1 file changed
+27
+27
View File
@@ -27,6 +27,33 @@ spec:
summary: "Pod is crash looping" summary: "Pod is crash looping"
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff." description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff."
- alert: ContainerOOMKilled
expr: max_over_time(kube_pod_container_status_terminated_reason{reason="OOMKilled"}[15m]) >= 1
for: 5m
labels:
severity: warning
annotations:
summary: "Container was OOMKilled"
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} was killed for exceeding its memory limit. Raise the limit or reduce the workload."
- alert: ContainerRestartingTooOften
expr: max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total[30m])) > 3
for: 5m
labels:
severity: warning
annotations:
summary: "Container restarting too often"
description: "Container {{ $labels.container }} in {{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value }} times in the last 30 minutes."
- alert: PodEvicted
expr: max_over_time(kube_pod_status_reason{reason="Evicted"}[15m]) >= 1
for: 5m
labels:
severity: warning
annotations:
summary: "Pod was evicted"
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} was evicted, usually for node disk or memory pressure."
- alert: PersistentVolumeClaimFillingUp - alert: PersistentVolumeClaimFillingUp
expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15 expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15
for: 15m for: 15m