feat(monitoring): align storage to live pvc, add loki datasource and alerts

Match grafana/alertmanager storageClass to live PVCs to avoid immutable-field failures. Add Loki datasource and LokiDown/AlloyDown alerts.
This commit is contained in:
forust committed 2026-09-18 19:02:38 +02:00
1 parent 34c33697bb
commit c98ea8b957
3 files changed
+34 -6

No files matched your search

+18
View File
@@ -66,3 +66,21 @@ spec:
annotations:
summary: "Node memory pressure"
description: "Node {{ $labels.instance }} has used more than 90% of memory for 15 minutes."
- alert: LokiDown
expr: kube_statefulset_status_replicas_unavailable{statefulset="loki"} > 0 or kube_deployment_status_replicas_unavailable{deployment="loki-gateway"} > 0
for: 10m
labels:
severity: warning
annotations:
summary: "Loki is down"
description: "Loki in namespace {{ $labels.namespace }} has unavailable replicas for more than 10 minutes. Logs are not queryable."
- alert: AlloyDown
expr: kube_daemonset_status_number_unavailable{daemonset="alloy"} > 0
for: 10m
labels:
severity: warning
annotations:
summary: "Alloy is down"
description: "Alloy DaemonSet in namespace {{ $labels.namespace }} has {{ $value }} unavailable pods for more than 10 minutes. Pod logs are not being shipped to Loki."