diff --git a/prometheus-stack/k8s/alerts.yaml b/prometheus-stack/k8s/alerts.yaml index 04a7306..489f276 100644 --- a/prometheus-stack/k8s/alerts.yaml +++ b/prometheus-stack/k8s/alerts.yaml @@ -66,3 +66,21 @@ spec: annotations: summary: "Node memory pressure" description: "Node {{ $labels.instance }} has used more than 90% of memory for 15 minutes." + + - alert: LokiDown + expr: kube_statefulset_status_replicas_unavailable{statefulset="loki"} > 0 or kube_deployment_status_replicas_unavailable{deployment="loki-gateway"} > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Loki is down" + description: "Loki in namespace {{ $labels.namespace }} has unavailable replicas for more than 10 minutes. Logs are not queryable." + + - alert: AlloyDown + expr: kube_daemonset_status_number_unavailable{daemonset="alloy"} > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Alloy is down" + description: "Alloy DaemonSet in namespace {{ $labels.namespace }} has {{ $value }} unavailable pods for more than 10 minutes. Pod logs are not being shipped to Loki." diff --git a/prometheus-stack/k8s/grafana-values.yaml b/prometheus-stack/k8s/grafana-values.yaml index 7e7b14c..fe9d8ea 100644 --- a/prometheus-stack/k8s/grafana-values.yaml +++ b/prometheus-stack/k8s/grafana-values.yaml @@ -14,8 +14,11 @@ grafana: persistence: enabled: true - storageClassName: local-path-retain - size: 20Gi + # Matches live PVC (10Gi/local-path). Retain migration is a separate task + # with data migration (see storage-audit doc); do NOT change SC/size here + # without migrating, helm upgrade fails on immutable PVC fields. + storageClassName: local-path + size: 10Gi ingress: enabled: false @@ -23,6 +26,12 @@ grafana: service: port: 80 + additionalDataSources: + - name: Loki + type: loki + url: http://loki-gateway.prometheus.svc.cluster.local + access: proxy + prometheus: prometheusSpec: retention: 60d @@ -30,7 +39,8 @@ prometheus: storageSpec: volumeClaimTemplate: spec: - storageClassName: "local-path-retain" + # Matches live PVC, see note on grafana.persistence above. + storageClassName: "local-path" accessModes: - ReadWriteOnce resources: @@ -48,12 +58,13 @@ alertmanager: storage: volumeClaimTemplate: spec: - storageClassName: local-path-retain + # Matches live PVC (20Gi), see note on grafana.persistence above. + storageClassName: local-path accessModes: - ReadWriteOnce resources: requests: - storage: 10Gi + storage: 20Gi defaultRules: disabled: diff --git a/prometheus-stack/k8s/ingress.yaml b/prometheus-stack/k8s/ingress.yaml index 5f2da35..1c7b3cf 100644 --- a/prometheus-stack/k8s/ingress.yaml +++ b/prometheus-stack/k8s/ingress.yaml @@ -12,7 +12,6 @@ spec: middlewares: - name: crowdsec-bouncer namespace: crowdsec - - name: "security-chain@file" services: - name: prometheus-stack-grafana port: 80