diff --git a/prometheus-stack/k8s/traefik-alerts.yaml b/prometheus-stack/k8s/traefik-alerts.yaml index 384cbe6..9e5bf8c 100644 --- a/prometheus-stack/k8s/traefik-alerts.yaml +++ b/prometheus-stack/k8s/traefik-alerts.yaml @@ -29,26 +29,34 @@ spec: - alert: TraefikServiceHigh5xxRate expr: | - sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) by (service) - / sum(rate(traefik_service_requests_total[5m])) by (service) * 100 > 5 - and sum(rate(traefik_service_requests_total[5m])) by (service) > 0 + sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) by (exported_service) + / sum(rate(traefik_service_requests_total[5m])) by (exported_service) * 100 > 5 + and sum(rate(traefik_service_requests_total[5m])) by (exported_service) > 0 for: 5m labels: severity: warning annotations: - summary: "High 5xx error rate for service {{ $labels.service }}" - description: "Service {{ $labels.service }} is returning 5xx errors for more than 5% of requests over the last 5 minutes (current: {{ $value | humanizePercentage }})." + summary: "High 5xx error rate for service {{ $labels.exported_service }}" + description: "Service {{ $labels.exported_service }} is returning 5xx errors for more than 5% of requests over the last 5 minutes (current: {{ $value | humanizePercentage }})." + # NOTE on `exported_service`: traefik emits per-route series labelled + # `service`, but the prometheus scrape already carries a target label + # called `service` (the monitored Service object), and with the + # default honorLabels=false the scraped value loses the collision and + # is renamed to `exported_service`. Grouping by `service` therefore + # measures one global aggregate mislabelled per backend, and any + # exclusion written against it matches nothing. The generated names + # below are namespace-routename-hash as traefik builds them. - alert: TraefikServiceHighLatency expr: | histogram_quantile(0.95, - sum(rate(traefik_service_request_duration_seconds_bucket{service!~"xui-service|netbird-server-service"}[5m])) by (le, service)) > 2 + sum(rate(traefik_service_request_duration_seconds_bucket{exported_service!~"netbird-netbird-(prod|local)-.*|xui-xray-(prod|local)-.*"}[5m])) by (le, exported_service)) > 2 for: 5m labels: severity: warning annotations: - summary: "High latency for service {{ $labels.service }}" - description: "P95 latency of {{ $labels.service }} exceeded 2 seconds over the last 5 minutes (current: {{ $value | humanizeDuration }})." + summary: "High latency for service {{ $labels.exported_service }}" + description: "P95 latency of {{ $labels.exported_service }} exceeded 2 seconds over the last 5 minutes (current: {{ $value | humanizeDuration }})." - alert: TraefikCertExpiringSoon expr: min(traefik_tls_certs_not_after) - time() < 14 * 24 * 60 * 60