fix(alerts): measure traefik latency per route via exported_service

The service label the rule grouped by is the scrape target name, not the backend: with honorLabels=false the per-route value traefik emits is renamed to exported_service, so the old rule measured one global aggregate and both exclusions matched nothing. Group the latency and 5xx alerts by exported_service with exclusions in traefik's real namespace-routename-hash format. Already applied live with kubectl; this commit keeps git in sync.

prometheus-stack/k8s/traefik-alerts.yaml

@@ -29,26 +29,34 @@ spec:
 
         - alert: TraefikServiceHigh5xxRate
           expr: |
-            sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) by (service)
+            sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) by (exported_service)
-              / sum(rate(traefik_service_requests_total[5m])) by (service) * 100 > 5
+              / sum(rate(traefik_service_requests_total[5m])) by (exported_service) * 100 > 5
-              and sum(rate(traefik_service_requests_total[5m])) by (service) > 0
+              and sum(rate(traefik_service_requests_total[5m])) by (exported_service) > 0
           for: 5m
           labels:
             severity: warning
           annotations:
-            summary: "High 5xx error rate for service {{ $labels.service }}"
+            summary: "High 5xx error rate for service {{ $labels.exported_service }}"
-            description: "Service {{ $labels.service }} is returning 5xx errors for more than 5% of requests over the last 5 minutes (current: {{ $value | humanizePercentage }})."
+            description: "Service {{ $labels.exported_service }} is returning 5xx errors for more than 5% of requests over the last 5 minutes (current: {{ $value | humanizePercentage }})."
 
+        # NOTE on `exported_service`: traefik emits per-route series labelled
+        # `service`, but the prometheus scrape already carries a target label
+        # called `service` (the monitored Service object), and with the
+        # default honorLabels=false the scraped value loses the collision and
+        # is renamed to `exported_service`. Grouping by `service` therefore
+        # measures one global aggregate mislabelled per backend, and any
+        # exclusion written against it matches nothing. The generated names
+        # below are namespace-routename-hash as traefik builds them.
         - alert: TraefikServiceHighLatency
           expr: |
             histogram_quantile(0.95,
-              sum(rate(traefik_service_request_duration_seconds_bucket{service!~"xui-service|netbird-server-service"}[5m])) by (le, service)) > 2
+              sum(rate(traefik_service_request_duration_seconds_bucket{exported_service!~"netbird-netbird-(prod|local)-.*|xui-xray-(prod|local)-.*"}[5m])) by (le, exported_service)) > 2
           for: 5m
           labels:
             severity: warning
           annotations:
-            summary: "High latency for service {{ $labels.service }}"
+            summary: "High latency for service {{ $labels.exported_service }}"
-            description: "P95 latency of {{ $labels.service }} exceeded 2 seconds over the last 5 minutes (current: {{ $value | humanizeDuration }})."
+            description: "P95 latency of {{ $labels.exported_service }} exceeded 2 seconds over the last 5 minutes (current: {{ $value | humanizeDuration }})."
 
         - alert: TraefikCertExpiringSoon
           expr: min(traefik_tls_certs_not_after) - time() < 14 * 24 * 60 * 60