From a5409edbf280250ceda0579f9ea0a1ee1677a4da Mon Sep 17 00:00:00 2001 From: mr-forust Date: Mon, 28 Sep 2026 09:13:24 +0200 Subject: [PATCH] fix(deploy): fail the smoke stage when Traefik has no route for a host The smoke stage treats any HTTP response as proof the service is serving, which is right -- a 302 to a login or a 404 from a path the app does not serve still means the chain is intact. But a 404 is not evidence of that on its own: a router Traefik refused to build answers with exactly the same 404 and nothing behind it. That is not hypothetical. The crowdsec bouncer is a plugin, and when Traefik cannot fetch it at startup it disables the plugin without failing, then drops every router whose chain referenced it. Sixteen routes answered 404 and the stage printed `ok` for all sixteen, because a dropped router and an unserved path are indistinguishable from outside. The Kubernetes objects cannot tell us either: the IngressRoute is still sitting there looking healthy, the router Traefik built from it is simply not there. So ask Traefik. api.insecure is already on for the internal entrypoint and the router list says which hosts it matches right now. Every probed host has to appear in that list. HTTP routers only -- the TCP ones match on a HostSNI wildcard and the UDP ones carry no rule at all, both selected by entrypoint and port, so neither can answer the question. A router mid-rollout is legitimately absent for a moment, so the list is re-read twice over 20s; a plugin that failed to load stays absent and waiting cannot rescue it. An unreadable router list fails the stage rather than skipping the check, since a check that cannot run is not a passing check. Verified against the live cluster: all 23 public routes have a router and the stage passes. With gitea, grafana and uptime removed from that list the probes still answer and the stage fails on exactly those three. --- .gitea/workflows/deploy-lib.sh | 76 +++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/deploy-lib.sh b/.gitea/workflows/deploy-lib.sh index 80d2cc7..f178ec7 100644 --- a/.gitea/workflows/deploy-lib.sh +++ b/.gitea/workflows/deploy-lib.sh @@ -863,6 +863,32 @@ smoke_hosts() { | sort -u } +# Traefik's own list of the routes it actually built. The Kubernetes CRs are the +# wrong source for this: when a middleware fails to load, Traefik drops the +# router that referenced it and leaves the CR behind looking perfectly healthy. +# +# api.insecure is already on for the internal `traefik` entrypoint, but the pod +# IP is not routable from the node, so read it through kubectl exec rather than +# standing up a port-forward. HTTP only: the TCP routers match on HostSNI(`*`) +# and the UDP ones carry no rule at all, both selected by entrypoint and port, +# so neither can answer whether a given host has a route. +traefik_http_routes() { + kubectl -n traefik exec deploy/traefik -- \ + wget -qO- --timeout=10 http://127.0.0.1:8080/api/http/routers 2>/dev/null \ + | jq -c '[.[] | {status, rule: (.rule // "")}]' +} + +# The hosts Traefik currently routes to, one per line. Every backticked token of +# an enabled rule counts, which is a superset of the hosts -- PathPrefix values +# land here too, harmlessly -- but it keeps the host syntax in one place instead +# of a matcher per host. The scan keeps the delimiters, so strip them: what +# belongs in a comparison against a hostname is the bare name. +traefik_routed_hosts() { + jq -r '[.[] | select(.status == "enabled") | (.rule // "") + | scan("`[^`]+`") | ltrimstr("`") | rtrimstr("`")] + | unique | .[]' <<<"$1" +} + # stage_verify_k8s watches the rollout, which reports that the pods converged. # It cannot tell a converged pod from a serving one: a route pointing at the # wrong port, a Service selector that matches nothing the app listens on, a 500 @@ -874,6 +900,13 @@ smoke_hosts() { # or a 404 from a path the service does not serve still means the chain is # intact. Only a transport failure (no DNS, refused, timeout) or a 5xx means # the service is not serving, and only those fail the run. +# +# Except that a 404 is not evidence on its own. A router Traefik refused to +# build answers with the same 404 and nothing behind it, so a middleware that +# fails to load -- the crowdsec bouncer, which Traefik disables silently when +# it cannot fetch the plugin -- takes down every route that referenced it while +# this stage reports `ok` for all of them. No status code separates those two +# cases, so ask Traefik which routes it built and fail on the difference. stage_smoke() { cd "$REPO" select_manifests >/dev/null @@ -920,11 +953,52 @@ stage_smoke() { esac done + # Second gate. The probe above only means something if a router matched the + # host in the first place, so compare the hosts we expect against the hosts + # Traefik reports and fail on the difference. + local routes routed + if ! routes="$(traefik_http_routes)"; then + echo "ERROR: could not read Traefik's router list, refusing to report success" + return 1 + fi + routed="$(traefik_routed_hosts "$routes")" + + local -a unrouted=() + local tries=3 + while :; do + unrouted=() + for h in "${hosts[@]}"; do + grep -qxF "$h" <<<"$routed" || unrouted+=("$h") + done + if [ "${#unrouted[@]}" -eq 0 ]; then + break + fi + # A router mid-rollout is legitimately absent for a moment. A middleware + # that failed to load stays absent, so waiting cannot paper over it. + if [ "$tries" -le 1 ]; then + break + fi + tries=$((tries - 1)) + warn "${#unrouted[@]} host(s) have no enabled route yet, re-checking in 10s" + sleep 10 + if ! routes="$(traefik_http_routes)"; then + break + fi + routed="$(traefik_routed_hosts "$routes")" + done + + if [ "${#unrouted[@]}" -ne 0 ]; then + for h in "${unrouted[@]}"; do + echo " NO ROUTE $h (Traefik has no enabled router for this host)" + done + bad=1 + fi + if [ "$bad" -ne 0 ]; then echo "ERROR: at least one active service is not serving over its public route" return 1 fi - echo "all ${#hosts[@]} route(s) answered" + echo "all ${#hosts[@]} route(s) answered and have a router" } stage_apply_compose() {