diff --git a/.gitea/workflows/deploy-lib.sh b/.gitea/workflows/deploy-lib.sh index 80d2cc7..f178ec7 100644 --- a/.gitea/workflows/deploy-lib.sh +++ b/.gitea/workflows/deploy-lib.sh @@ -863,6 +863,32 @@ smoke_hosts() { | sort -u } +# Traefik's own list of the routes it actually built. The Kubernetes CRs are the +# wrong source for this: when a middleware fails to load, Traefik drops the +# router that referenced it and leaves the CR behind looking perfectly healthy. +# +# api.insecure is already on for the internal `traefik` entrypoint, but the pod +# IP is not routable from the node, so read it through kubectl exec rather than +# standing up a port-forward. HTTP only: the TCP routers match on HostSNI(`*`) +# and the UDP ones carry no rule at all, both selected by entrypoint and port, +# so neither can answer whether a given host has a route. +traefik_http_routes() { + kubectl -n traefik exec deploy/traefik -- \ + wget -qO- --timeout=10 http://127.0.0.1:8080/api/http/routers 2>/dev/null \ + | jq -c '[.[] | {status, rule: (.rule // "")}]' +} + +# The hosts Traefik currently routes to, one per line. Every backticked token of +# an enabled rule counts, which is a superset of the hosts -- PathPrefix values +# land here too, harmlessly -- but it keeps the host syntax in one place instead +# of a matcher per host. The scan keeps the delimiters, so strip them: what +# belongs in a comparison against a hostname is the bare name. +traefik_routed_hosts() { + jq -r '[.[] | select(.status == "enabled") | (.rule // "") + | scan("`[^`]+`") | ltrimstr("`") | rtrimstr("`")] + | unique | .[]' <<<"$1" +} + # stage_verify_k8s watches the rollout, which reports that the pods converged. # It cannot tell a converged pod from a serving one: a route pointing at the # wrong port, a Service selector that matches nothing the app listens on, a 500 @@ -874,6 +900,13 @@ smoke_hosts() { # or a 404 from a path the service does not serve still means the chain is # intact. Only a transport failure (no DNS, refused, timeout) or a 5xx means # the service is not serving, and only those fail the run. +# +# Except that a 404 is not evidence on its own. A router Traefik refused to +# build answers with the same 404 and nothing behind it, so a middleware that +# fails to load -- the crowdsec bouncer, which Traefik disables silently when +# it cannot fetch the plugin -- takes down every route that referenced it while +# this stage reports `ok` for all of them. No status code separates those two +# cases, so ask Traefik which routes it built and fail on the difference. stage_smoke() { cd "$REPO" select_manifests >/dev/null @@ -920,11 +953,52 @@ stage_smoke() { esac done + # Second gate. The probe above only means something if a router matched the + # host in the first place, so compare the hosts we expect against the hosts + # Traefik reports and fail on the difference. + local routes routed + if ! routes="$(traefik_http_routes)"; then + echo "ERROR: could not read Traefik's router list, refusing to report success" + return 1 + fi + routed="$(traefik_routed_hosts "$routes")" + + local -a unrouted=() + local tries=3 + while :; do + unrouted=() + for h in "${hosts[@]}"; do + grep -qxF "$h" <<<"$routed" || unrouted+=("$h") + done + if [ "${#unrouted[@]}" -eq 0 ]; then + break + fi + # A router mid-rollout is legitimately absent for a moment. A middleware + # that failed to load stays absent, so waiting cannot paper over it. + if [ "$tries" -le 1 ]; then + break + fi + tries=$((tries - 1)) + warn "${#unrouted[@]} host(s) have no enabled route yet, re-checking in 10s" + sleep 10 + if ! routes="$(traefik_http_routes)"; then + break + fi + routed="$(traefik_routed_hosts "$routes")" + done + + if [ "${#unrouted[@]}" -ne 0 ]; then + for h in "${unrouted[@]}"; do + echo " NO ROUTE $h (Traefik has no enabled router for this host)" + done + bad=1 + fi + if [ "$bad" -ne 0 ]; then echo "ERROR: at least one active service is not serving over its public route" return 1 fi - echo "all ${#hosts[@]} route(s) answered" + echo "all ${#hosts[@]} route(s) answered and have a router" } stage_apply_compose() {