feat(ingress): replace traefik crowdsec plugin with firewall bouncer

Move L3 enforcement to the host firewall-bouncer (systemd, nftables): drop the Traefik plugin, its secrets volume and the crowdsec Middleware, remove bouncer refs from all IngressRoutes. Disable the http-generic-bf scenario (403-burst bans hurt legit automation under L3 enforcement). Add a Gateway API PoC for homepages prod and CrowdSec PrometheusRule alerts.

.gitea/workflows/deploy-lib.sh

@@ -994,8 +994,7 @@ traefik_routed_hosts() {
 #
 # Except that a 404 is not evidence on its own. A router Traefik refused to
 # build answers with the same 404 and nothing behind it, so a middleware that
-# fails to load -- the crowdsec bouncer, which Traefik disables silently when
-# it cannot fetch the plugin -- takes down every route that referenced it while
+# fails to load takes down every route that referenced it while
 # this stage reports `ok` for all of them. No status code separates those two
 # cases, so ask Traefik which routes it built and fail on the difference.
 stage_smoke() {

adguardhome/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`dns.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: adguard-service
           port: 3000

authentik/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`auth.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: authentik-server-service
           port: 9000

checkmk/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`cmk.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: checkmk-service
           port: 5000

crowdsec/k8s/crowdsec-middleware.yaml

@@ -1,69 +0,0 @@
-# crowdsec/k8s is NOT managed by deploy.yaml - apply this by hand, and apply it
-# together with a restart:
-#   kubectl apply -f crowdsec/k8s/crowdsec-middleware.yaml
-#   kubectl -n traefik rollout restart deploy/traefik
-#
-# The restart is not optional. In stream mode the plugin runs a package-level
-# ticker goroutine (handleStreamTicker over the isCrowdsecStreamHealthy and
-# updateFailure globals) that no reconfiguration stops. Applying a change
-# wedges the instance: every route referencing it answers 404 and traefik logs
-# 'invalid middleware crowdsec-crowdsec-bouncer@kubernetescrd' until the pod is
-# replaced. Re-applying the previous config does NOT recover it, and the config
-# is not the cause - a valid CIDR cannot fail NewChecker, which is a plain
-# net.ParseCIDR. Only a new pod clears it. Measured cost: ~35s down for all
-# 20 hosts behind this middleware.
-apiVersion: traefik.io/v1alpha1
-kind: Middleware
-metadata:
-  name: crowdsec-bouncer
-  namespace: crowdsec
-spec:
-  plugin:
-    crowdsec-bouncer:
-      enabled: true
-      LogLevel: INFO
-      # `live` blocked on a `GET /v1/decisions` per request, so a burst
-      # saturated the LAPI and the plugin 403'd IPs that were never banned.
-      # v1.3.3 ignores UpdateMaxFailure in `live`, so fail-open is only
-      # reachable in stream mode, which polls into a cache instead - no
-      # per-request call to saturate. 15s rather than the 60s default: the
-      # deploy runner shares one public IP with the house, so this bounds
-      # both how late a ban lands and how long a lifted one lingers.
-      CrowdsecMode: stream
-      UpdateIntervalSeconds: 15
-      # -1 = never block because the LAPI is unreachable. In v1.3.3
-      # handleStreamTicker only clears isCrowdsecStreamHealthy when
-      # updateMaxFailure != -1, and ServeHTTP 403s once it is false, so this
-      # makes a CrowdSec outage mean "no protection", not "every site 403".
-      UpdateMaxFailure: -1
-      CrowdsecLapiScheme: http
-      CrowdsecLapiHost: crowdsec-service.crowdsec.svc.cluster.local:8080
-      CrowdsecLapiKeyFile: "/etc/traefik/secrets/traefik-api-key"
-      # Bypasses the bouncer and the decision cache, no LAPI round-trip.
-      # Keep in sync with forust/local-network in crowdsec-values.yaml.
-      ClientTrustedIPs:
-        - "127.0.0.0/8"
-        - "10.0.0.0/8"
-        - "172.16.0.0/12"
-        - "192.168.0.0/16"
-        - "100.64.0.0/10"
-        - "169.254.0.0/16"
-        - "fc00::/7"
-        - "fe80::/10"
-        # The mobile operator range from forust/mobile-whitelist, repeated
-        # deliberately rather than relying on the parser whitelist alone.
-        # That whitelist drops the event before it reaches a bucket, so no
-        # decision is ever created - but it is one config away from not
-        # firing, and the bouncer would then enforce a ban that was never
-        # justified. This is the last line: even a decision that exists for
-        # any reason is not served against the phone.
-        - "84.245.64.0/18"
-      # The name is HTTPTimeoutSeconds, an int in seconds (min 1) - there is
-      # no CrowdsecLapiTimeout, and an unrecognised key is silently dropped,
-      # which is how this sat at the 10s default. Nothing rides on it per
-      # request any more, so this only bounds the stream pull - and too low
-      # is the dangerous direction: the LAPI needs ~2s to answer
-      # /v1/decisions/stream, and a pull that times out leaves the ban cache
-      # frozen at its startup contents ("failed sending new decisions"),
-      # i.e. new bans silently never apply. Keep it above the pull latency.
-      HTTPTimeoutSeconds: 10

crowdsec/k8s/crowdsec-values.yaml

@@ -16,6 +16,12 @@ agent:
       value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
     - name: DISABLE_COLLECTIONS
       value: crowdsecurity/sshd
+    # Bans on 401/403 bursts hurt more than they protect: with L3 enforcement
+    # a false positive cuts the IP off everything (SSH included), and past
+    # incidents show legit automation (deploy runner, mesh peers, registry
+    # pulls) tripping this probe. Probing/XSS/SQLi/CVE scenarios stay.
+    - name: DISABLE_SCENARIOS
+      value: crowdsecurity/http-generic-bf
   metrics:
     enabled: true
     serviceMonitor:

dockmon/k8s/ingress.yaml

@@ -18,8 +18,6 @@ spec:
     - match: Host(`dockmon.forust.xyz`)
       kind: Rule
       middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
         - name: security-headers@file
       services:
         - name: dockmon-service

downtify/k8s/ingress.yaml

@@ -10,8 +10,6 @@ spec:
     - match: Host(`downtify.forust.xyz`)
       kind: Rule
       middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
         - name: security-chain@file
       services:
         - name: downtify-service

gitea/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`gitea.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: gitea-service
           port: 3000

headscale/k8s/routing/ingress.yaml

@@ -23,17 +23,11 @@ spec:
           port: 8080
     - match: Host(`hs.forust.xyz`) && PathPrefix(`/admin`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: headscale-ui-external
           port: 80
     - match: Host(`hs.forust.xyz`) && PathPrefix(`/metrics`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: headscale-server-external
           port: 9090
@@ -53,8 +47,6 @@ spec:
       kind: Rule
       middlewares:
         - name: headplane-prefix
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: headplane-external
           port: 3000

homepages/k8s/gateway.yaml

@@ -0,0 +1,31 @@
+# Gateway API PoC for homepages. Lives next to the TLS secrets so
+# certificateRefs stay same-namespace and no ReferenceGrant is needed.
+# Listener ports must match the Traefik entryPoints (80/443),
+# otherwise Traefik marks the listener Invalid.
+# Local .internal hosts are deliberately left on IngressRoute,
+# only prod is migrated here.
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: homepages
+  namespace: homepages
+spec:
+  gatewayClassName: traefik
+  listeners:
+    - name: http
+      protocol: HTTP
+      port: 80
+      allowedRoutes:
+        namespaces:
+          from: Same
+    - name: https
+      protocol: HTTPS
+      port: 443
+      tls:
+        mode: Terminate
+        certificateRefs:
+          - name: forust-homepage-prod-tls
+          - name: xdfnx-homepage-prod-tls
+      allowedRoutes:
+        namespaces:
+          from: Same

homepages/k8s/httproute.yaml

@@ -0,0 +1,69 @@
+# PoC: homepages prod hosts via Gateway API.
+# Runs alongside k8s/ingress.yaml - delete the prod IngressRoutes only after verification.
+# There are no local .internal hosts here, they stay on the local IngressRoute.
+# No per-route security middlewares: L3 enforcement moved to the host
+# firewall bouncer, so HTTPRoutes stay clean.
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: homepages-http-redirect
+  namespace: homepages
+spec:
+  parentRefs:
+    - name: homepages
+      kind: Gateway
+      sectionName: http
+  hostnames:
+    - forust.xyz
+    - www.forust.xyz
+    - xdfnx.cfd
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect:
+            scheme: https
+            statusCode: 301
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: forust-homepage-https
+  namespace: homepages
+spec:
+  parentRefs:
+    - name: homepages
+      kind: Gateway
+      sectionName: https
+  hostnames:
+    - forust.xyz
+    - www.forust.xyz
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: forust-homepage-service
+          port: 80
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: xdfnx-homepage-https
+  namespace: homepages
+spec:
+  parentRefs:
+    - name: homepages
+      kind: Gateway
+      sectionName: https
+  hostnames:
+    - xdfnx.cfd
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: xdfnx-homepage-service
+          port: 80

homepages/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`forust.xyz`) || Host(`www.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       priority: 10
       services:
         - name: forust-homepage-service
@@ -49,9 +46,6 @@ spec:
   routes:
     - match: Host(`xdfnx.cfd`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: xdfnx-homepage-service
           port: 80

immich/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`immich.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: immich-service
           port: 2283

kener/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`status.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: kener-service
           port: 3000

n8n/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`n8n.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: n8n-service
           port: 5678

netbird/k8s/ingress.yaml

@@ -32,9 +32,6 @@ spec:
     - match: Host(`nb.forust.xyz`)
       kind: Rule
       priority: 1
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: netbird-dashboard-service
           port: 80

netbox/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`netbox.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: netbox-service
           port: 8080

netronome/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`nm.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: netronome-service
           port: 7575

nextcloud/k8s/routing/ingress.yaml

@@ -12,8 +12,6 @@ spec:
       kind: Rule
       middlewares:
         - name: nextcloud-chain@file
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: nextcloud-apache
           port: 11000
@@ -32,8 +30,6 @@ spec:
     - match: Host(`nextcloud.workstation.internal`) || Host(`nextcloud.gigaforust.internal`)
       kind: Rule
       middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
         - name: nextcloud-chain@file
       services:
         - name: nextcloud-apache

portainer/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`portainer.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: portainer-service
           port: 9000

prometheus-stack/k8s/crowdsec-alerts.yaml

@@ -0,0 +1,41 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: crowdsec
+  namespace: prometheus
+  labels:
+    release: prometheus-stack
+spec:
+  groups:
+    - name: crowdsec
+      rules:
+        - alert: CrowdsecFirewallBouncerStale
+          expr: |
+            absent(cs_lapi_bouncer_requests_total{bouncer="firewall-workstation"})
+            or sum(rate(cs_lapi_bouncer_requests_total{bouncer="firewall-workstation"}[10m])) == 0
+          for: 15m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Firewall bouncer stopped pulling decisions"
+            description: "No LAPI pulls from firewall-workstation for 15 minutes. L3 enforcement is decaying: existing bans expire, new ones never land. Check the systemd unit on the node."
+
+        - alert: CrowdsecDecisionsSpike
+          expr: |
+            sum(cs_active_decisions) - sum(cs_active_decisions offset 30m) > 20
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Spike in active CrowdSec decisions"
+            description: "Active decisions grew by more than 20 in 30 minutes (current: {{ $value }}). Possible ban storm or self-ban - check cscli decisions list."
+
+        - alert: CrowdsecLAPIDecisionErrors
+          expr: |
+            sum(rate(cs_lapi_decisions_ko_total[5m])) > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CrowdSec LAPI decision errors"
+            description: "LAPI is failing decision lookups. Bouncers may be failing open. Check LAPI logs and DB health."

searxng/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`s.forust.xyz`) || Host(`search.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: searxng-service
           port: 8080

termix/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`termix.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: termix-service
           port: 8080

traefik/k8s/ingress.yaml

@@ -10,9 +10,6 @@ spec:
   routes:
     - match: Host(`traefik.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: api@internal
           kind: TraefikService

traefik/k8s/traefik-values.yaml

@@ -68,7 +68,7 @@ providers:
   kubernetesCRD:
     enabled: true
   kubernetesGateway:
-    enabled: false
+    enabled: true
   file:
     enabled: false
 
@@ -161,14 +161,13 @@ persistence:
   path: /data
 
 # No certificatesResolvers: public TLS comes from cert-manager.
+# No plugins: L3 enforcement moved to the host firewall bouncer,
+# nothing runs in the request path anymore.
 
 volumes:
   - name: traefik-dynamic
     mountPath: /etc/traefik/dynamic
     type: configMap
-  - name: crowdsec-bouncer-secrets
-    mountPath: /etc/traefik/secrets
-    type: secret
 additionalArguments:
   - "--providers.file.directory=/etc/traefik/dynamic"
   - "--providers.file.watch=true"
@@ -177,12 +176,6 @@ additionalArguments:
   - "--entryPoints.websecure.forwardedHeaders.trustedIPs=173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,104.16.0.0/13,104.24.0.0/14,172.64.0.0/13,131.0.72.0/22,192.168.1.1,192.168.1.0/24,192.168.88.0/24,192.168.88.1"
   - "--entryPoints.web.forwardedHeaders.trustedIPs=173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,104.16.0.0/13,104.24.0.0/14,172.64.0.0/13,131.0.72.0/22,192.168.1.1,192.168.1.0/24,192.168.88.0/24,192.168.88.1"
 
-experimental:
-  plugins:
-    crowdsec-bouncer:
-      moduleName: github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin
-      version: v1.3.3
-
 log:
   level: INFO
 accessLog:

uptime-kuma/k8s/ingress.yaml

@@ -9,9 +9,6 @@ spec:
   routes:
     - match: Host(`uptime.forust.xyz`)
       kind: Rule
-      middlewares:
-        - name: crowdsec-bouncer
-          namespace: crowdsec
       services:
         - name: uptime-kuma-service
           port: 3001