chore(deps): update container patch updates

Reviewed-on: https://gitea.forust.xyz/forust/homelab/pulls/59

.gitea/actionlint.yaml

@@ -0,0 +1,10 @@
+# actionlint configuration. Passed explicitly from the ci workflow:
+#   actionlint -config-file .gitea/actionlint.yaml .gitea/workflows/*.yaml
+#
+# The self-hosted act_runner registers custom labels that actionlint cannot know
+# about, so declare them here instead of silencing the whole runner-label check.
+self-hosted-runner:
+  labels:
+    - arch
+    - homelab
+    - prod

.gitea/workflows/ci.yaml

@@ -7,19 +7,115 @@ on:
   pull_request:
   workflow_dispatch:
 
+# Every job here is checkout plus local tools. The token needs to read the tree
+# and nothing else, and saying so keeps a future step that reaches for the API
+# from quietly holding a token that can write to the repository.
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
 env:
   REGISTRY: gcr.forust.xyz
 
 jobs:
-  lint-prettier:
+  lint-compose:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # Structure check for every committed Compose file, active or not.
+      # Interpolation, env-file and bind-mount resolution are all switched off,
+      # because inactive stacks have no .env here and would only fail on their
+      # ${VAR:?} guards. Active stacks get the full check with interpolation in
+      # the deploy workflow, where the real .env files live.
+      - name: Validate Compose files
+        shell: bash
+        run: |
+          set -euo pipefail
+          source .gitea/workflows/compose-lint.sh
+
+          mapfile -t safe_flags < <(compose_safe_flags)
+          echo "docker compose config ${safe_flags[*]-}"
+
+          mapfile -t files < <(compose_files)
+          if [ "${#files[@]}" -eq 0 ]; then
+            echo "No Compose files found."
+            exit 0
+          fi
+
+          failed=0
+          for f in "${files[@]}"; do
+            if ! out="$(validate_compose_file "$f" ${safe_flags[@]+"${safe_flags[@]}"} 2>&1)"; then
+              failed=1
+              echo "::error file=${f}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+
+          if [ "$failed" -ne 0 ]; then
+            echo "Compose validation failed."
+            exit 1
+          fi
+          echo "checked ${#files[@]} Compose file(s)"
+
+  lint-actionlint:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Lint Gitea Actions workflows with actionlint
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh actionlint)"
+          export PATH="$tools_dir:$PATH"
+          actionlint -config-file .gitea/actionlint.yaml -color .gitea/workflows/*.yaml
+
+  lint-shellcheck:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Lint shell scripts with ShellCheck
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh shellcheck)"
+          export PATH="$tools_dir:$PATH"
+          # userbot/ is a git subtree synced from forust/userbot, so its shell
+          # scripts are upstream's to maintain, not ours. Linting them would let a
+          # routine subtree pull turn the deploy gate red on code we do not own.
+          mapfile -t scripts < <(
+            git ls-files '*.sh' ':(glob)**/*.bash' ':!userbot/**'
+          )
+          if [ "${#scripts[@]}" -eq 0 ]; then
+            echo "No shell scripts found."
+            exit 0
+          fi
+          shellcheck --external-sources --source-path=SCRIPTDIR --severity=style "${scripts[@]}"
+
+  lint-prettier:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Check formatting with Prettier
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh prettier)"
+          export PATH="$tools_dir:$PATH"
+
           mapfile -t prettier_files < <(
             git ls-files \
               | grep -E '\.(md|json|ya?ml|html|css)$' \
@@ -31,51 +127,65 @@ jobs:
             exit 0
           fi
 
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            node:22-alpine \
-            sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
+          prettier --check --ignore-unknown "${prettier_files[@]}"
 
   lint-ruff:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Lint Python with Ruff
+      - name: Lint and format-check Python with Ruff
         shell: bash
         run: |
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/astral-sh/ruff:latest \
-            check .
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh ruff)"
+          export PATH="$tools_dir:$PATH"
+          ruff check .
+          ruff format --check .
 
   lint-yaml:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Lint YAML syntax
         shell: bash
         run: |
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            cytopia/yamllint:latest \
-            -c .yamllint .
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh yamllint)"
+          export PATH="$tools_dir:$PATH"
+
+          mapfile -t yaml_files < <(
+            git ls-files '*.yaml' '*.yml' \
+              ':!node_modules/**' \
+              ':!**/.venv/**'
+          )
+
+          if [ "${#yaml_files[@]}" -eq 0 ]; then
+            echo "No YAML files found."
+            exit 0
+          fi
+
+          yamllint -c .yamllint "${yaml_files[@]}"
 
   lint-dockerfiles:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
       - name: Lint Dockerfiles
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh hadolint)"
+          export PATH="$tools_dir:$PATH"
+
           mapfile -t dockerfiles < <(
             git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
           )
@@ -85,22 +195,171 @@ jobs:
             exit 0
           fi
 
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            --entrypoint hadolint \
-            hadolint/hadolint:latest-debian \
-            -c .hadolint.yaml "${dockerfiles[@]}"
+          hadolint -c .hadolint.yaml "${dockerfiles[@]}"
+
+  # Known, accepted, and recorded. Each line is a real advisory against a
+  # package we build into the panel image, kept in this workflow rather than in
+  # the package manifest so that a subtree sync from forust/userbot cannot
+  # silently widen the exemption.
+  #
+  # starlette is the reason this job is not simply "fail on everything":
+  # fastapi 0.115.12 pins `starlette<0.47.0`, and the fixes for the last four
+  # below need 0.49.1 through 1.3.1, so clearing them means a jump from fastapi
+  # 0.115.12 to 0.141.x. That is upstream's call, not a drive-by in a lint
+  # commit. Of the seven, four are reachable here in principle: 1942 is a
+  # crafted Range header hitting FileResponse, and the panel serves its built
+  # SPA through exactly that; 249 is request.form() ignoring max_fields for
+  # x-www-form-urlencoded, which is the login form; 1941 is a large multipart
+  # body blocking the event loop; 161 and 248 are unvalidated Host and request
+  # path reaching request.url. 2280 needs HTTPEndpoint, which the panel does
+  # not use, and 2281 is Windows-only, and this deploys on Linux.
+  #
+  # The panel answers on userbot.workstation.internal and has no public
+  # forust.xyz route, which is what keeps the four reachable ones from being
+  # an internet-facing DoS. It still manages Telegram credentials.
+  #
+  # Deleting an entry here is how you accept a new advisory, so the diff says
+  # so out loud.
+  scan-deps:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Audit the Python dependencies that ship in the image
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh pip-audit)"
+          export PATH="$tools_dir:$PATH"
+          # requirements.txt, not requirements-dev.txt: this is what the image
+          # installs, and the test tooling is not a shipped attack surface.
+          pip-audit -r userbot/panel/backend/requirements.txt --strict \
+            --ignore-vuln CVE-2025-67720 \
+            --ignore-vuln PYSEC-2026-161 \
+            --ignore-vuln PYSEC-2026-1941 \
+            --ignore-vuln PYSEC-2026-1942 \
+            --ignore-vuln PYSEC-2026-2280 \
+            --ignore-vuln PYSEC-2026-2281 \
+            --ignore-vuln PYSEC-2026-248 \
+            --ignore-vuln PYSEC-2026-249
+
+      # devDependencies are excluded on purpose. `npm audit` on the full tree
+      # reports 7 findings, and every one of them is a build- or test-time
+      # package: the esbuild CORS advisory needs a vite dev server serving to
+      # the internet, and nanoid's infinite loop needs a custom generator
+      # called with size 0, which postcss does not do. None of them are in the
+      # 91 kB bundle the panel serves. The one production finding, devalue
+      # via svelte, is moderate, which is where --audit-level draws the line;
+      # this fails on the next high or critical one.
+      - name: Audit the production npm dependencies
+        shell: bash
+        run: |
+          set -euo pipefail
+          # The pinned node, not whatever the runner has. Its system node is a
+          # rolling Arch package: during this very push its npm was missing
+          # entirely, and an hour later it was npm 12 on node 26. Both are the
+          # wrong major anyway — the panel image is node:22-alpine.
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh node)"
+          export PATH="$tools_dir:$PATH"
+          cd userbot/panel/frontend
+          npm ci
+          npm audit --omit=dev --audit-level=high
+
+  test-backend:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # 25 tests over the panel's pydantic models, its auth flow, the SPA
+      # fallback and the Kubernetes client it shells out with. They existed and
+      # had never been executed by anything.
+      #
+      # Note that userbot/ is a subtree synced from forust/userbot, so a routine
+      # sync can turn this red on upstream's code. Unlike the shellcheck job,
+      # which skips that tree because style disagreements there are ours to
+      # lose, a failing test here is a real defect in a service we deploy.
+      - name: Run the panel backend test suite
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh uv)"
+          export PATH="$tools_dir:$PATH"
+
+          # A venv in a temp dir rather than a checked-out one: the runner is
+          # shared, and a leftover .venv would let a dependency the
+          # requirements no longer pin still satisfy an import.
+          #
+          # --python is not optional. uv otherwise takes whatever interpreter it
+          # finds first, and which one that is depends on the machine: this
+          # runner runs jobs on the host, where the only interpreter is 3.14,
+          # and pyrogram's sync.py calls the bare asyncio.get_event_loop() that
+          # 3.14 no longer auto-creates, so three tests fail at collection. The
+          # image is python:3.13-slim, so 3.13 is also the version worth
+          # testing: uv fetches a managed build of it when the host has none,
+          # which is what makes this job independent of the runner.
+          venv="$(mktemp -d)/venv"
+          uv venv --python 3.13 --quiet "$venv"
+          uv pip install --quiet --python "$venv/bin/python" \
+            -r userbot/panel/backend/requirements-dev.txt
+
+          # `python -m`, not bare `pytest`: the tests import `app.*` relative to
+          # the backend directory, which only works if the cwd is on sys.path,
+          # and only `python -m` puts it there.
+          cd userbot/panel/backend
+          "$venv/bin/python" -m pytest tests/ -q
+
+  test-frontend:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 15
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # One `npm ci` for both checks below: it is by far the slowest part of
+      # this job, and a second one would learn nothing the first did not.
+      #
+      # `npm ci`, not `npm install`, for the same reason the Dockerfile uses it:
+      # the lockfile is what makes the tree that gets checked the tree that
+      # gets shipped.
+      - name: Type-check and test the panel frontend
+        shell: bash
+        run: |
+          set -euo pipefail
+          # The pinned node, not whatever the runner has. Its system node is a
+          # rolling Arch package: during this very push its npm was missing
+          # entirely, and an hour later it was npm 12 on node 26. Both are the
+          # wrong major anyway — the panel image is node:22-alpine.
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh node)"
+          export PATH="$tools_dir:$PATH"
+          cd userbot/panel/frontend
+          npm ci
+
+          # svelte-check has been a devDependency all along with no script
+          # pointing at it, so the type errors it reports had nowhere to
+          # surface. It is clean today, which is the only reason it can be a
+          # gate: it stops at whatever upstream introduces rather than
+          # reporting a backlog we inherited.
+          npm run check
+          npm test
 
   validate:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Validate Kubernetes manifests
+      - name: Validate Kubernetes manifests against JSON schemas
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh kubeconform)"
+          export PATH="$tools_dir:$PATH"
+
           mapfile -t manifests < <(
             git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
               | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
@@ -111,24 +370,126 @@ jobs:
             exit 0
           fi
 
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/yannh/kubeconform:latest \
+          kubeconform \
             -strict \
             -ignore-missing-schemas \
             -summary \
             "${manifests[@]}"
 
+      # kubeconform has no schemas for CRDs, so every IngressRoute, Certificate,
+      # PrometheusRule, Middleware, ServersTransport and ServiceMonitor is silently
+      # skipped above. The live API server knows the real CRD schemas (and runs the
+      # cert-manager / Traefik admission webhooks), so validate there too.
+      #
+      # Only services marked with a k8s/active marker are checked: server-side
+      # dry-run needs the target namespace to exist, and inactive services are not
+      # deployed. Services being enabled for the first time are still covered by
+      # the JSON-schema pass above.
+      #
+      # Main pushes only. `--dry-run=server` persists nothing, but it does execute
+      # the admission webhooks of the production API server, so anyone able to open
+      # a pull request would be able to run arbitrary manifest content through
+      # cert-manager and Traefik. A pull request has nothing to gain from it either:
+      # only main is ever deployed, and this job runs to completion before the
+      # deploy workflow is allowed to start, so a bad CRD is still caught before
+      # anything reaches the cluster -- just on the push rather than on the PR.
+      - name: Note the server-side check is not running here
+        if: github.event_name == 'pull_request' || github.ref != 'refs/heads/main'
+        shell: bash
+        run: |
+          echo "::notice::Skipping the server-side dry-run. It executes the cert-manager and" \
+            "Traefik admission webhooks against the production API server, so it is limited" \
+            "to pushes to main. CRDs are still schema-checked by kubeconform above, and the" \
+            "server-side pass still runs on main before the deploy."
+
+      - name: Validate active manifests against the live API server
+        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if ! kubectl get --raw='/readyz' --request-timeout=10s >/dev/null 2>&1; then
+            echo "::warning::Cluster unreachable — skipped server-side validation of CRDs (IngressRoute, Certificate, PrometheusRule). Review manifest changes manually."
+            exit 0
+          fi
+
+          mapfile -t k8s_dirs < <(
+            git ls-files '*.yaml' '*.yml' \
+              | grep -E '(^|/)k8s/' \
+              | sed -E 's#((^|.*/)k8s)/.*#\1#' \
+              | sort -u
+          )
+
+          manifests=()
+          kustomize_apps=()
+          for dir in "${k8s_dirs[@]}"; do
+            if [ ! -f "${dir}/active" ]; then
+              echo "skip (no k8s/active): ${dir}"
+              continue
+            fi
+            if [ -f "${dir}/overlays/prod/kustomization.yaml" ]; then
+              kustomize_apps+=("${dir}/overlays/prod")
+            elif [ -f "${dir}/base/kustomization.yaml" ]; then
+              kustomize_apps+=("${dir}/base")
+            else
+              while IFS= read -r f; do
+                [ -n "$f" ] && manifests+=("$f")
+              done < <(
+                git ls-files "${dir}/*.yaml" "${dir}/*.yml" \
+                  | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
+              )
+            fi
+          done
+
+          echo "server-side dry-run: ${#manifests[@]} manifests, ${#kustomize_apps[@]} kustomize apps"
+          failed=0
+          for m in ${manifests[@]+"${manifests[@]}"}; do
+            if ! out="$(kubectl apply --dry-run=server -f "$m" 2>&1)"; then
+              failed=1
+              echo "::error file=${m}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+          for k in ${kustomize_apps[@]+"${kustomize_apps[@]}"}; do
+            if ! out="$(kubectl apply -k "$k" --dry-run=server 2>&1)"; then
+              failed=1
+              echo "::error file=${k}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+
+          if [ "$failed" -ne 0 ]; then
+            echo "Server-side validation failed. The API server (or an admission webhook) rejected these manifests."
+            exit 1
+          fi
+          echo "server-side dry-run: all active manifests accepted by the API server"
+
   build:
-    needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
+    needs:
+      # scan-deps and the two test jobs were missing here, so a commit with a
+      # known-vulnerable dependency or a failing test still moved the :prod tag.
+      # The deploy was blocked either way - it requires the whole workflow to
+      # have succeeded - but the tag had already moved, and the next deploy to
+      # run resolved it. Publishing and passing the checks are the same gate.
+      [
+        lint-actionlint,
+        lint-shellcheck,
+        lint-compose,
+        lint-prettier,
+        lint-ruff,
+        lint-yaml,
+        lint-dockerfiles,
+        scan-deps,
+        test-backend,
+        test-frontend,
+        validate,
+      ]
     if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 60
     outputs:
       services: ${{ steps.services.outputs.services }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 0
 
@@ -136,12 +497,20 @@ jobs:
         id: services
         shell: bash
         run: |
+          set -euo pipefail
           base="${{ github.event.before }}"
           if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
             base="$(git rev-list --max-parents=0 HEAD)"
           fi
 
-          mapfile -t changed_files < <(git diff --name-only "$base" "${GITHUB_SHA}")
+          # A failed diff used to leave changed_files empty, which reads exactly
+          # like "nothing to build": the job went green having built nothing and
+          # the tag never moved. The status is checked, not assumed.
+          if ! changed="$(git diff --name-only "$base" "${GITHUB_SHA}")"; then
+            echo "::error::cannot diff ${base}..${GITHUB_SHA}"
+            exit 1
+          fi
+          mapfile -t changed_files <<<"$changed"
 
           services=()
 
@@ -191,69 +560,86 @@ jobs:
       - name: Log in to registry
         if: steps.services.outputs.services != ''
         shell: bash
+        # Through env, not by substitution into the script. A secret written
+        # into a run: block is pasted into the shell source before bash parses
+        # it, so a password containing a quote, a backtick or $(...) becomes
+        # code that runs. Masking the value in the log does not prevent that.
+        env:
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
         run: |
-          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-            -u "${{ secrets.REGISTRY_USERNAME }}" \
+          set -euo pipefail
+          printf '%s' "$REGISTRY_PASSWORD" | docker login "${REGISTRY}" \
+            -u "$REGISTRY_USERNAME" \
             --password-stdin
 
       - name: Build and push changed images
         if: steps.services.outputs.services != ''
         shell: bash
         run: |
+          # This step was the one run: block in the workflow without it, and it
+          # is the one that cannot afford it: a docker push that failed partway
+          # through the loop used to be followed by more pushes, the loop's exit
+          # status came from the last one, and the job went green with half the
+          # images missing from the registry.
+          set -euo pipefail
           IFS=, read -r -a services <<< "${{ steps.services.outputs.services }}"
 
+          # Tags for this push. The commit-pinned name is the point of this
+          # step: the deploy resolves it in preference to :prod, so a deploy
+          # that sat in the queue behind a later push still gets the build of
+          # the commit CI validated, instead of whatever :prod points at by the
+          # time it runs. See render_pinned in deploy-lib.sh.
+          commit_tag=""
+          if [ "${GITHUB_REF_NAME}" = "main" ]; then
+            commit_tag="sha-${GITHUB_SHA:0:12}"
+          fi
+
+          set_tags() {
+            tags=()
+            case "${GITHUB_REF_NAME}" in
+              main) tags+=("main" "prod") ;;
+              dev) tags+=("dev") ;;
+            esac
+            if [ -n "$commit_tag" ]; then
+              tags+=("$commit_tag")
+            fi
+          }
+
           for service in "${services[@]}"; do
             case "$service" in
               dtek_notif)
                 image="${REGISTRY}/forust/dtek-notif"
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
+                set_tags
                 build_args=()
                 for tag in "${tags[@]}"; do
                   build_args+=(-t "${image}:${tag}")
                 done
-                docker build "${build_args[@]}" dtek_notif
+                docker build \
+                  --cache-from "type=registry,ref=${image}:buildcache" \
+                  --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                  "${build_args[@]}" dtek_notif
                 for tag in "${tags[@]}"; do
                   docker push "${image}:${tag}"
                 done
                 ;;
               errorpages)
                 image="${REGISTRY}/forust/error-pages"
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
+                set_tags
                 build_args=()
                 for tag in "${tags[@]}"; do
                   build_args+=(-t "${image}:${tag}")
                 done
-                docker build "${build_args[@]}" errorpages
+                docker build \
+                  --cache-from "type=registry,ref=${image}:buildcache" \
+                  --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                  "${build_args[@]}" errorpages
                 for tag in "${tags[@]}"; do
                   docker push "${image}:${tag}"
                 done
                 ;;
               userbot)
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
+                set_tags
                 for target in runtime panel; do
                   case "$target" in
                     runtime)
@@ -269,15 +655,18 @@ jobs:
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
                   done
-                  docker build "${build_args[@]}" "$context"
+                  docker build \
+                    --cache-from "type=registry,ref=${image}:buildcache" \
+                    --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                    "${build_args[@]}" "$context"
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
                 done
                 ;;
               homepages)
-                for service in forust xdfnx; do
-                  case "$service" in
+                for variant in forust xdfnx; do
+                  case "$variant" in
                     forust)
                       image="${REGISTRY}/forust/forust-homepage"
                       ;;
@@ -285,28 +674,23 @@ jobs:
                       image="${REGISTRY}/forust/xdfnx-homepage"
                       ;;
                   esac
-                  tags=("latest")
-                  case "${GITHUB_REF_NAME}" in
-                    main)
-                      tags+=("main" "prod")
-                      ;;
-                    dev)
-                      tags+=("dev")
-                      ;;
-                  esac
+                  set_tags
                   build_args=()
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
                   done
-                  docker build "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
+                  docker build \
+                    --cache-from "type=registry,ref=${image}:buildcache" \
+                    --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                    "${build_args[@]}" -f "homepages/Dockerfile.${variant}" homepages
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
                 done
                 ;;
               edu_master)
-                for service in session-keeper webinar-checker; do
-                  case "$service" in
+                for variant in session-keeper webinar-checker; do
+                  case "$variant" in
                     session-keeper)
                       context="edu_master/phpsessid-bot"
                       image="${REGISTRY}/forust/session-keeper"
@@ -316,20 +700,15 @@ jobs:
                       image="${REGISTRY}/forust/webinar-checker"
                       ;;
                   esac
-                  tags=("latest")
-                  case "${GITHUB_REF_NAME}" in
-                    main)
-                      tags+=("main" "prod")
-                      ;;
-                    dev)
-                      tags+=("dev")
-                      ;;
-                  esac
+                  set_tags
                   build_args=()
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
                   done
-                  docker build "${build_args[@]}" "$context"
+                  docker build \
+                    --cache-from "type=registry,ref=${image}:buildcache" \
+                    --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
+                    "${build_args[@]}" "$context"
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
@@ -338,22 +717,41 @@ jobs:
             esac
           done
 
-  deploy-userbot-panel:
-    needs: build
-    if: github.ref_name == 'main' && contains(needs.build.outputs.services, 'userbot')
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Apply and roll out userbot panel
+      # Every image the tree names has to carry the commit-pinned name, not only
+      # the ones this push rebuilt. A push that touches nothing but manifests
+      # builds nothing, and its deploy would then find no commit-pinned tag to
+      # resolve and quietly fall back to the moving :prod - which is the whole
+      # failure the commit-pinned name exists to remove.
+      #
+      # Re-tagging copies the manifest list and transfers no layers, so pinning
+      # six images that already exist costs six registry writes.
+      #
+      # The list is derived from the tree rather than written out here, so an
+      # image added to a manifest is covered without a second place to update.
+      - name: Pin the commit name on the images this push did not rebuild
+        if: github.ref_name == 'main'
         shell: bash
         run: |
-          kubectl apply -f userbot/k8s/base/panel.yaml
-          kubectl get secret userbot-common-secrets -n default -o json \
-            | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
-            | kubectl apply -f -
-          # Keep legacy deployments (forust/anna) in sync with manifests; they have no replicas field, so apply leaves scaling to the user manager only.
-          kubectl apply -f userbot/k8s/base/userbots.yaml
-          kubectl rollout restart deployment/userbot-panel -n userbot
-          kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s
+          set -euo pipefail
+          commit_tag="sha-${GITHUB_SHA:0:12}"
+          mapfile -t repos < <(
+            git grep -hoE 'gcr\.forust\.xyz/forust/[A-Za-z0-9._-]+' -- '*.yaml' '*.yml' \
+              | sort -u
+          )
+          if [ "${#repos[@]}" -eq 0 ]; then
+            echo "No own images referenced by the tree."
+            exit 0
+          fi
+          echo "pinning ${#repos[@]} image(s) to $commit_tag"
+          for repo in "${repos[@]}"; do
+            if docker buildx imagetools inspect "$repo:$commit_tag" >/dev/null 2>&1; then
+              echo "  already built by this push: ${repo##*/}"
+              continue
+            fi
+            if ! docker buildx imagetools inspect "$repo:prod" >/dev/null 2>&1; then
+              echo "  WARNING: ${repo##*/} has no :prod to pin and no build produced it"
+              continue
+            fi
+            docker buildx imagetools create --tag "$repo:$commit_tag" "$repo:prod"
+            echo "  pinned ${repo##*/}"
+          done

.gitea/workflows/compose-lint.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Shared helpers for validating Compose files. Sourced both by steps in
+# .gitea/workflows/ci.yaml and by deploy-lib.sh on the workstation.
+#
+# Two levels of checking, matching how the repo is structured:
+#
+#   general  every committed Compose file, active or not. Pure structure check:
+#            no ${VAR} interpolation, no .env lookup, no bind-mount path
+#            resolution. Disabled stacks deliberately have no .env in the repo
+#            and no values on the CI runner, so a full `config` run would fail on
+#            their `${VAR:?}` guards for reasons that have nothing to do with the
+#            change under review.
+#
+#   full     active stacks only, with interpolation and env-file resolution, so
+#            required variables and referenced files are actually resolved. Needs
+#            the gitignored .env files, so this only runs in the deploy workflow
+#            on the workstation.
+#
+# This file is meant to be sourced, not executed.
+
+# All committed Compose files, including the ones deploy never starts.
+compose_files() {
+  git ls-files \
+    '*/compose.yaml' '*/compose.yml' 'compose.yaml' 'compose.yml' \
+    '*/docker-compose.yaml' '*/docker-compose.yml'
+}
+
+# Prints the flags that turn `docker compose config` into the general check.
+# Probed rather than hardcoded so an older Compose without --no-env-resolution
+# still gets the flags it does support.
+compose_safe_flags() {
+  local help flag
+  help="$(docker compose config --help 2>/dev/null || true)"
+  for flag in --no-interpolate --no-env-resolution --no-path-resolution; do
+    if printf '%s' "$help" | grep -q -- "$flag"; then
+      printf '%s\n' "$flag"
+    fi
+  done
+}
+
+# validate_compose_file <file> [extra docker compose config flags...]
+validate_compose_file() {
+  local file="$1"
+  shift
+  docker compose -f "$file" config --quiet "$@"
+}

.gitea/workflows/deploy.yaml

@@ -1,152 +1,192 @@
 name: deploy
 
 on:
+  # Deploy only what CI already validated. workflow_run is used instead of
+  # workflow_dispatch so a red lint/validate run can never reach the cluster.
+  workflow_run:
+    workflows: [ci]
+    types: [completed]
   workflow_dispatch:
 
+# The deploy jobs read the tree, then reach the cluster over SSH with the
+# deploy key. The Actions token itself is not part of that path, so it gets
+# read-only contents and no more.
+permissions:
+  contents: read
+
 concurrency:
   group: deploy-main
+  # Queue instead of cancelling. Cancelling a run kills the apply job mid-loop and
+  # takes the verify job down with it, so a superseded deploy would leave the
+  # cluster half-applied and unchecked — the exact failure the verify job exists
+  # to catch. kubectl apply and docker compose up are both idempotent, so letting
+  # the older run finish and then deploying the newer commit costs little.
   cancel-in-progress: false
 
+env:
+  DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
+  DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
+  DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
+  DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
+  DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+  APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
+  # workflow_run's own GITHUB_SHA points at the branch head, not at the commit the
+  # finished ci run checked. Pin the exact validated commit instead, so a push
+  # landing mid-deploy cannot make the workstation deploy something else. Also
+  # what the verify job checks the snapshot against. Empty for workflow_dispatch,
+  # which falls back to the current origin/main.
+  DEPLOY_SHA: ${{ github.event.workflow_run.head_sha }}
+
 jobs:
-  redeploy:
+  preflight:
+    if: >-
+      github.event_name != 'workflow_run' ||
+      (github.event.workflow_run.conclusion == 'success' &&
+       github.event.workflow_run.head_branch == 'main')
     runs-on: [self-hosted, linux, arch, homelab, prod]
+    timeout-minutes: 10
     steps:
-      - name: Redeploy workstation
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Fetch and reset workstation
         shell: bash
-        env:
-          DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
-          DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
-          DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
-          DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
-          DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-          # Set APPLY_PRUNE=true to enable kubectl apply --prune. Requires every
-          # manifest to carry label app.kubernetes.io/managed-by=homelab-deploy,
-          # otherwise previously applied resources get deleted on the next run.
-          APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
         run: |
           set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh preflight
 
-          : "${DEPLOY_HOST:?missing DEPLOY_HOST}"
-          : "${DEPLOY_USER:?missing DEPLOY_USER}"
-          : "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
+  validate:
+    needs: [preflight]
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-          deploy_port="${DEPLOY_PORT:-22}"
-          deploy_path="${DEPLOY_PATH:-/srv/homelab}"
-
-          ssh_key="$RUNNER_TEMP/deploy_key"
-          mkdir -p "$RUNNER_TEMP"
-          printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
-          chmod 600 "$ssh_key"
-
-          ssh_opts=(
-            -i "$ssh_key"
-            -p "$deploy_port"
-            -o BatchMode=yes
-            -o StrictHostKeyChecking=accept-new
-          )
-
-          ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
-            "DEPLOY_PATH=$(printf '%q' \"$deploy_path\") APPLY_PRUNE=$(printf '%q' \"${APPLY_PRUNE:-false}\") bash -se" <<'EOF'
+      - name: Dry-run manifests and check Secrets
+        shell: bash
+        run: |
           set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh validate
 
-          repo="${DEPLOY_PATH:-/srv/homelab}"
+  apply-k8s:
+    needs: [validate]
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    # Apply only, no verification, so this is just the work itself: snapshot,
+    # then sequential `helm upgrade --atomic --timeout 10m`, then the apply loop.
+    # Verification has its own job and its own budget.
+    #
+    # 45 is roughly four times the measured cost of the stage, which is
+    # deliberately not raised on a theory:
+    #
+    #   helm, healthy          3 no-op upgrades              ~3-5 min
+    #   helm, one release bad  --atomic spends its 10m,      ~10-15 min
+    #                            then rolls that one back
+    #   apply loop             ~40 manifests, 4 of which     ~1 min
+    #                           resolve an image digest
+    #   restart_stale_images   7.6s to find 8 workloads,     ~0.5 min
+    #                           9.8s to resolve their digests
+    #
+    # The helm figure is one release, not three: `set -e` aborts
+    # upgrade_helm_releases on the first failure, so a broken release costs
+    # 10m and the other two are never attempted. Multiplying 10m by three
+    # overstates the worst case by 20 minutes.
+    #
+    # The 45 minutes this was last raised to 45 were still not enough, and the
+    # job logs for those runs no longer exist, so what actually consumed the
+    # budget is not known - the two measurable candidates above account for
+    # ~15 of it. The one unbounded thing left in this stage is
+    # `docker manifest inspect` at deploy-lib.sh:236, which has no timeout
+    # against a registry with a known hang mode. Bound it, and make the stage
+    # announce what it is working on, before spending any of that on a larger
+    # ceiling: a stage that is killed with a diagnosable last line is a bug
+    # report, one that vanishes is not.
+    timeout-minutes: 45
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-          if [ ! -d "$repo/.git" ]; then
-            echo "Repository not found at $repo"
-            exit 1
-          fi
+      - name: Apply Kubernetes manifests
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh apply-k8s
 
-          git -C "$repo" fetch origin main
-          git -C "$repo" reset --hard origin/main
+  apply-compose:
+    needs: [validate]
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-          # Runtime selection: a service is k8s-managed when $SERVICE/k8s/active
-          # exists. Otherwise it is compose-managed, and only k8s/routing/*
-          # manifests (external Services / EndpointSlices / ServersTransport /
-          # Ingresses that route to docker backends) are applied.
-          #   migrate:   touch SERVICE/k8s/active   (+ move routing files up)
-          #   rollback:  rm SERVICE/k8s/active
-          collect_k8s() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              ! -path '*/routing/*' ! -path '*/overlays/*' \
-              ! -name 'kustomization.y*ml' ! -name '*.example.y*ml' \
-              ! -name '*values.y*ml' ! -name 'patch-*.y*ml' \
-              | sort
-          }
+      - name: Redeploy docker compose stacks
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh apply-compose
 
-          collect_k8s_inactive() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              \( -name 'namespace.y*ml' -o -path '*/routing/*' \) \
-              ! -path '*/overlays/*' ! -name '*.example.y*ml' \
-              | sort
-          }
+  # Watches the workloads this deploy changed and rolls back the ones that never
+  # became healthy. Runs even when the apply jobs failed, timed out or were
+  # cancelled — that is the whole point of splitting it out. `always()` is what
+  # lets it start after a failed dependency; the needs on apply-compose are a
+  # barrier, so verification begins only once both applies are done.
+  verify-k8s:
+    needs: [apply-k8s, apply-compose]
+    if: >-
+      always() &&
+      needs.apply-k8s.result != 'skipped' &&
+      needs.apply-compose.result != 'skipped'
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    # Not raised, because the arithmetic does not close.
+    #
+    # 32 workloads are under management and the wave width is 8, so the verify
+    # itself is 4 waves of ROLLOUT_TIMEOUT (300s) = 20 minutes worst case, when
+    # every rollout times out rather than converging. That is already 20 of 30.
+    #
+    # The other 10 would have to absorb rollback, and rollback_workloads is a
+    # serial `while read` loop at 300s per failed workload. 10 minutes buys two.
+    # Any larger number is buying a bigger multiple of an unbounded term rather
+    # than covering a known cost: 60 minutes buys eight, and 60 minutes is
+    # therefore not a bound, it is a guess with two digits.
+    #
+    # The number becomes derivable the moment rollback uses the same wave width
+    # as the verify: 32 failures then cost 4 waves = 20 minutes instead of 160,
+    # and 45 covers verify plus rollback at full width. That change is to the
+    # recovery path and is not folded into a timeout edit.
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-          mapfile -t compose_stacks < <(
-            find "$repo" -type f \( -name 'compose.yaml' -o -name 'compose.yml' \) | sort
-          )
+      - name: Verify workloads and roll back on failure
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh verify-k8s
 
-          mapfile -t k8s_manifests < <(
-            for kd in $(find "$repo" -type d -name k8s ! -path '*/.git/*' | sort); do
-              if [ -f "$kd/active" ]; then
-                collect_k8s "$kd"
-              else
-                collect_k8s_inactive "$kd"
-              fi
-            done
-          )
+  # Asks the public route of every active service whether it is actually
+  # serving, which the rollout check above structurally cannot: a pod can
+  # converge and still be crash-looping, or be listening on a port no Service
+  # points at, or answer 500.
+  #
+  # `always()` for the same reason verify-k8s has it, and it runs after that job
+  # specifically because a rollback is when a route most needs re-checking. The
+  # needs is a barrier, not a filter: whether verify-k8s passed, failed or was
+  # cancelled, the probes are what say whether the cluster is serving, and
+  # suppressing them on a rollback would hide the one run where the answer
+  # matters most.
+  smoke:
+    needs: [verify-k8s]
+    if: always() && needs.verify-k8s.result != 'skipped'
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-          echo "== Validate compose stacks =="
-          for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
-            echo "  config: $cf"
-            docker compose -f "$cf" config --quiet
-          done
-
-          echo "== Validate k8s manifests (kubectl dry-run) =="
-          for m in "${k8s_manifests[@]}"; do
-            echo "  apply --dry-run=client $m"
-            kubectl apply --dry-run=client -f "$m" >/dev/null
-          done
-
-          echo "== Applying Kubernetes manifests =="
-          ns_files=()
-          other_files=()
-          for m in "${k8s_manifests[@]}"; do
-            case "$m" in
-              */namespace.y?ml) ns_files+=("$m") ;;
-              *) other_files+=("$m") ;;
-            esac
-          done
-
-          prune_opts=()
-          if [ "${APPLY_PRUNE:-false}" = "true" ]; then
-            prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
-          fi
-
-          if [ "${#ns_files[@]}" -gt 0 ]; then
-            echo "  namespaces first: ${ns_files[*]}"
-            kubectl apply -f "${ns_files[@]}"
-          fi
-          if [ "${#other_files[@]}" -gt 0 ]; then
-            echo "  resources: ${other_files[*]}"
-            kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
-          fi
-
-          echo "== Redeploying docker compose stacks =="
-          for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
-            echo "  compose: $dir"
-            if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
-              docker compose -f "$cf" build
-              docker compose -f "$cf" push
-            fi
-            docker compose -f "$cf" up -d --pull always --remove-orphans
-          done
-          EOF
+      - name: Probe the public route of every active service
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh smoke

.gitea/workflows/install-ci-tools.sh

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+# Installs the pinned CI tools into "$TOOLS_DIR/bin" and echoes that directory
+# on stdout, so callers can do:
+#
+#   export PATH="$(bash .gitea/workflows/install-ci-tools.sh kubeconform shellcheck):$PATH"
+#
+# Versions come from tool-versions.env next to this script and are kept fresh by
+# Renovate. Re-running is cheap: an already-installed tool at the pinned version
+# is left alone.
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tool-versions.env
+. "$here/tool-versions.env"
+
+TOOLS_DIR="${TOOLS_DIR:-${RUNNER_TEMP:-/tmp}/homelab-tools}"
+BIN_DIR="$TOOLS_DIR/bin"
+mkdir -p "$BIN_DIR"
+
+arch="$(uname -m)"
+# Upstream projects disagree on arch spelling: kubeconform and actionlint use
+# Go names (amd64/arm64), shellcheck uses uname names (x86_64/aarch64), node
+# uses neither (x64/arm64), and hadolint mixes the two in a single release
+# (x86_64 but arm64).
+case "$arch" in
+  x86_64 | amd64)
+    goarch=amd64
+    sharch=x86_64
+    nodearch=x64
+    hadolintarch=x86_64
+    ;;
+  aarch64 | arm64)
+    goarch=arm64
+    sharch=aarch64
+    nodearch=arm64
+    hadolintarch=arm64
+    ;;
+  *)
+    echo "install-ci-tools: unsupported architecture: $arch" >&2
+    exit 1
+    ;;
+esac
+
+fetch() {
+  # fetch <url> <dest>
+  if command -v curl >/dev/null 2>&1; then
+    curl -sSLf --retry 3 -o "$2" "$1"
+  elif command -v wget >/dev/null 2>&1; then
+    wget -q -O "$2" "$1"
+  else
+    echo "install-ci-tools: neither curl nor wget is available" >&2
+    exit 1
+  fi
+}
+
+# installed_version <command>
+# Prints the version of an already-installed tool, or nothing. Each tool spells
+# its version flag differently, hence the case.
+installed_version() {
+  local out
+  case "$1" in
+    kubeconform) out="$("$1" -v 2>/dev/null | head -1 || true)" ;;
+    *) out="$("$1" --version 2>/dev/null | head -1 || true)" ;;
+  esac
+  printf '%s' "$out"
+}
+
+# at_version <command> <expected>
+at_version() {
+  case "$(installed_version "$1")" in
+    *"$2"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+install_kubeconform() {
+  if at_version kubeconform "v${KUBECONFORM_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${goarch}.tar.gz" \
+    "$tmp/kubeconform.tar.gz"
+  tar -xzf "$tmp/kubeconform.tar.gz" -C "$tmp" kubeconform
+  install -m 0755 "$tmp/kubeconform" "$BIN_DIR/kubeconform"
+  rm -rf "$tmp"
+}
+
+install_shellcheck() {
+  if at_version shellcheck "${SHELLCHECK_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.${sharch}.tar.xz" \
+    "$tmp/shellcheck.tar.xz"
+  tar -xJf "$tmp/shellcheck.tar.xz" -C "$tmp" --strip-components=1 "shellcheck-v${SHELLCHECK_VERSION}/shellcheck"
+  install -m 0755 "$tmp/shellcheck" "$BIN_DIR/shellcheck"
+  rm -rf "$tmp"
+}
+
+install_uv() {
+  if at_version uv "${UV_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  # uv release tags carry no leading v, unlike every other tool installed here.
+  fetch "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-${sharch}-unknown-linux-gnu.tar.gz" \
+    "$tmp/uv.tar.gz"
+  tar -xzf "$tmp/uv.tar.gz" -C "$tmp" --strip-components=1 "uv-${sharch}-unknown-linux-gnu/uv"
+  install -m 0755 "$tmp/uv" "$BIN_DIR/uv"
+  rm -rf "$tmp"
+}
+
+install_hadolint() {
+  if at_version hadolint "${HADOLINT_VERSION}"; then
+    return 0
+  fi
+  # A bare binary, no archive: hadolint ships one file per platform.
+  fetch "https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VERSION}/hadolint-linux-${hadolintarch}" \
+    "$BIN_DIR/hadolint"
+  chmod 0755 "$BIN_DIR/hadolint"
+}
+
+# ruff and yamllint both come from PyPI as wheels, which uv unpacks for us.
+install_uv_tool() {
+  # <package> <pinned version>
+  if at_version "$1" "$2"; then
+    return 0
+  fi
+  install_uv
+  UV_TOOL_BIN_DIR="$BIN_DIR" uv tool install --force "$1==$2" >/dev/null
+}
+
+install_ruff() {
+  install_uv_tool ruff "${RUFF_VERSION}"
+}
+
+install_yamllint() {
+  install_uv_tool yamllint "${YAMLLINT_VERSION}"
+}
+
+install_pip_audit() {
+  install_uv_tool pip-audit "${PIP_AUDIT_VERSION}"
+}
+
+install_prettier() {
+  if at_version prettier "${PRETTIER_VERSION}"; then
+    return 0
+  fi
+  # Not a standalone binary: prettier's entry point requires ../package.json
+  # relative to its own real path, so the package directory has to survive
+  # next to it. Hence a versioned directory plus a relative symlink, rather
+  # than copying the one file out as the other installers do.
+  local dir="$BIN_DIR/prettier-${PRETTIER_VERSION}"
+  if [ ! -f "$dir/package/package.json" ]; then
+    rm -rf "$dir"
+    mkdir -p "$dir"
+    fetch "https://registry.npmjs.org/prettier/-/prettier-${PRETTIER_VERSION}.tgz" "$dir/prettier.tgz"
+    tar -xzf "$dir/prettier.tgz" -C "$dir"
+    rm -f "$dir/prettier.tgz"
+    # npm strips the exec bit from bin/ on the way into the tarball.
+    chmod 0755 "$dir/package/bin/prettier.cjs"
+  fi
+  # Relative, so the whole tree stays valid if TOOLS_DIR is relocated.
+  ln -sfn "prettier-${PRETTIER_VERSION}/package/bin/prettier.cjs" "$BIN_DIR/prettier"
+}
+
+install_node() {
+  # npm gets checked by running it, not by looking it up: what matters is that
+  # it answers, so a stub, a half-removed Arch package or a name that resolves
+  # to something broken all have to read as "not installed". The runner's npm
+  # is a symlink into /usr/lib/node_modules/npm, which is exactly the kind of
+  # thing that disappears between runs.
+  if at_version node "v${NODE_VERSION}" && [ -n "$(installed_version npm)" ]; then
+    return 0
+  fi
+  # Same shape as prettier above: the tarball's bin/npm and bin/npx are links
+  # into lib/node_modules, so the whole tree has to survive next to them.
+  local dir="$BIN_DIR/node-${NODE_VERSION}"
+  if [ ! -x "$dir/bin/node" ]; then
+    rm -rf "$dir"
+    mkdir -p "$dir"
+    fetch "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${nodearch}.tar.xz" \
+      "$dir/node.tar.xz"
+    tar -xJf "$dir/node.tar.xz" -C "$dir" --strip-components=1 "node-v${NODE_VERSION}-linux-${nodearch}"
+    rm -f "$dir/node.tar.xz"
+  fi
+  # Relative, so the whole tree stays valid if TOOLS_DIR is relocated.
+  for bin in node npm npx; do
+    ln -sfn "node-${NODE_VERSION}/bin/${bin}" "$BIN_DIR/${bin}"
+  done
+}
+
+install_actionlint() {
+  if at_version actionlint "${ACTIONLINT_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_${goarch}.tar.gz" \
+    "$tmp/actionlint.tar.gz"
+  tar -xzf "$tmp/actionlint.tar.gz" -C "$tmp" actionlint
+  install -m 0755 "$tmp/actionlint" "$BIN_DIR/actionlint"
+  rm -rf "$tmp"
+}
+
+wanted=("$@")
+if [ "${#wanted[@]}" -eq 0 ]; then
+  wanted=(kubeconform shellcheck actionlint prettier ruff yamllint hadolint)
+fi
+
+for tool in "${wanted[@]}"; do
+  case "$tool" in
+    kubeconform) install_kubeconform ;;
+    shellcheck) install_shellcheck ;;
+    actionlint) install_actionlint ;;
+    prettier) install_prettier ;;
+    ruff) install_ruff ;;
+    yamllint) install_yamllint ;;
+    pip-audit) install_pip_audit ;;
+    hadolint) install_hadolint ;;
+    node) install_node ;;
+    uv) install_uv ;;
+    *)
+      echo "install-ci-tools: unknown tool: $tool" >&2
+      exit 1
+      ;;
+  esac
+done
+
+printf '%s\n' "$BIN_DIR"

.gitea/workflows/renovate-ci.yaml

@@ -0,0 +1,76 @@
+name: renovate-ci
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  validate-renovate:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # renovate/k8s/cronjob.yaml is the single source of truth for the image tag,
+      # so the same version that runs in the cluster is the one validated here.
+      - name: Resolve the deployed Renovate image
+        id: image
+        shell: bash
+        run: |
+          set -euo pipefail
+          image="$(sed -n 's|.*image:[[:space:]]*\(renovate/renovate:[^[:space:]]*\).*|\1|p' \
+            renovate/k8s/cronjob.yaml | head -1)"
+          if [ -z "$image" ]; then
+            echo "::error::no renovate/renovate image found in renovate/k8s/cronjob.yaml"
+            exit 1
+          fi
+          echo "using $image"
+          echo "image=$image" >> "$GITHUB_OUTPUT"
+
+      - name: Validate Renovate repository config
+        shell: bash
+        run: |
+          set -euo pipefail
+          docker run --rm \
+            -v "$PWD/renovate:/opt/renovate:ro" \
+            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
+            "${{ steps.image.outputs.image }}" \
+            renovate-config-validator /opt/renovate/renovate.json
+
+      # The CronJob cannot read the repository, so renovate/k8s/configmap.yaml
+      # carries an inlined copy of the config. Fail if it no longer matches.
+      - name: Check the generated Renovate ConfigMap
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./.gitea/workflows/sync-renovate-configmap.sh --check
+
+      - name: Validate Renovate Kubernetes manifests
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh kubeconform)"
+          export PATH="$tools_dir:$PATH"
+          kubeconform \
+            -strict \
+            -ignore-missing-schemas \
+            -summary \
+            renovate/k8s/namespace.yaml \
+            renovate/k8s/configmap.yaml \
+            renovate/k8s/cronjob.yaml
+
+      - name: Validate Renovate Compose file
+        shell: bash
+        run: |
+          set -euo pipefail
+          source .gitea/workflows/compose-lint.sh
+          mapfile -t safe_flags < <(compose_safe_flags)
+          validate_compose_file renovate/renovate-compose.yaml \
+            ${safe_flags[@]+"${safe_flags[@]}"}

.gitea/workflows/renovate-run.yaml

@@ -0,0 +1,92 @@
+name: renovate-run
+
+on:
+  workflow_dispatch:
+    inputs:
+      repositories:
+        description: "Repositories to scan (comma-separated)"
+        required: false
+        default: "forust/homelab"
+      log_level:
+        description: "Renovate log level"
+        required: false
+        default: "info"
+        type: choice
+        options:
+          - info
+          - debug
+      dry_run:
+        description: "Plan only, do not open or update PRs"
+        required: false
+        default: false
+        type: boolean
+
+# Renovate writes through its own bot PAT, passed in as RENOVATE_TOKEN, so the
+# Actions token is only ever used to read the checkout.
+permissions:
+  contents: read
+
+concurrency:
+  group: renovate-run
+  cancel-in-progress: false
+
+jobs:
+  run-renovate:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # renovate/k8s/cronjob.yaml is the single source of truth for the image tag.
+      # Reading it here means this workflow validates and runs the exact version
+      # that is deployed, instead of a copy that silently goes stale.
+      - name: Resolve the deployed Renovate image
+        id: image
+        shell: bash
+        run: |
+          set -euo pipefail
+          image="$(sed -n 's|.*image:[[:space:]]*\(renovate/renovate:[^[:space:]]*\).*|\1|p' \
+            renovate/k8s/cronjob.yaml | head -1)"
+          if [ -z "$image" ]; then
+            echo "::error::no renovate/renovate image found in renovate/k8s/cronjob.yaml"
+            exit 1
+          fi
+          echo "using $image"
+          echo "image=$image" >> "$GITHUB_OUTPUT"
+
+      - name: Validate Renovate config
+        shell: bash
+        run: |
+          set -euo pipefail
+          docker run --rm \
+            -v "$PWD/renovate/renovate.json:/opt/renovate/renovate.json:ro" \
+            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
+            "${{ steps.image.outputs.image }}" \
+            renovate-config-validator
+
+      - name: Run Renovate
+        shell: bash
+        env:
+          RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
+          RENOVATE_GITHUB_COM_TOKEN: ${{ secrets.RENOVATE_GITHUB_COM_TOKEN }}
+          RENOVATE_REPOSITORIES: ${{ inputs.repositories }}
+          RENOVATE_DRY_RUN: ${{ inputs.dry_run && 'full' || '' }}
+          LOG_LEVEL: ${{ inputs.log_level }}
+        run: |
+          set -euo pipefail
+
+          : "${RENOVATE_TOKEN:?missing RENOVATE_TOKEN secret — add a renovate-bot PAT in repo/org Actions secrets}"
+
+          docker run --rm \
+            -v "$PWD/renovate/renovate.json:/opt/renovate/renovate.json:ro" \
+            -e RENOVATE_PLATFORM=gitea \
+            -e RENOVATE_ENDPOINT=https://gitea.forust.xyz/api/v1 \
+            -e RENOVATE_TOKEN="$RENOVATE_TOKEN" \
+            -e RENOVATE_GITHUB_COM_TOKEN="${RENOVATE_GITHUB_COM_TOKEN:-}" \
+            -e RENOVATE_REPOSITORIES="${RENOVATE_REPOSITORIES:-forust/homelab}" \
+            -e RENOVATE_DRY_RUN="${RENOVATE_DRY_RUN:-}" \
+            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
+            -e RENOVATE_BASE_DIR=/tmp/renovate \
+            -e LOG_LEVEL="${LOG_LEVEL:-info}" \
+            "${{ steps.image.outputs.image }}"

.gitea/workflows/ssh-run.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# usage: ssh-run.sh <stage>
+# Runs one deploy-lib.sh stage on the workstation over SSH.
+set -euo pipefail
+
+: "${DEPLOY_HOST:?missing DEPLOY_HOST}"
+: "${DEPLOY_USER:?missing DEPLOY_USER}"
+: "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
+
+deploy_port="${DEPLOY_PORT:-22}"
+deploy_path="${DEPLOY_PATH:-/srv/homelab}"
+deploy_path="$(printf '%s' "$deploy_path" | tr -d '\"' | tr -d '\r' | xargs)"
+
+# The private key is written to a per-run directory that is removed on exit, so a
+# failed or cancelled job cannot leave deploy credentials in the runner's temp
+# directory. Do not use a fixed path: apply-k8s and apply-compose run in parallel.
+key_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/homelab-deploy-key.XXXXXXXX")"
+trap 'rm -rf "$key_dir"' EXIT INT TERM
+
+ssh_key="$key_dir/deploy_key"
+printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
+chmod 600 "$ssh_key"
+
+# A connection that died silently used to hang until the job timeout, and the
+# stage was never re-run: one flaky TCP session cost a whole 45-minute apply.
+# ServerAlive* bounds how long a dead peer goes unnoticed, ConnectTimeout bounds
+# setup. Only exit 255 - ssh's own transport failures - is retried. A stage that
+# fails on its own merits exits with the remote's status, so a real failure
+# still surfaces its own log instead of burning three attempts. The stages are
+# declarative applies, so re-running one that had already committed is harmless.
+ssh_opts=(
+  -i "$ssh_key" -p "$deploy_port"
+  -o BatchMode=yes -o StrictHostKeyChecking=accept-new
+  -o ConnectTimeout=15
+  -o ServerAliveInterval=15 -o ServerAliveCountMax=4
+)
+
+rc=0
+for attempt in 1 2 3; do
+  if [ "$attempt" -gt 1 ]; then
+    echo ":: warning::ssh transport failed, retrying (${attempt}/3)"
+    sleep $((attempt * 5))
+  fi
+  rc=0
+  ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
+    env "REPO=$deploy_path" "APPLY_PRUNE=${APPLY_PRUNE:-false}" \
+    "DEPLOY_SHA=${DEPLOY_SHA:-}" "DEPLOY_SNAPSHOT_DIR=${DEPLOY_SNAPSHOT_DIR:-}" \
+    "STAGE=$1" bash -se <<'EOF' || rc=$?
+source "$REPO/.gitea/workflows/deploy-lib.sh"
+run_stage "$STAGE"
+EOF
+  [ "$rc" -eq 0 ] && break
+  [ "$rc" -ne 255 ] && break
+done
+
+if [ "$rc" -ne 0 ]; then
+  echo ":: error::stage $1 failed over ssh (exit $rc)"
+fi
+exit "$rc"

.gitea/workflows/sync-renovate-configmap.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Regenerates renovate/k8s/configmap.yaml from renovate/renovate.json.
+#
+# renovate/renovate.json is the single source of truth: the CronJob, the Compose
+# file and the renovate-run workflow all mount that exact file. A ConfigMap cannot
+# read a file from the repository, so the same bytes are inlined here as a literal
+# block. This script keeps the copy honest:
+#
+#   .gitea/workflows/sync-renovate-configmap.sh          # rewrite in place
+#   .gitea/workflows/sync-renovate-configmap.sh --check  # fail if out of date
+#
+# renovate-ci runs the --check form on every PR and push, so a config change that
+# forgets to regenerate the ConfigMap cannot be merged.
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo="$(git -C "$here" rev-parse --show-toplevel)"
+
+src="$repo/renovate/renovate.json"
+dst="$repo/renovate/k8s/configmap.yaml"
+[ -f "$src" ] || {
+  echo "missing $src" >&2
+  exit 1
+}
+
+render() {
+  cat <<'HEADER'
+# GENERATED FILE - do not edit by hand.
+# Source: renovate/renovate.json
+# Regenerate: .gitea/workflows/sync-renovate-configmap.sh
+# Verify:     .gitea/workflows/sync-renovate-configmap.sh --check
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: renovate-config
+  namespace: renovate
+data:
+  renovate.json: |
+HEADER
+  sed 's/^/    /' "$src"
+}
+
+if [ "${1:-}" = "--check" ]; then
+  if ! diff -u "$dst" <(render) >/dev/null 2>&1; then
+    echo "ERROR: $dst is out of sync with renovate/renovate.json"
+    echo "Run: .gitea/workflows/sync-renovate-configmap.sh"
+    diff -u "$dst" <(render) || true
+    exit 1
+  fi
+  echo "renovate/k8s/configmap.yaml is in sync with renovate/renovate.json"
+  exit 0
+fi
+
+render >"$dst"
+echo "wrote $dst"

.gitea/workflows/tool-versions.env

@@ -0,0 +1,33 @@
+# Pinned versions of the CI tools installed by install-ci-tools.sh.
+# Renovate keeps these up to date (see customManagers in renovate/renovate.json).
+#
+# Every version here except NODE_VERSION matches what was already installed on
+# the runner, so pinning them changes what CI does not at all. It changes what
+# CI does when the runner is rebuilt with something else: today
+# install-ci-tools.sh finds the pinned version already on PATH and installs
+# nothing, and a runner that drifts gets the pinned one installed over it.
+#
+# The renovate image version is NOT pinned here: renovate/k8s/cronjob.yaml is the
+# single source of truth and the workflows read the tag from it, so there is
+# nothing to drift.
+ACTIONLINT_VERSION="1.7.7"
+SHELLCHECK_VERSION="0.11.0"
+KUBECONFORM_VERSION="0.8.0"
+PRETTIER_VERSION="3.8.1"
+RUFF_VERSION="0.16.8"
+YAMLLINT_VERSION="1.38.0"
+HADOLINT_VERSION="2.14.0"
+# pip-audit reads the advisory database over the network, so a floating version
+# would make the same commit report different things on different days. Pin it
+# like the rest: the advisories themselves are the moving part, not the tool.
+PIP_AUDIT_VERSION="2.10.1"
+# uv builds the throwaway venv the pytest job runs in, and unpacks the PyPI
+# wheels for ruff, yamllint and pip-audit.
+UV_VERSION="0.12.17"
+# node runs `npm ci` for the frontend tests and the npm audit, and it is the one
+# pin here that does NOT come from the runner: the runner's system node is a
+# rolling Arch package (it was node 26 with no npm at all when this was pinned),
+# and the panel image is node:22-alpine. Pinned to the image's major on purpose,
+# so the tree that gets tested is the tree that gets built. Renovate keeps this
+# in step with the Dockerfile's node: tag via the "node runtime" group.
+NODE_VERSION="22.23.3"

.github/workflows/ci.yaml

@@ -1,359 +0,0 @@
-name: ci
-
-on:
-  push:
-    branches:
-      - "**"
-  pull_request:
-  workflow_dispatch:
-
-env:
-  REGISTRY: gcr.forust.xyz
-
-jobs:
-  lint-prettier:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Check formatting with Prettier
-        shell: bash
-        run: |
-          mapfile -t prettier_files < <(
-            git ls-files \
-              | grep -E '\.(md|json|ya?ml|html|css)$' \
-              | grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
-          )
-
-          if [ "${#prettier_files[@]}" -eq 0 ]; then
-            echo "No Prettier-managed files found."
-            exit 0
-          fi
-
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            node:22-alpine \
-            sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
-
-  lint-ruff:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Lint Python with Ruff
-        shell: bash
-        run: |
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/astral-sh/ruff:latest \
-            check .
-
-  lint-yaml:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Lint YAML syntax
-        shell: bash
-        run: |
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            cytopia/yamllint:latest \
-            -c .yamllint .
-
-  lint-dockerfiles:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Lint Dockerfiles
-        shell: bash
-        run: |
-          mapfile -t dockerfiles < <(
-            git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
-          )
-
-          if [ "${#dockerfiles[@]}" -eq 0 ]; then
-            echo "No Dockerfiles found."
-            exit 0
-          fi
-
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            --entrypoint hadolint \
-            hadolint/hadolint:latest-debian \
-            -c .hadolint.yaml "${dockerfiles[@]}"
-
-  validate:
-    runs-on: [self-hosted, linux, arch, homelab]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Validate Kubernetes manifests
-        shell: bash
-        run: |
-          mapfile -t manifests < <(
-            git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
-              | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
-          )
-
-          if [ "${#manifests[@]}" -eq 0 ]; then
-            echo "No Kubernetes manifests found."
-            exit 0
-          fi
-
-          docker run --rm \
-            -v "$PWD:/work" \
-            -w /work \
-            ghcr.io/yannh/kubeconform:latest \
-            -strict \
-            -ignore-missing-schemas \
-            -summary \
-            "${manifests[@]}"
-
-  build:
-    needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
-    if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
-    runs-on: [self-hosted, linux, arch, homelab]
-    outputs:
-      services: ${{ steps.services.outputs.services }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Detect changed docker-built services
-        id: services
-        shell: bash
-        run: |
-          base="${{ github.event.before }}"
-          if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
-            base="$(git rev-list --max-parents=0 HEAD)"
-          fi
-
-          mapfile -t changed_files < <(git diff --name-only "$base" "${GITHUB_SHA}")
-
-          services=()
-
-          add_service() {
-            local name="$1"
-            local seen=0
-            for existing in "${services[@]}"; do
-              if [ "$existing" = "$name" ]; then
-                seen=1
-                break
-              fi
-            done
-            if [ "$seen" -eq 0 ]; then
-              services+=("$name")
-            fi
-          }
-
-          for file in "${changed_files[@]}"; do
-            case "$file" in
-              dtek_notif/*)
-                add_service dtek_notif
-                ;;
-              errorpages/*)
-                add_service errorpages
-                ;;
-              userbot/*)
-                add_service userbot
-                ;;
-              homepages/*)
-                add_service homepages
-                ;;
-              edu_master/phpsessid-bot/*|edu_master/webinar-checker/*|edu_master/compose.yaml)
-                add_service edu_master
-                ;;
-            esac
-          done
-
-          if [ "${#services[@]}" -eq 0 ]; then
-            echo "No docker-built services changed."
-            echo "services=" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-
-          printf '%s\n' "${services[@]}" | tee /tmp/services.txt
-          echo "services=$(paste -sd, /tmp/services.txt)" >> "$GITHUB_OUTPUT"
-
-      - name: Log in to registry
-        if: steps.services.outputs.services != ''
-        shell: bash
-        run: |
-          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
-            -u "${{ secrets.REGISTRY_USERNAME }}" \
-            --password-stdin
-
-      - name: Build and push changed images
-        if: steps.services.outputs.services != ''
-        shell: bash
-        run: |
-          IFS=, read -r -a services <<< "${{ steps.services.outputs.services }}"
-
-          for service in "${services[@]}"; do
-            case "$service" in
-              dtek_notif)
-                image="${REGISTRY}/forust/dtek-notif"
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
-                build_args=()
-                for tag in "${tags[@]}"; do
-                  build_args+=(-t "${image}:${tag}")
-                done
-                docker build "${build_args[@]}" dtek_notif
-                for tag in "${tags[@]}"; do
-                  docker push "${image}:${tag}"
-                done
-                ;;
-              errorpages)
-                image="${REGISTRY}/forust/error-pages"
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
-                build_args=()
-                for tag in "${tags[@]}"; do
-                  build_args+=(-t "${image}:${tag}")
-                done
-                docker build "${build_args[@]}" errorpages
-                for tag in "${tags[@]}"; do
-                  docker push "${image}:${tag}"
-                done
-                ;;
-              userbot)
-                tags=("latest")
-                case "${GITHUB_REF_NAME}" in
-                  main)
-                    tags+=("main" "prod")
-                    ;;
-                  dev)
-                    tags+=("dev")
-                    ;;
-                esac
-                for target in runtime panel; do
-                  case "$target" in
-                    runtime)
-                      context="userbot"
-                      image="${REGISTRY}/forust/userbot"
-                      ;;
-                    panel)
-                      context="userbot/panel"
-                      image="${REGISTRY}/forust/userbot-panel"
-                      ;;
-                  esac
-                  build_args=()
-                  for tag in "${tags[@]}"; do
-                    build_args+=(-t "${image}:${tag}")
-                  done
-                  docker build "${build_args[@]}" "$context"
-                  for tag in "${tags[@]}"; do
-                    docker push "${image}:${tag}"
-                  done
-                done
-                ;;
-              homepages)
-                for service in forust xdfnx; do
-                  case "$service" in
-                    forust)
-                      image="${REGISTRY}/forust/forust-homepage"
-                      ;;
-                    xdfnx)
-                      image="${REGISTRY}/forust/xdfnx-homepage"
-                      ;;
-                  esac
-                  tags=("latest")
-                  case "${GITHUB_REF_NAME}" in
-                    main)
-                      tags+=("main" "prod")
-                      ;;
-                    dev)
-                      tags+=("dev")
-                      ;;
-                  esac
-                  build_args=()
-                  for tag in "${tags[@]}"; do
-                    build_args+=(-t "${image}:${tag}")
-                  done
-                  docker build "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
-                  for tag in "${tags[@]}"; do
-                    docker push "${image}:${tag}"
-                  done
-                done
-                ;;
-              edu_master)
-                for service in session-keeper webinar-checker; do
-                  case "$service" in
-                    session-keeper)
-                      context="edu_master/phpsessid-bot"
-                      image="${REGISTRY}/forust/session-keeper"
-                      ;;
-                    webinar-checker)
-                      context="edu_master/webinar-checker"
-                      image="${REGISTRY}/forust/webinar-checker"
-                      ;;
-                  esac
-                  tags=("latest")
-                  case "${GITHUB_REF_NAME}" in
-                    main)
-                      tags+=("main" "prod")
-                      ;;
-                    dev)
-                      tags+=("dev")
-                      ;;
-                  esac
-                  build_args=()
-                  for tag in "${tags[@]}"; do
-                    build_args+=(-t "${image}:${tag}")
-                  done
-                  docker build "${build_args[@]}" "$context"
-                  for tag in "${tags[@]}"; do
-                    docker push "${image}:${tag}"
-                  done
-                done
-                ;;
-            esac
-          done
-
-  deploy-userbot-panel:
-    needs: build
-    if: github.ref_name == 'main' && contains(needs.build.outputs.services, 'userbot')
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Apply and roll out userbot panel
-        shell: bash
-        run: |
-          kubectl apply -f userbot/k8s/base/panel.yaml
-          kubectl get secret userbot-common-secrets -n default -o json \
-            | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
-            | kubectl apply -f -
-          # Keep legacy deployments (forust/anna) in sync with manifests; they have no replicas field, so apply leaves scaling to the user manager only.
-          kubectl apply -f userbot/k8s/base/userbots.yaml
-          kubectl rollout restart deployment/userbot-panel -n userbot
-          kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s

.github/workflows/deploy.yaml

@@ -1,152 +0,0 @@
-name: deploy
-
-on:
-  workflow_dispatch:
-
-concurrency:
-  group: deploy-main
-  cancel-in-progress: false
-
-jobs:
-  redeploy:
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    steps:
-      - name: Redeploy workstation
-        shell: bash
-        env:
-          DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
-          DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
-          DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
-          DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
-          DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-          # Set APPLY_PRUNE=true to enable kubectl apply --prune. Requires every
-          # manifest to carry label app.kubernetes.io/managed-by=homelab-deploy,
-          # otherwise previously applied resources get deleted on the next run.
-          APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
-        run: |
-          set -euo pipefail
-
-          : "${DEPLOY_HOST:?missing DEPLOY_HOST}"
-          : "${DEPLOY_USER:?missing DEPLOY_USER}"
-          : "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
-
-          deploy_port="${DEPLOY_PORT:-22}"
-          deploy_path="${DEPLOY_PATH:-/srv/homelab}"
-
-          ssh_key="$RUNNER_TEMP/deploy_key"
-          mkdir -p "$RUNNER_TEMP"
-          printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
-          chmod 600 "$ssh_key"
-
-          ssh_opts=(
-            -i "$ssh_key"
-            -p "$deploy_port"
-            -o BatchMode=yes
-            -o StrictHostKeyChecking=accept-new
-          )
-
-          ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
-            "DEPLOY_PATH=$(printf '%q' \"$deploy_path\") APPLY_PRUNE=$(printf '%q' \"${APPLY_PRUNE:-false}\") bash -se" <<'EOF'
-          set -euo pipefail
-
-          repo="${DEPLOY_PATH:-/srv/homelab}"
-
-          if [ ! -d "$repo/.git" ]; then
-            echo "Repository not found at $repo"
-            exit 1
-          fi
-
-          git -C "$repo" fetch origin main
-          git -C "$repo" reset --hard origin/main
-
-          # Runtime selection: a service is k8s-managed when $SERVICE/k8s/active
-          # exists. Otherwise it is compose-managed, and only k8s/routing/*
-          # manifests (external Services / EndpointSlices / ServersTransport /
-          # Ingresses that route to docker backends) are applied.
-          #   migrate:   touch SERVICE/k8s/active   (+ move routing files up)
-          #   rollback:  rm SERVICE/k8s/active
-          collect_k8s() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              ! -path '*/routing/*' ! -path '*/overlays/*' \
-              ! -name 'kustomization.y*ml' ! -name '*.example.y*ml' \
-              ! -name '*values.y*ml' ! -name 'patch-*.y*ml' \
-              | sort
-          }
-
-          collect_k8s_inactive() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              \( -name 'namespace.y*ml' -o -path '*/routing/*' \) \
-              ! -path '*/overlays/*' ! -name '*.example.y*ml' \
-              | sort
-          }
-
-          mapfile -t compose_stacks < <(
-            find "$repo" -type f \( -name 'compose.yaml' -o -name 'compose.yml' \) | sort
-          )
-
-          mapfile -t k8s_manifests < <(
-            for kd in $(find "$repo" -type d -name k8s ! -path '*/.git/*' | sort); do
-              if [ -f "$kd/active" ]; then
-                collect_k8s "$kd"
-              else
-                collect_k8s_inactive "$kd"
-              fi
-            done
-          )
-
-          echo "== Validate compose stacks =="
-          for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
-            echo "  config: $cf"
-            docker compose -f "$cf" config --quiet
-          done
-
-          echo "== Validate k8s manifests (kubectl dry-run) =="
-          for m in "${k8s_manifests[@]}"; do
-            echo "  apply --dry-run=client $m"
-            kubectl apply --dry-run=client -f "$m" >/dev/null
-          done
-
-          echo "== Applying Kubernetes manifests =="
-          ns_files=()
-          other_files=()
-          for m in "${k8s_manifests[@]}"; do
-            case "$m" in
-              */namespace.y?ml) ns_files+=("$m") ;;
-              *) other_files+=("$m") ;;
-            esac
-          done
-
-          prune_opts=()
-          if [ "${APPLY_PRUNE:-false}" = "true" ]; then
-            prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
-          fi
-
-          if [ "${#ns_files[@]}" -gt 0 ]; then
-            echo "  namespaces first: ${ns_files[*]}"
-            kubectl apply -f "${ns_files[@]}"
-          fi
-          if [ "${#other_files[@]}" -gt 0 ]; then
-            echo "  resources: ${other_files[*]}"
-            kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
-          fi
-
-          echo "== Redeploying docker compose stacks =="
-          for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
-            echo "  compose: $dir"
-            if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
-              docker compose -f "$cf" build
-              docker compose -f "$cf" push
-            fi
-            docker compose -f "$cf" up -d --pull always --remove-orphans
-          done
-          EOF

.gitignore

@@ -21,6 +21,9 @@ checkmk/checkmk/*
 downtify/Downtify_downloads
 headscale/config/*
 headscale/data/*
+# NetBird local hostnames and generated secrets
+netbird/.env
+netbird/secrets/
 searxng/core-config/*
 
 # Steaming services files
@@ -93,6 +96,8 @@ replacements.txt
 # Temp files
 edu_master/temp/
 temp/*
+# Local-only tooling scratch space (pinned CI tools, verification scripts)
+tmp/
 
 # Environment
 .env
@@ -104,6 +109,10 @@ temp/*
 # kubernetes
 */k8s/*secret*
 !*/k8s/*secret*.example
+**/k8s/*secret*
+!**/k8s/*secret*.example
+# Local-only tweaks, not for upstream
+prometheus-stack/k8s/grafana-values.yaml
 traefik/k8s/local-tls.yaml
 converters/k8s/config.yaml
 convertx/k8s/config.yaml

adguardhome/compose.yaml

@@ -1,6 +1,6 @@
 services:
   adguard:
-    image: adguard/adguardhome:latest
+    image: adguard/adguardhome:v0.107.79
     container_name: adguardhome
     restart: unless-stopped
     ports:

adguardhome/k8s/adguard.yaml

@@ -62,16 +62,18 @@ spec:
     metadata:
       labels:
         app: adguard
+      annotations:
+        reloader.stakater.com/auto: "true"
     spec:
       containers:
         - name: adguard
-          image: adguard/adguardhome:latest
+          image: adguard/adguardhome:v0.107.79
           resources:
             limits:
               memory: "1.5Gi"
               cpu: "300m"
             requests:
-              memory: "500Mi"
+              memory: "1Gi"
               cpu: "50m"
           ports:
             - containerPort: 3000

adguardhome/k8s/certificate.yaml

@@ -0,0 +1,12 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: adguard-certs
+  namespace: adguard
+spec:
+  secretName: adguard-certs
+  dnsNames:
+    - dns.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer

adguardhome/k8s/ingress.yaml

@@ -7,18 +7,21 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`adguard.forust.xyz`) || Host(`dns.forust.xyz`)
+    - match: Host(`dns.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: adguard-service
           port: 3000
-    - match: (Host(`adguard.forust.xyz`) || Host(`dns.forust.xyz`)) && PathPrefix(`/dns-query`)
+    - match: (Host(`dns.forust.xyz`)) && PathPrefix(`/dns-query`)
       kind: Rule
       services:
         - name: adguard-service
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: adguard-certs
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -39,3 +42,5 @@ spec:
       services:
         - name: adguard-service
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls

adguardhome/k8s/internal-certificate.yaml

@@ -0,0 +1,15 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: adguard
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

authentik/compose.yaml

@@ -1,6 +1,6 @@
 services:
   postgresql:
-    image: docker.io/library/postgres:15-alpine
+    image: docker.io/library/postgres:15.19-alpine
     restart: unless-stopped
     env_file:
       - .env

authentik/k8s/authentik.yaml

@@ -41,7 +41,7 @@ spec:
     spec:
       containers:
         - name: authentik-server
-          image: ghcr.io/goauthentik/server:2025.10.2
+          image: ghcr.io/goauthentik/server:2026.8.3
           args: ["server"]
           envFrom:
             - configMapRef:
@@ -52,7 +52,7 @@ spec:
             - containerPort: 9000
           resources:
             requests:
-              memory: "700Mi"
+              memory: "768Mi"
               cpu: "300m"
             limits:
               memory: "1.5Gi"
@@ -75,7 +75,7 @@ spec:
     spec:
       containers:
         - name: authentik-worker
-          image: ghcr.io/goauthentik/server:2025.10.2
+          image: ghcr.io/goauthentik/server:2026.8.3
           args: ["worker"]
           securityContext:
             runAsUser: 0
@@ -86,8 +86,8 @@ spec:
                 name: authentik-secrets
           resources:
             requests:
-              memory: "512Mi"
+              memory: "320Mi"
               cpu: "300m"
             limits:
-              memory: "1Gi"
+              memory: "768Mi"
               cpu: "700m"

authentik/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: authentik-prod-tls
+  namespace: authentik
+spec:
+  secretName: authentik-prod-tls
+  dnsNames:
+    - auth.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: authentik
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

authentik/k8s/config.yaml

@@ -6,6 +6,6 @@ metadata:
 data:
   AUTHENTIK_IMAGE: ghcr.io/goauthentik/server
   AUTHENTIK_TAG: "2025.10.2"
-  AUTHENTIK_POSTGRESQL__HOST: authentik-postgres-service
+  AUTHENTIK_POSTGRESQL__HOST: postgres.database.svc.cluster.local
   AUTHENTIK_POSTGRESQL__NAME: authentik
   AUTHENTIK_ERROR_REPORTING__ENABLED: "true"

authentik/k8s/ingress.yaml

@@ -9,11 +9,14 @@ spec:
   routes:
     - match: Host(`auth.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: authentik-server-service
           port: 9000
   tls:
-    certResolver: letsencrypt
+    secretName: authentik-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -29,3 +32,5 @@ spec:
       services:
         - name: authentik-server-service
           port: 9000
+  tls:
+    secretName: internal-wildcard-tls

authentik/k8s/postgresql.yaml

@@ -1,66 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: authentik-postgres-service
-  namespace: authentik
-spec:
-  clusterIP: None
-  selector:
-    app: authentik-postgres
-  ports:
-    - port: 5432
-      targetPort: 5432
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: authentik-postgres-statefulset
-  namespace: authentik
-spec:
-  selector:
-    matchLabels:
-      app: authentik-postgres
-  serviceName: authentik-postgres-service
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: authentik-postgres
-    spec:
-      containers:
-        - name: postgres
-          image: docker.io/library/postgres:15-alpine
-          env:
-            - name: POSTGRES_DB
-              value: authentik
-            - name: POSTGRES_USER
-              valueFrom:
-                secretKeyRef:
-                  name: authentik-secrets
-                  key: AUTHENTIK_POSTGRESQL__USER
-            - name: POSTGRES_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: authentik-secrets
-                  key: AUTHENTIK_POSTGRESQL__PASSWORD
-          ports:
-            - containerPort: 5432
-              name: postgres
-          volumeMounts:
-            - name: postgres-data
-              mountPath: /var/lib/postgresql/data
-          resources:
-            requests:
-              memory: "256Mi"
-              cpu: "200m"
-            limits:
-              memory: "1Gi"
-              cpu: "500m"
-  volumeClaimTemplates:
-    - metadata:
-        name: postgres-data
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 5Gi

cert-manager/k8s/cert-manager-values.yaml

@@ -0,0 +1,9 @@
+crds:
+  enabled: true
+prometheus:
+  servicemonitor:
+    enabled: true
+    interval: 60s
+    scrapeTimeout: 30s
+    labels:
+      release: prometheus-stack

cert-manager/k8s/clusterissuer.yaml

@@ -0,0 +1,29 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    email: bobrovod@national.shitposting.agency
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: letsencrypt-staging-account-key
+    solvers:
+      - http01:
+          ingress:
+            class: traefik
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    email: bobrovod@national.shitposting.agency
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: letsencrypt-prod-account-key
+    solvers:
+      - http01:
+          ingress:
+            class: traefik

cert-manager/k8s/homelab-internal-root-ca.crt

@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFjCCAv6gAwIBAgIUetKpTfEDOn2985FFMu6G26itT+wwDQYJKoZIhvcNAQEN
+BQAwIzEhMB8GA1UEAxMYaG9tZWxhYiBpbnRlcm5hbCByb290IENBMB4XDTI2MDky
+MzEyNDA0N1oXDTM2MDkyMDEyNDA0N1owIzEhMB8GA1UEAxMYaG9tZWxhYiBpbnRl
+cm5hbCByb290IENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvmNP
+ZCOoD8NtNuYJKVXBlTPjX7D7sJCSK5neH7ZbYV5+lmUlEErY8Mik7j37V5k5NfpF
+Ig85pOjP7RckTPz5V6ek3yaN40s4AL053sN5ZPauDVYjalaEHTgj5sEMqlLACQWI
+yZmJOZspZykae8dIpQnqCoFpRT4FurJ78v4a0ylnFVLMQn/lyCHedwTjkEdtYWYr
+ccJy8vQwqkzs/rWvEH1lDqZhennLOrmcCjfonG7D/pruMn4z+6E28p4+ejkRrI6x
+luak3KnpT1XMeHtgU21hiRGaMDBchHMFgAhnY1qosymKenXvfTZItwgjZbwa1hJI
+GAiDm+jQDKMjzRZ3rH6Xfc0auUcykNz73PpNu1NGm78nndXwCXcXn1LFKNQJ+r1U
+sJiyAmUZmXVn4aM4OMf2F38k7wTYIKg7nRGaUkNeKDlNkjA4HvgWw+jwO1KmdHQ/
+mOem1rosDWHRK01wg+Gga9mQCnhNhxglg3t/UeSic6uOaRsvaz4qkzHq8MbCujVz
+DpKQjqdikYOAXZOs4KlBLWrS7NaK4NzfSD02pBUErh54ruJfY/bWz9KyXzBD/lQZ
+VUTKyvUVB0bkVHEdf1jJmX3H4IZRQSF5JPqOBotW6bJI5fEGNBvj9Zxy4nm2WWGz
+yyP3uWsQz8U/Wdx9nXZLHInTZBsvgLYtKUAWA30CAwEAAaNCMEAwDgYDVR0PAQH/
+BAQDAgKkMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFEkKm2rxPaK6+O9WD80z
+BLC6F9QsMA0GCSqGSIb3DQEBDQUAA4ICAQAnFyHz97Umf5VIu+dKTJid7C73VugJ
+TIar/xJBs/4CxP+znBxhJjXygRoyIfzoVGWcB2ZSL//vL78Qlts79K/Imc9a4RFF
+wMvCxsRXAEQ4TpeWi3ophPNcs4rhsP+gQKQFtnyKP9519bqpfxp0bTqwOV2o18fn
+za7rlQViiEnNV58j7CVoM9+mJvVVfBEX1Km+GyJL9GadzbIQ7FxClVJZefCbft93
+zHVk9gDOw8ys1XGSR2OUCyCLinXO6mqS16CmBb2MAKXq/YyH7E0N8iotAPGtfA8V
+M/0ddy947rY0xCrtECfWwvGQpJS7NRv/Z9b2jCfXrI5LXmL2nfQRg0y9GE4Vjwr+
+WxtGU5jOeFt0jQ+xRzcgG0Op+qK3x55l5LSo2hOcOVYbiHxcHEJFgwNi1ADeBFwb
+q/HdysfURSOghqjIpMMAUabBp+DBUg2EUF7pIaUqbdqExFYcr9EYisEMiNsmKmN+
+8ZbcOeerFKDQj+t/R0bFXa7UBn2UWsjI8zlR74aa2kLDXwtyz/XlO/FlYm66eBFo
+2/eYUSeU+S4ej+wUAs/dvjF7f190/DUQGuwOTlLTahqWDztmhCk7qzbECu56CwKT
+E5Ect2P72UleYwdblkVOVd352AmiwEzdOaziIRrPh8uenEknH6JBYPO3mjk7cCg+
+GPGcNIBctjdXhg==
+-----END CERTIFICATE-----

cert-manager/k8s/internal-ca.yaml

@@ -0,0 +1,33 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: selfsigned
+spec:
+  selfSigned: {}
+---
+# Homelab internal root CA (10y). Install the .crt on clients (see below).
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-ca-root
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: homelab internal root CA
+  duration: 87600h
+  renewBefore: 7200h
+  secretName: internal-ca-root
+  privateKey:
+    algorithm: RSA
+    size: 4096
+  issuerRef:
+    name: selfsigned
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: internal-ca
+spec:
+  ca:
+    secretName: internal-ca-root

cert-manager/k8s/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager

cfddns/compose.yaml

@@ -1,6 +1,6 @@
 services:
   cloudflare-ddns:
-    image: timothyjmiller/cloudflare-ddns:latest
+    image: timothyjmiller/cloudflare-ddns:2.2.0
     container_name: cloudflare-ddns
     restart: unless-stopped
     security_opt:

cfddns/k8s/deployment.yaml

@@ -18,14 +18,14 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: cloudflare-ddns
-          image: timothyjmiller/cloudflare-ddns:latest
+          image: timothyjmiller/cloudflare-ddns:2.2.0
           imagePullPolicy: Always
           resources:
             requests:
-              memory: "20Mi"
+              memory: "32Mi"
               cpu: "30m"
             limits:
-              memory: "64Mi"
+              memory: "128Mi"
               cpu: "50m"
           envFrom:
             - secretRef:

checkmk/compose.yaml

@@ -1,6 +1,6 @@
 services:
   checkmk:
-    image: "checkmk/check-mk-raw:2.4.0-latest"
+    image: "checkmk/check-mk-raw:2.4.0-2026.09.14"
     container_name: "checkmk"
     restart: unless-stopped
     # ports:

checkmk/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: checkmk-prod-tls
+  namespace: checkmk
+spec:
+  secretName: checkmk-prod-tls
+  dnsNames:
+    - cmk.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: checkmk
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

checkmk/k8s/checkmk.yaml

@@ -31,7 +31,7 @@ spec:
     spec:
       containers:
         - name: checkmk
-          image: checkmk/check-mk-raw:2.4.0-latest
+          image: checkmk/check-mk-raw:2.4.0-2026.09.14
           envFrom:
             - secretRef:
                 name: checkmk-secrets

checkmk/k8s/ingress.yaml

@@ -9,11 +9,14 @@ spec:
   routes:
     - match: Host(`cmk.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: checkmk-service
           port: 5000
   tls:
-    certResolver: letsencrypt
+    secretName: checkmk-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRouteTCP
@@ -45,3 +48,5 @@ spec:
       services:
         - name: checkmk-service
           port: 5000
+  tls:
+    secretName: internal-wildcard-tls

cloudflared/k8s/.gitignore

@@ -0,0 +1 @@
+secret.yaml

cloudflared/k8s/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflared
+  labels:
+    app: cloudflared
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cloudflared
+  template:
+    metadata:
+      labels:
+        app: cloudflared
+    spec:
+      containers:
+        - name: cloudflared
+          image: cloudflare/cloudflared:2026.9.3
+          imagePullPolicy: IfNotPresent
+          args:
+            - tunnel
+            - --no-autoupdate
+            - run
+          env:
+            - name: TUNNEL_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: cloudflared-secrets
+                  key: TUNNEL_TOKEN
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "30m"
+            limits:
+              memory: "256Mi"
+              cpu: "200m"

cloudflared/k8s/secret.yaml.example

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloudflared-secrets
+type: Opaque
+stringData:
+  TUNNEL_TOKEN: your_tunnel_token_here

converters/compose.yaml

@@ -1,7 +1,7 @@
 services:
   convertx:
     container_name: convertx
-    image: ghcr.io/c4illin/convertx:latest
+    image: ghcr.io/c4illin/convertx:v0.18.0
     restart: unless-stopped
     ports:
       - "9992:3000"
@@ -42,7 +42,7 @@ services:
 
   bentopdf:
     container_name: bentopdf
-    image: bentopdf/bentopdf:latest
+    image: bentopdf/bentopdf@sha256:4eb4ec8f5030faf87c29a73d3d5a2781f28a597cf440c3ab111eb96aee550871
     restart: unless-stopped
     labels:
       - "traefik.enable=true"
@@ -54,7 +54,7 @@ services:
       - "traefik.http.routers.bentopdf.tls.certresolver=letsencrypt"
       - "traefik.http.routers.bentopdf.tls=true"
       # Local router
-      - "traefik.http.routers.bentopdf-local.rule=Host(`pdf.wokstation.internal`)"
+      - "traefik.http.routers.bentopdf-local.rule=Host(`pdf.workstation.internal`)"
       - "traefik.http.routers.bentopdf-local.entrypoints=websecure"
       - "traefik.http.routers.bentopdf-local.tls=true"
       # Dev router

converters/k8s/bentopdf.yaml

@@ -26,17 +26,18 @@ spec:
         app: bentopdf
     spec:
       containers:
-        - image: bentopdf/bentopdf:latest
+        - image: bentopdf/bentopdf@sha256:4eb4ec8f5030faf87c29a73d3d5a2781f28a597cf440c3ab111eb96aee550871
           imagePullPolicy: Always
           name: bentopdf
           ports:
             - containerPort: 8080
+          # p95 4M, max 11M over 7 days. Was 50Mi/700Mi.
           resources:
             requests:
-              memory: "50Mi"
+              memory: "32Mi"
               cpu: "50m"
               ephemeral-storage: "100Mi"
             limits:
-              memory: "700Mi"
+              memory: "128Mi"
               cpu: "700m"
               ephemeral-storage: "5Gi"

converters/k8s/certificates.yaml

@@ -0,0 +1,42 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: convertx-prod-tls
+  namespace: converters
+spec:
+  secretName: convertx-prod-tls
+  dnsNames:
+    - forust.xyz
+    - www.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: bentopdf-prod-tls
+  namespace: converters
+spec:
+  secretName: bentopdf-prod-tls
+  dnsNames:
+    - pdf.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: converters
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

converters/k8s/convertx.yaml

@@ -26,7 +26,7 @@ spec:
         app: convertx
     spec:
       containers:
-        - image: ghcr.io/c4illin/convertx:latest
+        - image: ghcr.io/c4illin/convertx:v0.18.0
           name: convertx
           envFrom:
             - configMapRef:
@@ -38,13 +38,14 @@ spec:
           volumeMounts:
             - mountPath: /data
               name: data
+          # p95 85M, max 136M over 7 days, spikes while converting. Was 250Mi/1.5Gi.
           resources:
             requests:
-              memory: "250Mi"
+              memory: "128Mi"
               cpu: "100m"
             limits:
               cpu: "1500m"
-              memory: "1.5Gi"
+              memory: "512Mi"
       volumes:
         - name: data
           persistentVolumeClaim:

converters/k8s/ingress.yaml

@@ -14,7 +14,7 @@ spec:
         - name: convertx-service
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: convertx-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -31,6 +31,9 @@ spec:
       services:
         - name: convertx-service
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -47,7 +50,7 @@ spec:
         - name: bentopdf-service
           port: 8080
   tls:
-    certResolver: letsencrypt
+    secretName: bentopdf-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -63,3 +66,5 @@ spec:
       services:
         - name: bentopdf-service
           port: 8080
+  tls:
+    secretName: internal-wildcard-tls

crowdsec/k8s/crowdsec-middleware.yaml

@@ -0,0 +1,69 @@
+# crowdsec/k8s is NOT managed by deploy.yaml - apply this by hand, and apply it
+# together with a restart:
+#   kubectl apply -f crowdsec/k8s/crowdsec-middleware.yaml
+#   kubectl -n traefik rollout restart deploy/traefik
+#
+# The restart is not optional. In stream mode the plugin runs a package-level
+# ticker goroutine (handleStreamTicker over the isCrowdsecStreamHealthy and
+# updateFailure globals) that no reconfiguration stops. Applying a change
+# wedges the instance: every route referencing it answers 404 and traefik logs
+# 'invalid middleware crowdsec-crowdsec-bouncer@kubernetescrd' until the pod is
+# replaced. Re-applying the previous config does NOT recover it, and the config
+# is not the cause - a valid CIDR cannot fail NewChecker, which is a plain
+# net.ParseCIDR. Only a new pod clears it. Measured cost: ~35s down for all
+# 20 hosts behind this middleware.
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: crowdsec-bouncer
+  namespace: crowdsec
+spec:
+  plugin:
+    crowdsec-bouncer:
+      enabled: true
+      LogLevel: INFO
+      # `live` blocked on a `GET /v1/decisions` per request, so a burst
+      # saturated the LAPI and the plugin 403'd IPs that were never banned.
+      # v1.3.3 ignores UpdateMaxFailure in `live`, so fail-open is only
+      # reachable in stream mode, which polls into a cache instead - no
+      # per-request call to saturate. 15s rather than the 60s default: the
+      # deploy runner shares one public IP with the house, so this bounds
+      # both how late a ban lands and how long a lifted one lingers.
+      CrowdsecMode: stream
+      UpdateIntervalSeconds: 15
+      # -1 = never block because the LAPI is unreachable. In v1.3.3
+      # handleStreamTicker only clears isCrowdsecStreamHealthy when
+      # updateMaxFailure != -1, and ServeHTTP 403s once it is false, so this
+      # makes a CrowdSec outage mean "no protection", not "every site 403".
+      UpdateMaxFailure: -1
+      CrowdsecLapiScheme: http
+      CrowdsecLapiHost: crowdsec-service.crowdsec.svc.cluster.local:8080
+      CrowdsecLapiKeyFile: "/etc/traefik/secrets/traefik-api-key"
+      # Bypasses the bouncer and the decision cache, no LAPI round-trip.
+      # Keep in sync with forust/local-network in crowdsec-values.yaml.
+      ClientTrustedIPs:
+        - "127.0.0.0/8"
+        - "10.0.0.0/8"
+        - "172.16.0.0/12"
+        - "192.168.0.0/16"
+        - "100.64.0.0/10"
+        - "169.254.0.0/16"
+        - "fc00::/7"
+        - "fe80::/10"
+        # The mobile operator range from forust/mobile-whitelist, repeated
+        # deliberately rather than relying on the parser whitelist alone.
+        # That whitelist drops the event before it reaches a bucket, so no
+        # decision is ever created - but it is one config away from not
+        # firing, and the bouncer would then enforce a ban that was never
+        # justified. This is the last line: even a decision that exists for
+        # any reason is not served against the phone.
+        - "84.245.64.0/18"
+      # The name is HTTPTimeoutSeconds, an int in seconds (min 1) - there is
+      # no CrowdsecLapiTimeout, and an unrecognised key is silently dropped,
+      # which is how this sat at the 10s default. Nothing rides on it per
+      # request any more, so this only bounds the stream pull - and too low
+      # is the dangerous direction: the LAPI needs ~2s to answer
+      # /v1/decisions/stream, and a pull that times out leaves the ban cache
+      # frozen at its startup contents ("failed sending new decisions"),
+      # i.e. new bans silently never apply. Keep it above the pull latency.
+      HTTPTimeoutSeconds: 10

crowdsec/k8s/crowdsec-values.yaml

@@ -0,0 +1,190 @@
+container_runtime: containerd
+
+agent:
+  acquisition: []
+  additionalAcquisition:
+    - labels:
+        type: traefik
+      limit: 1000
+      query: |
+        {namespace="traefik"}
+      source: loki
+      url: http://loki.prometheus.svc.cluster.local:3100/
+      wait_for_ready: 30s
+  env:
+    - name: COLLECTIONS
+      value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
+    - name: DISABLE_COLLECTIONS
+      value: crowdsecurity/sshd
+  metrics:
+    enabled: true
+    serviceMonitor:
+      additionalLabels:
+        release: prometheus-stack
+      enabled: true
+  # Static machine identity: agent pods mount pre-created LAPI credentials
+  # (Secret crowdsec-agent-credentials, key local_api_credentials.yaml)
+  # at the exact path the agent entrypoint expects. Together with the
+  # patched register-init (enforced by janitor-cronjob.yaml) the agent
+  # never calls `cscli lapi register` in steady state, so pod names,
+  # restarts and reboots can no longer break it.
+  extraVolumes:
+    - name: static-creds
+      secret:
+        secretName: crowdsec-agent-credentials
+        items:
+          - key: local_api_credentials.yaml
+            path: local_api_credentials.yaml
+  extraVolumeMounts:
+    - name: static-creds
+      mountPath: /tmp_config/local_api_credentials.yaml
+      subPath: local_api_credentials.yaml
+      readOnly: true
+  resources:
+    limits:
+      cpu: 200m
+      memory: 500Mi
+    requests:
+      cpu: 50m
+      memory: 100Mi
+
+config:
+  parsers:
+    s02-enrich:
+      mobile-whitelist.yaml: |
+        name: forust/mobile-whitelist
+        description: "Whitelist SWAN/4ka mobile network"
+        whitelist:
+          reason: "Mobile IP whitelist"
+          cidr:
+            - "84.245.64.0/18"
+      # CrowdSec's own guidance: CIDR allowlisting belongs at the parser stage.
+      # A parser whitelist discards the event before it reaches a bucket, so
+      # these addresses never produce an overflow and never become a decision.
+      # A postoverflow whitelist is checked only *after* the ban exists, and
+      # the bouncer answers 403 for as long as it does - which is a window we
+      # do not want the deploy sitting in.
+      local-network.yaml: |
+        name: forust/local-network
+        description: "Whitelist loopback, private and VPN networks"
+        whitelist:
+          reason: "Local network"
+          cidr:
+            - "127.0.0.0/8"
+            - "10.0.0.0/8"
+            - "172.16.0.0/12"
+            - "192.168.0.0/16"
+            # CGNAT range (RFC 6598). The workstation and the k0s node live
+            # here on WireGuard, and 100.64.0.0/10 is not covered by the
+            # RFC 1918 blocks above.
+            - "100.64.0.0/10"
+            - "169.254.0.0/16"
+            - "fc00::/7"
+            - "fe80::/10"
+
+  postoverflows:
+    s01-whitelist:
+      # The one whitelist that has to stay here: resolving a hostname is a
+      # network call, and the docs put expensive lookups in postoverflows on
+      # purpose - it runs only when a bucket actually overflows.
+      # ddns.forust.xyz is the public home address, not a private one, so
+      # forust/local-network does not cover it.
+      home-dynamic-ip.yaml: |
+        name: forust/home-dynamic-ip
+        description: "Whitelist home dynamic IP"
+        whitelist:
+          reason: "Home dynamic IP"
+          expression:
+            - evt.Overflow.Alert.Source.IP in LookupHost("ddns.forust.xyz")
+
+  # LAPI-only main config override, merged over config.yaml. NOTE: the
+  # chart's own default for this key is REPLACED, not merged, so its
+  # auto_registration block is repeated verbatim below - drop it and the
+  # agent can no longer register itself.
+  config.yaml.local: |
+    api:
+      server:
+        auto_registration: # Activate if not using TLS for authentication
+          enabled: true
+          token: "${REGISTRATION_TOKEN}" # /!\ Do not modify this variable (auto-generated and handled by the chart)
+          allowed_ranges: # /!\ Make sure to adapt to the pod IP ranges used by your cluster
+            - "127.0.0.1/32"
+            - "192.168.0.0/16"
+            - "10.0.0.0/8"
+            - "172.16.0.0/12"
+        # This homelab has no egress to console.crowdsec.cloud: DNS does
+        # not resolve. The LAPI kept trying anyway ("Signal push: N
+        # signals to push", "capi metrics: sending" every 10s) and each
+        # attempt sat on a resolver timeout WHILE HOLDING A WRITE
+        # TRANSACTION, which is what kept stalling per-request decision
+        # lookups even with WAL enabled. Nothing to share and nothing to
+        # pull - turn the Central API off instead of letting it block the
+        # only database writer we have.
+        online_client:
+          sharing: false
+          pull:
+            community: false
+            blocklists: false
+        disable_usage_metrics_export: true
+    db_config:
+      # SQLite without WAL serialises every reader behind the writer's
+      # rollback journal, and the LAPI writes constantly: the agent pushes
+      # Traefik alerts read from Loki, the metrics collector counts
+      # decisions, the bouncer touches "last pull" on every request.
+      # Symptom: decision lookups taking 10-30s (and a second connection
+      # that could not even open the database) while the LAPI sat at 28m
+      # CPU - the process was blocked in fsync, not computing. Every
+      # bouncer-protected request then blew through the plugin timeout and
+      # fail-closed with 403, on every site at once.
+      # The PVC is local-path-retain (hostPath), not a network share, so
+      # WAL is safe here; the crowdsec docs recommend it for exactly this
+      # ("allowing more concurrency in SQLite that will improve
+      # performances in most scenarios").
+      use_wal: true
+      # Keeps the alert table bounded. At the 5000/7d default the file
+      # reached 54MB in 15 days off the Traefik access log alone, and the
+      # metrics collector counts decisions on a timer; a smaller working
+      # set means fewer full scans. Crowdsec only prunes - SQLite never
+      # shrinks the file, so the size stays until a manual VACUUM.
+      flush:
+        max_items: 1000
+        max_age: 24h
+
+lapi:
+  env:
+    - name: COLLECTIONS
+      value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
+    - name: DISABLE_COLLECTIONS
+      value: crowdsecurity/linux crowdsecurity/sshd
+  metrics:
+    enabled: true
+    serviceMonitor:
+      additionalLabels:
+        release: prometheus-stack
+      enabled: true
+  persistentVolume:
+    config:
+      enabled: true
+      size: 100Mi
+      storageClassName: local-path-retain
+    data:
+      enabled: true
+      size: 1Gi
+      storageClassName: local-path-retain
+  # LAPI answers a blocking /v1/decisions lookup for EVERY bouncer-protected
+  # request (whole Traefik front door), so it is the hot path of the proxy.
+  # At 400m/500Mi it went CPU-throttled and idle lookups measured 1.3-7.4s,
+  # which pushed requests into the bouncer's fail-closed 403.
+  # Single replica on purpose: LAPI is stateful (BoltDB on the `data` PVC,
+  # credentials on the `config` PVC) - two replicas sharing those RWO
+  # volumes would corrupt the decision store. Scale up CPU, not replicas.
+  resources:
+    limits:
+      cpu: 1500m
+      memory: 1Gi
+    requests:
+      cpu: 250m
+      memory: 500Mi
+  service:
+    type: ClusterIP
+  storeLAPICscliCredentialsInSecret: true

crowdsec/k8s/grafana-dashboards.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+data:
+  crowdsec-overview.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph (old)\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"stat\",\n      \"name\": \"Stat\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"timeseries\",\n      \"name\": \"Time series\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 24,\n      \"panels\": [],\n      \"title\": \"Summary\",\n      \"type\": \"row\"\n    },\n    {\n      \"cacheTimeout\": null,\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [\n            {\n              \"options\": {\n                \"match\": \"null\",\n                \"result\": {\n                  \"text\": \"N/A\"\n                }\n              },\n              \"type\": \"special\"\n            }\n          ],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"#E02F44\",\n                \"value\": null\n              },\n              {\n                \"color\": \"#E02F44\",\n                \"value\": 10\n              },\n              {\n                \"color\": \"#299c46\",\n                \"value\": 10\n              }\n            ]\n          },\n          \"unit\": \"none\"\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 6,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2,\n      \"interval\": null,\n      \"links\": [],\n      \"maxDataPoints\": 100,\n      \"options\": {\n        \"colorMode\": \"background\",\n        \"graphMode\": \"none\",\n        \"justifyMode\": \"auto\",\n        \"orientation\": \"horizontal\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"text\": {},\n        \"textMode\": \"auto\"\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"targets\": [\n        {\n          \"exemplar\": true,\n          \"expr\": \"count(cs_info)\",\n          \"interval\": \"\",\n          \"legendFormat\": \"\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"timeFrom\": null,\n      \"timeShift\": null,\n      \"title\": \"Running Crowdsec\",\n      \"transparent\": true,\n      \"type\": \"stat\"\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"decimals\": 1,\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"links\": []\n        },\n        \"overrides\": []\n      },\n      \"fill\": 1,\n      \"fillGradient\": 0,\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 18,\n        \"x\": 6,\n        \"y\": 1\n      },\n      \"hiddenSeries\": false,\n      \"id\": 8,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": false,\n        \"max\": false,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sort\": \"total\",\n        \"sortDesc\": true,\n        \"total\": true,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"nullPointMode\": \"null\",\n      \"options\": {\n        \"alertThreshold\": true\n      },\n      \"percentage\": false,\n      \"pluginVersion\": \"8.1.2\",\n      \"pointradius\": 2,\n      \"points\": false,\n      \"renLine truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: "1"
+  name: crowdsec-crowdsec-overview
+  namespace: prometheus
+---
+apiVersion: v1
+data:
+  crowdsec-lapi-metrics.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"bargauge\",\n      \"name\": \"Bar gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1655915193937,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 10,\n      \"panels\": [],\n      \"title\": \"Agents\",\n      \"type\": \"row\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"green\",\n                \"value\": null\n              },\n              {\n                \"color\": \"red\",\n                \"value\": 80\n              }\n            ]\n          }\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2,\n      \"options\": {\n        \"displayMode\": \"gradient\",\n        \"orientation\": \"vertical\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"showUnfilled\": false,\n        \"text\": {}\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"repeat\": \"query0\",\n      \"repeatDirection\": \"h\",\n      \"targets\": [\n        {\n          \"exemplar\": false,\n          \"expr\": \"sum(rate(cs_lapi_request_duration_seconds_bucket{endpoint=\\\"/v1/watchers/login\\\", instance=\\\"$lapi\\\"}[$__rate_interval])) by (le)\",\n          \"format\": \"heatmap\",\n          \"interval\": \"\",\n          \"legendFormat\": \"{{le}}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"title\": \"Agents Login\",\n      \"type\": \"heatmap\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"green\",\n                \"value\": null\n              }\n            ]\n          },\n          \"unit\": \"none\"\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 1\n      },\n      \"id\": 6,\n      \"options\": {\n        \"displayMode\": \"gradient\",\n        \"orientation\": \"auto\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"showUnfilled\": false,\n        \"text\": {}\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"targets\": [\n        {\n          \"exemplar\": true,\n          \"expr\": \"sum(rate(cs_lapi_request_duration_seconds_bucket{endpoint=\\\"/v1/watchers/login\\\"}[$__rate_interval])) by (le)\",\n          \"format\": \"heatmap\",\n          \"interval\": \"\",\n          \"legendFormat\": \"{{le}}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"title\": \"Heartbeat\",\n      \"type\": \"heatmap\"\n    },\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 12,\n      \"panels\": [],\n      \"title\": \"Decisions\",\n      \"type\": \"row\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n  Line truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: "1"
+  name: crowdsec-crowdsec-lapi-metrics
+  namespace: prometheus
+---
+apiVersion: v1
+data:
+  crowdsec-insight.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"bargauge\",\n      \"name\": \"Bar gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"gauge\",\n      \"name\": \"Gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"stat\",\n      \"name\": \"Stat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1655915159751,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 22,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"datasource\": \"${DS_PROMETHEUS}\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"color\": {\n                \"mode\": \"thresholds\"\n              },\n              \"mappings\": [\n                {\n                  \"options\": {\n                    \"match\": \"null\",\n                    \"result\": {\n                      \"text\": \"N/A\"\n                    }\n                  },\n                  \"type\": \"special\"\n                }\n              ],\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  },\n                  {\n                    \"color\": \"red\",\n                    \"value\": 80\n                  }\n                ]\n              },\n              \"unit\": \"dateTimeAsIso\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 5,\n            \"x\": 2,\n            \"y\": 1\n          },\n          \"id\": 2,\n          \"interval\": null,\n          \"links\": [],\n          \"maxDataPoints\": 100,\n          \"options\": {\n            \"colorMode\": \"none\",\n            \"graphMode\": \"none\",\n            \"justifyMode\": \"auto\",\n            \"orientation\": \"horizontal\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"lastNotNull\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\n            \"text\": {},\n            \"textMode\": \"auto\"\n          },\n          \"pluginVersion\": \"8.1.2\",\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"(process_start_time_seconds{instance=\\\"$instance\\\"})*1000\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Up since\",\n          \"type\": \"stat\"\n        },\n        {\n          \"datasource\": \"${DS_PROMETHEUS}\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"displayName\": \"\",\n              \"mappings\": [],\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  }\n                ]\n              },\n              \"unit\": \"decbytes\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 5,\n            \"x\": 7,\n            \"y\": 1\n          },\n          \"id\": 4,\n          \"options\": {\n            \"orientation\": \"auto\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"mean\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\Line truncated
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: manual
+    grafana_dashboard: "1"
+  name: crowdsec-crowdsec-insight
+  namespace: prometheus

crowdsec/k8s/janitor-cronjob.yaml

@@ -0,0 +1,202 @@
+# CrowdSec self-healing: static machine identity + enforcement loops.
+#
+# Problem it fixes: the chart's agent init container runs
+#   `cscli lapi register --machine "$POD_NAME" ...`
+# unconditionally. Credentials live in an emptyDir, the machine row lives
+# in LAPI's persistent DB. Any init re-run for an already-known pod name
+# (kubelet restart, node reboot) dies with
+#   403 Forbidden: user '<pod>' already exist
+# and the DaemonSet pod sticks in Init forever. Every DS restart also
+# leaves an orphan machine row that is never cleaned.
+#
+# Design (name-independent):
+#   * Agent identity is a STATIC machine `crowdsec-agent-workstation`
+#     whose password lives in Secret `crowdsec-agent-credentials`
+#     (created once, manually - like all other secrets in this repo).
+#     The secret is mounted into agent pods at
+#     /tmp_config/local_api_credentials.yaml (see extraVolumeMounts in
+#     crowdsec-values.yaml), which is exactly the path the agent's main
+#     container copies into place at startup.
+#   * The DS init command is patched (strategic merge, by container name)
+#     to SKIP registration when that file exists, keeping the legacy
+#     register path only as fallback. Detection marker in the patched
+#     command: `[ -s /tmp_config`.
+#   * This CronJob enforces the desired state hourly, so recovery is
+#     automatic even after `helm upgrade` reverts the DS patch or the
+#     LAPI database is wiped:
+#       1. patch DS init if it still has the unconditional register
+#          (no-op otherwise - no restart churn);
+#       2. prune machines with no heartbeat for 2h (orphan hygiene);
+#       3. ensure the static machine exists, recreating it with the
+#          Secret password if missing (agent retry loops reconnect
+#          on their own - same name + same password);
+#       4. prune bouncer entries idle for 30d.
+#
+# It used to also delete LePresidente/http-generic-403-bf decisions hourly.
+# That was a workaround for the bouncer failing closed on a slow LAPI and
+# 403-ing the deploy runner into a 4h ban. The bouncer now polls decisions
+# into a cache and never blocks on an unreachable LAPI, so it cannot
+# manufacture those 403s any more, and the scenario only fires against real
+# scanners - deleting their decisions hourly was undoing a working ban.
+#
+# Manual apply (crowdsec/k8s is NOT managed by deploy.yaml):
+#   kubectl apply -f crowdsec/k8s/janitor-cronjob.yaml
+# Force a run:
+#   kubectl create job -n crowdsec --from=cronjob/crowdsec-janitor janitor-now
+#
+# Helm upgrades: the janitor's strategic patch puts the DS field under
+# the `kubectl-patch` field manager, so a plain `helm upgrade` FAILS
+# with an SSA conflict on initContainers[].command. Procedure:
+#   1. revert init to chart state (kills the conflict):
+#        helm template crowdsec crowdsec/crowdsec --version <ver> \
+#          -n crowdsec -f crowdsec/k8s/crowdsec-values.yaml > /tmp/r.yaml
+#        python3 -c "import yaml,json; ..." # build revert patch from
+#        the rendered DaemonSet init command, then
+#        kubectl patch ds crowdsec-agent -n crowdsec \
+#          --type strategic -p "\$(cat /tmp/revert_patch.json)"
+#   2. helm upgrade --install crowdsec ... (no --force needed)
+#   3. janitor-now right away (upgrade reverts init; new pods would
+#      sit in Init until the next hourly run otherwise).
+#
+# One-time bootstrap (order matters):
+#   1. Create Secret + static machine (see commands in chat).
+#   2. Apply this file, trigger janitor-now, wait for agent 1/1.
+#   3. One-time orphan cleanup:
+#        kubectl exec -n crowdsec deploy/crowdsec-lapi -- \
+#          cscli machines prune --duration 1h --force
+#   4. Only then `helm upgrade` crowdsec with the extraVolumes values.
+#      Upgrade reverts the DS patch; trigger janitor-now right after it
+#      (otherwise new pods sit in Init until the next hourly run, then
+#      self-heal anyway).
+#
+# Password rotation: update the Secret, delete the machine
+# (`cscli machines delete crowdsec-agent-workstation`), trigger
+# janitor-now (recreates it), then `kubectl rollout restart
+# ds/crowdsec-agent -n crowdsec` (agent reads the file at startup only).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+  - apiGroups: ["apps"]
+    resources: ["daemonsets"]
+    verbs: ["get", "patch"]
+  # `kubectl exec deploy/<name>` resolves deploy -> replicaset -> pod,
+  # which needs read access to these (exec itself is pods/exec above).
+  - apiGroups: ["apps"]
+    resources: ["deployments", "replicasets"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+subjects:
+  - kind: ServiceAccount
+    name: crowdsec-janitor
+    namespace: crowdsec
+roleRef:
+  kind: Role
+  name: crowdsec-janitor
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: crowdsec-janitor
+  namespace: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec
+spec:
+  schedule: "17 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 300
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/part-of: crowdsec
+        spec:
+          serviceAccountName: crowdsec-janitor
+          restartPolicy: OnFailure
+          containers:
+            - name: janitor
+              # Same image the chart itself uses for registration jobs;
+              # IfNotPresent so it works while the node is offline
+              # (layer cached from the chart install).
+              image: alpine/kubectl:latest
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: AGENT_PASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: crowdsec-agent-credentials
+                      key: password
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -eu
+                  LAPI_EXEC="kubectl exec -n crowdsec deploy/crowdsec-lapi --"
+                  echo "== 1. enforce patched agent init =="
+                  CUR=$(kubectl get ds crowdsec-agent -n crowdsec \
+                    -o jsonpath='{.spec.template.spec.initContainers[0].command[2]}')
+                  case "$CUR" in
+                    *'-s /tmp_config'*)
+                      echo "init already patched"
+                      ;;
+                    *)
+                      echo "patching init"
+                      WAIT='until nc "$LAPI_HOST" "$LAPI_PORT" -z'
+                      WAIT="$WAIT; do echo waiting for lapi to start; sleep 5; done"
+                      LINK='ln -s /staging/etc/crowdsec /etc/crowdsec'
+                      REG='cscli lapi register --machine "$USERNAME"'
+                      REG="$REG -u \"\$LAPI_URL\" --token \"\$REGISTRATION_TOKEN\""
+                      CREDS=/tmp_config/local_api_credentials.yaml
+                      CMD="$WAIT; $LINK; [ -s $CREDS ] || {"
+                      CMD="$CMD $REG && cp"
+                      CMD="$CMD /etc/crowdsec/local_api_credentials.yaml $CREDS; }"
+                      ESC=$(printf '%s' "$CMD" | sed 's/"/\\"/g')
+                      PATCH='{"spec":{"template":{"spec":{"initContainers":'
+                      PATCH=$PATCH'[{"name":"wait-for-lapi-and-register",'
+                      PATCH=$PATCH'"command":["sh","-c","'$ESC'"]}]}}}}'
+                      kubectl patch ds crowdsec-agent -n crowdsec \
+                        --type strategic -p "$PATCH"
+                      ;;
+                  esac
+                  echo "== 2. prune orphan machines (no heartbeat for 2h) =="
+                  $LAPI_EXEC cscli machines prune --duration 2h --force
+                  echo "== 3. ensure static machine exists =="
+                  if $LAPI_EXEC cscli machines inspect \
+                    crowdsec-agent-workstation >/dev/null 2>&1; then
+                    echo "static machine present"
+                  else
+                    echo "recreating static machine"
+                    $LAPI_EXEC cscli machines add crowdsec-agent-workstation \
+                      --password "$AGENT_PASSWORD" --force
+                  fi
+                  echo "== 4. prune stale bouncers (no pull for 30d) =="
+                  $LAPI_EXEC cscli bouncers prune -d 720h --force

crowdsec/k8s/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: crowdsec
+  labels:
+    app.kubernetes.io/part-of: crowdsec

crowdsec/k8s/network-policy.yaml

@@ -0,0 +1,34 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: crowdsec-lapi
+  namespace: crowdsec
+spec:
+  podSelector:
+    matchLabels:
+      k8s-app: crowdsec
+      type: lapi
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: traefik
+          podSelector:
+            matchLabels:
+              app.kubernetes.io/name: traefik
+        - podSelector:
+            matchLabels:
+              k8s-app: crowdsec
+              type: agent
+      ports:
+        - protocol: TCP
+          port: 8080
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: prometheus
+      ports:
+        - protocol: TCP
+          port: 6060

dockmon/compose.yaml

@@ -1,6 +1,6 @@
 services:
   dockmon:
-    image: darthnorse/dockmon:latest
+    image: darthnorse/dockmon:2.5.0
     container_name: dockmon
     restart: unless-stopped
     # ports:

dockmon/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: dockmon-prod-tls
+  namespace: dockmon
+spec:
+  secretName: dockmon-prod-tls
+  dnsNames:
+    - dockmon.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: dockmon
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

dockmon/k8s/dockmon.yaml

@@ -29,7 +29,7 @@ spec:
     spec:
       containers:
         - name: dockmon
-          image: darthnorse/dockmon:latest
+          image: darthnorse/dockmon:2.5.0
           ports:
             - containerPort: 443
           volumeMounts:

dockmon/k8s/ingress.yaml

@@ -18,13 +18,15 @@ spec:
     - match: Host(`dockmon.forust.xyz`)
       kind: Rule
       middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
         - name: security-headers@file
       services:
         - name: dockmon-service
           port: 443
           serversTransport: dockmon-transport
   tls:
-    certResolver: letsencrypt
+    secretName: dockmon-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -41,3 +43,5 @@ spec:
         - name: dockmon-service
           port: 443
           serversTransport: dockmon-transport
+  tls:
+    secretName: internal-wildcard-tls

downtify/compose.yaml

@@ -1,7 +1,7 @@
 services:
   downtify:
     container_name: downtify
-    image: ghcr.io/henriquesebastiao/downtify:latest
+    image: ghcr.io/henriquesebastiao/downtify:3.1.0
     restart: unless-stopped
     # ports:
     #   - '7077:8000'

downtify/k8s/downtify.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: downtify
-          image: ghcr.io/henriquesebastiao/downtify:latest
+          image: ghcr.io/henriquesebastiao/downtify:3.1.0
           ports:
             - containerPort: 8000
           volumeMounts:

downtify/k8s/ingress.yaml

@@ -10,12 +10,14 @@ spec:
     - match: Host(`downtify.forust.xyz`)
       kind: Rule
       middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
         - name: security-chain@file
       services:
         - name: downtify-service
           port: 8000
   tls:
-    certResolver: letsencrypt
+    secretName: downtify-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -31,3 +33,5 @@ spec:
       services:
         - name: downtify-service
           port: 8000
+  tls:
+    secretName: internal-wildcard-tls

edu_master/PLAYWRIGHT_VERSION

@@ -0,0 +1 @@
+1.56.0

edu_master/compose.yaml

@@ -1,6 +1,6 @@
 services:
   redis:
-    image: redis:alpine
+    image: redis:8.10.2-alpine
     restart: unless-stopped
     volumes:
       - redis-data:/data
@@ -17,7 +17,7 @@ services:
 
   session-keeper:
     build: ./phpsessid-bot
-    image: gcr.forust.xyz/forust/session-keeper:latest
+    image: gcr.forust.xyz/forust/session-keeper:prod
     pull_policy: build
     env_file: .env
     restart: unless-stopped
@@ -33,7 +33,7 @@ services:
 
   webinar-checker:
     build: ./webinar-checker
-    image: gcr.forust.xyz/forust/webinar-checker:latest
+    image: gcr.forust.xyz/forust/webinar-checker:prod
     pull_policy: build
     env_file: .env
     restart: unless-stopped

edu_master/k8s/alerts.yaml

@@ -0,0 +1,96 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: edu-master-webinar
+  namespace: edu-master
+  labels:
+    release: prometheus-stack
+spec:
+  groups:
+    - name: edu_master.webinar
+      rules:
+        # No successful webinar check for 5m (~2-3 missed 2-min checks).
+        # Catches: playwright hangs/timeouts, version skew, site changes, hung job.
+        # The last_success > 0 guard is mandatory: checker.py initialises
+        # last_success to 0, so without it `time() - 0` equals the current epoch
+        # and humanizeDuration renders ~20722d on every pod restart. Keep the
+        # duration expression on the left so $value stays the real gap.
+        - alert: WebinarCheckerNoSuccessfulCheck
+          expr: |
+            ((time() - webinar_check_last_success_timestamp_seconds) > 300)
+              and (webinar_check_last_success_timestamp_seconds > 0)
+              and (webinar_check_last_run_timestamp_seconds > 0)
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker has no successful check for 5m"
+            description: "edu-master/webinar-checker: last successful webinar check was {{ $value | humanizeDuration }} ago. Checks are failing or hanging (see consecutive failures alert). Notifications about new webinars are NOT being sent."
+
+        # Checks are running but none has ever succeeded since pod start.
+        # Split out from the rule above so a zeroed gauge never feeds
+        # humanizeDuration.
+        - alert: WebinarCheckerNeverSucceeded
+          expr: |
+            (webinar_check_last_success_timestamp_seconds == 0)
+              and (webinar_check_last_run_timestamp_seconds > 0)
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker has never completed a successful check"
+            description: 'edu-master/webinar-checker: checks have been running for 10m but not one has ever succeeded since the pod started, so every check is failing. Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).'
+
+        # Fast path: 3 consecutive failures (~6+ min at 2-min interval).
+        - alert: WebinarCheckerConsecutiveFailures
+          expr: |
+            webinar_check_consecutive_failures >= 3
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker failing consecutively"
+            description: 'edu-master/webinar-checker: {{ $value }} consecutive webinar check failures (timeout / playwright error / page error). Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).'
+
+        # Metrics endpoint not scraped for 10m: pod down, metrics server dead, or ServiceMonitor broken.
+        - alert: WebinarCheckerScrapeDown
+          expr: |
+            absent(webinar_check_last_run_timestamp_seconds) == 1
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker metrics missing"
+            description: "edu-master/webinar-checker: no metrics series for 10m. Pod may be down, metrics server dead, or ServiceMonitor/Service broken. Webinar checks are unobserved."
+
+        # EDU session lost: session-keeper down or credentials expired. Without PHPSESSID every check is skipped.
+        - alert: EduPhpsessidMissing
+          expr: |
+            edu_phpsessid_present == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "EDU_PHPSESSID missing"
+            description: "edu-master: EDU_PHPSESSID absent from redis for 10m. Webinar/diari/schedule checks are all skipped. Check session-keeper logs and EDU credentials."
+
+        # Hard deps: checker and playwright deployments unavailable.
+        - alert: WebinarCheckerDeploymentDown
+          expr: |
+            kube_deployment_status_replicas_unavailable{deployment="webinar-checker", namespace="edu-master"} > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Webinar checker deployment unavailable"
+            description: "edu-master/webinar-checker deployment has {{ $value }} unavailable replica(s) for 10m."
+
+        - alert: PlaywrightServiceDown
+          expr: |
+            kube_deployment_status_replicas_unavailable{deployment="playwright-service", namespace="edu-master"} > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Playwright service unavailable"
+            description: "edu-master/playwright-service deployment has {{ $value }} unavailable replica(s) for 10m. All webinar/diari/schedule checks fail without it."

edu_master/k8s/playwright.yaml

@@ -17,8 +17,18 @@ spec:
     spec:
       containers:
         - name: playwright
+          # renovate: datasource=docker depName=mcr.microsoft.com/playwright versioning=docker
           image: mcr.microsoft.com/playwright:v1.56.0-jammy
           imagePullPolicy: IfNotPresent
+          # p95 412M, max 478M over 7 days, no limit before. Request is set at p95
+          # so the pod is not an eviction candidate; the limit stays above 2x the
+          # request because browser page lifetimes are unpredictable.
+          resources:
+            requests:
+              cpu: "200m"
+              memory: "416Mi"
+            limits:
+              memory: "1Gi"
           command:
             - npx
             - -y

edu_master/k8s/redis.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: redis
-          image: redis:alpine
+          image: redis:8.10.2-alpine
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 6379
@@ -28,10 +28,10 @@ spec:
           resources:
             requests:
               cpu: 25m
-              memory: 64Mi
+              memory: 32Mi
             limits:
               cpu: 250m
-              memory: 256Mi
+              memory: 128Mi
           readinessProbe:
             exec:
               command: ["redis-cli", "ping"]

edu_master/k8s/secrets.yaml.example

@@ -21,6 +21,8 @@ stringData:
   WEBINAR_TELEGRAM_TOKEN: ""
   WEBINAR_ADMIN_ID: ""
   WEBINAR_CHECK_INTERVAL: "60"
+  # Prometheus metrics endpoint (scraped via ServiceMonitor, alerts in k8s/alerts.yaml)
+  METRICS_PORT: "8000"
   # Database
   REDIS_HOST: "redis"
   REDIS_PORT: "6379"

edu_master/k8s/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: webinar-checker
+  namespace: edu-master
+  labels:
+    app: edu-master-webinar-checker
+spec:
+  selector:
+    app: edu-master-webinar-checker
+  ports:
+    - name: metrics
+      port: 8000
+      targetPort: metrics
+      protocol: TCP

edu_master/k8s/servicemonitor.yaml

@@ -0,0 +1,16 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: webinar-checker
+  namespace: edu-master
+  labels:
+    release: prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      app: edu-master-webinar-checker
+  endpoints:
+    - port: metrics
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

edu_master/k8s/session-keeper.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       initContainers:
         - name: wait-redis
-          image: redis:alpine
+          image: redis:8.10.2-alpine
           command:
             - /bin/sh
             - -ec
@@ -31,18 +31,17 @@ spec:
               echo "redis is ready"
       containers:
         - name: session-keeper
-          image: gcr.forust.xyz/forust/session-keeper:latest
-          imagePullPolicy: Always
+          image: gcr.forust.xyz/forust/session-keeper:prod
           envFrom:
             - secretRef:
                 name: edu-master-secrets
           resources:
             requests:
               cpu: 25m
-              memory: 96Mi
+              memory: 32Mi
             limits:
               cpu: 250m
-              memory: 256Mi
+              memory: 128Mi
           readinessProbe:
             exec:
               command: ["/bin/sh", "-ec", "redis-cli -h redis EXISTS EDU_PHPSESSID | grep -q 1"]

edu_master/k8s/webinar-checker.yaml

@@ -19,7 +19,7 @@ spec:
       # redis healthy -> session-keeper healthy (EXISTS EDU_PHPSESSID) -> playwright started
       initContainers:
         - name: wait-deps
-          image: redis:alpine
+          image: redis:8.10.2-alpine
           command:
             - /bin/sh
             - -ec
@@ -45,8 +45,19 @@ spec:
               echo "playwright ok"
       containers:
         - name: webinar-checker
-          image: gcr.forust.xyz/forust/webinar-checker:latest
-          imagePullPolicy: Always
+          image: gcr.forust.xyz/forust/webinar-checker:prod
+          ports:
+            - name: metrics
+              containerPort: 8000
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: metrics
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 12
+            initialDelaySeconds: 10
           envFrom:
             - secretRef:
                 name: edu-master-secrets
@@ -55,8 +66,8 @@ spec:
               value: "Europe/Kyiv"
           resources:
             requests:
-              cpu: 25m
-              memory: 128Mi
+              cpu: "50m"
+              memory: "192Mi"
             limits:
-              cpu: 300m
-              memory: 384Mi
+              cpu: "600m"
+              memory: "384Mi"

edu_master/webinar-checker/Dockerfile

@@ -2,8 +2,11 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# Install dependencies
-RUN pip install --no-cache-dir pip==25.0.1 && pip install --no-cache-dir playwright==1.56.0 redis==5.2.1 requests==2.32.3 "python-telegram-bot[job-queue]==21.10"
+# renovate: datasource=pypi depName=playwright versioning=pep440
+ARG PLAYWRIGHT_VERSION=1.56.0
+
+# Install dependencies - PLAYWRIGHT_VERSION is single-source, renovate updates ARG above and all other places via regexManagers
+RUN pip install --no-cache-dir pip==25.0.1 && pip install --no-cache-dir playwright==${PLAYWRIGHT_VERSION} redis==5.2.1 requests==2.32.3 "python-telegram-bot[job-queue]==21.10"
 
 COPY checker.py .
 

edu_master/webinar-checker/checker.py

@@ -1,12 +1,15 @@
+import asyncio
 import contextlib
 import json
 import logging
 import os
 import re
 import tempfile
+import threading
 import time
 from datetime import datetime, timedelta
 from html import escape
+from http.server import BaseHTTPRequestHandler, HTTPServer
 
 import redis
 from playwright.async_api import async_playwright
@@ -48,6 +51,106 @@ USER_AGENT = _env(
 )
 WEBINAR_TELEGRAM_TOKEN = _env('WEBINAR_TELEGRAM_TOKEN')
 ADMIN_ID = int(_env('WEBINAR_ADMIN_ID', '0'))
+METRICS_PORT = int(_env('METRICS_PORT', '8000'))
+
+# --- Prometheus metrics (stdlib only, no extra deps) ---
+# Scraped by prometheus-stack via ServiceMonitor (edu_master/k8s/servicemonitor.yaml).
+# Critical alerts in edu_master/k8s/alerts.yaml fire to Telegram via Alertmanager.
+_METRICS_LOCK = threading.Lock()
+_METRICS = {
+    'last_run': 0.0,  # Unix ts of last check start
+    'last_success': 0.0,  # Unix ts of last successful check
+    'last_duration': 0.0,  # Duration of last check in seconds
+    'success_total': 0,
+    'failure_total': 0,
+    'consecutive_failures': 0,
+    'phpsessid_present': 1,  # 1 if EDU_PHPSESSID found in redis, else 0
+}
+
+
+def _metric_check_start():
+    with _METRICS_LOCK:
+        _METRICS['last_run'] = time.time()
+
+
+def _metric_check_ok(duration: float):
+    now = time.time()
+    with _METRICS_LOCK:
+        _METRICS['last_success'] = now
+        _METRICS['last_duration'] = duration
+        _METRICS['success_total'] += 1
+        _METRICS['consecutive_failures'] = 0
+        _METRICS['phpsessid_present'] = 1
+
+
+def _metric_check_fail(duration: float, phpsessid_missing: bool = False):
+    with _METRICS_LOCK:
+        _METRICS['last_duration'] = duration
+        _METRICS['failure_total'] += 1
+        _METRICS['consecutive_failures'] += 1
+        _METRICS['phpsessid_present'] = 0 if phpsessid_missing else 1
+
+
+def _metrics_render() -> bytes:
+    with _METRICS_LOCK:
+        m = dict(_METRICS)
+    lines = [
+        '# HELP webinar_check_last_run_timestamp_seconds Unix timestamp of last webinar check start.',
+        '# TYPE webinar_check_last_run_timestamp_seconds gauge',
+        f'webinar_check_last_run_timestamp_seconds {m["last_run"]}',
+        '# HELP webinar_check_last_success_timestamp_seconds Unix timestamp of last successful webinar check.',
+        '# TYPE webinar_check_last_success_timestamp_seconds gauge',
+        f'webinar_check_last_success_timestamp_seconds {m["last_success"]}',
+        '# HELP webinar_check_last_duration_seconds Duration of last webinar check in seconds.',
+        '# TYPE webinar_check_last_duration_seconds gauge',
+        f'webinar_check_last_duration_seconds {m["last_duration"]}',
+        '# HELP webinar_check_success_total Total successful webinar checks.',
+        '# TYPE webinar_check_success_total counter',
+        f'webinar_check_success_total {m["success_total"]}',
+        '# HELP webinar_check_failure_total Total failed webinar checks (timeout, playwright error, page error).',
+        '# TYPE webinar_check_failure_total counter',
+        f'webinar_check_failure_total {m["failure_total"]}',
+        '# HELP webinar_check_consecutive_failures Consecutive failed webinar checks (reset on success).',
+        '# TYPE webinar_check_consecutive_failures gauge',
+        f'webinar_check_consecutive_failures {m["consecutive_failures"]}',
+        '# HELP edu_phpsessid_present 1 if EDU_PHPSESSID exists in redis, 0 otherwise.',
+        '# TYPE edu_phpsessid_present gauge',
+        f'edu_phpsessid_present {m["phpsessid_present"]}',
+    ]
+    return ('\n'.join(lines) + '\n').encode()
+
+
+class _MetricsHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == '/metrics':
+            body = _metrics_render()
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain; version=0.0.4')
+            self.send_header('Content-Length', str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        elif self.path in ('/healthz', '/health'):
+            body = b'ok\n'
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain')
+            self.send_header('Content-Length', str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+    def log_message(self, *args):
+        pass  # keep bot logs clean
+
+
+def start_metrics_server(port: int = METRICS_PORT):
+    server = HTTPServer(('0.0.0.0', port), _MetricsHandler)  # noqa: S104 - k8s ServiceMonitor scrapes pod IP
+    thread = threading.Thread(target=server.serve_forever, name='metrics-server', daemon=True)
+    thread.start()
+    logger.info(f'Metrics server listening on :{port}/metrics')
+    return server
+
 
 # Redis Keys
 KEY_WHITELIST = 'bot:whitelist'
@@ -597,59 +700,66 @@ async def _collect_event_times(page) -> dict:
 async def fetch_diary_data(phpsessid: str) -> dict | None:
     logger.info('Fetching diary data via Playwright...')
     try:
-        async with async_playwright() as p:
-            browser = await p.chromium.connect(PLAYWRIGHT_WS)
-            try:
-                context_browser = await browser.new_context(user_agent=USER_AGENT)
-                await context_browser.add_cookies(
-                    [{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
-                )
-                page = await context_browser.new_page()
-
+        async with asyncio.timeout(60):
+            async with async_playwright() as p:
+                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
                 try:
-                    await page.goto(DIARY_URL, wait_until='domcontentloaded')
-                    await page.wait_for_selector('table.calendar', timeout=10000)
-                    await page.wait_for_timeout(1500)
-
-                    table_html = await page.evaluate("""
-                        () => {
-                            const t = document.querySelector('table.calendar');
-                            return t ? t.outerHTML : null;
-                        }
-                    """)
-                    if not table_html:
-                        logger.error('table.calendar not found in DOM')
-                        return None
-
-                    # Debug: save HTML for troubleshooting
-                    with contextlib.suppress(Exception), open('/tmp/diary_debug.html', 'w', encoding='utf-8') as f:  # noqa: S108
-                        f.write(table_html)
-
-                    month_text, days = _parse_calendar_html(table_html)
-
-                    # Read event times by opening each event's AJAX popup.
-                    times_by_id = await _collect_event_times(page)
-                    if times_by_id:
-                        for day_data in days.values():
-                            for ev in day_data.get('events', []):
-                                eid = ev.get('id')
-                                if eid and eid in times_by_id:
-                                    ev['time'] = times_by_id[eid]
-
-                    logger.info(
-                        f'Diary parsed: month={month_text!r}, days_with_events={sum(1 for d in days.values() if d["events"])}/{len(days)}'
+                    context_browser = await browser.new_context(user_agent=USER_AGENT)
+                    await context_browser.add_cookies(
+                        [{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
                     )
+                    page = await context_browser.new_page()
 
-                    return {'monthFullText': month_text, 'days': days}
+                    try:
+                        await asyncio.wait_for(page.goto(DIARY_URL, wait_until='domcontentloaded'), timeout=30)
+                        await page.wait_for_selector('table.calendar', timeout=10000)
+                        await page.wait_for_timeout(1500)
 
-                except Exception as e:
-                    logger.error(f'Error parsing diary: {e}')
-                    return None
+                        table_html = await page.evaluate("""
+                            () => {
+                                const t = document.querySelector('table.calendar');
+                                return t ? t.outerHTML : null;
+                            }
+                        """)
+                        if not table_html:
+                            logger.error('table.calendar not found in DOM')
+                            return None
+
+                        # Debug: save HTML for troubleshooting
+                        with contextlib.suppress(Exception), open('/tmp/diary_debug.html', 'w', encoding='utf-8') as f:  # noqa: S108
+                            f.write(table_html)
+
+                        month_text, days = _parse_calendar_html(table_html)
+
+                        # Read event times by opening each event's AJAX popup.
+                        times_by_id = await _collect_event_times(page)
+                        if times_by_id:
+                            for day_data in days.values():
+                                for ev in day_data.get('events', []):
+                                    eid = ev.get('id')
+                                    if eid and eid in times_by_id:
+                                        ev['time'] = times_by_id[eid]
+
+                        logger.info(
+                            f'Diary parsed: month={month_text!r}, days_with_events={sum(1 for d in days.values() if d["events"])}/{len(days)}'
+                        )
+
+                        return {'monthFullText': month_text, 'days': days}
+
+                    except Exception as e:
+                        logger.error(f'Error parsing diary: {e}')
+                        return None
+                    finally:
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(page.close(), timeout=5)
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(context_browser.close(), timeout=5)
                 finally:
-                    await page.close()
-                    await context_browser.close()
-            finally:
-                await browser.close()
+                    with contextlib.suppress(Exception):
+                        await asyncio.wait_for(browser.close(), timeout=5)
+    except TimeoutError:
+        logger.error('Diary fetch timed out (60s)')
+        return None
     except Exception as e:
         logger.error(f'Playwright error in diary fetch: {e}')
         return None
@@ -931,48 +1041,55 @@ def _parse_schedule_html(table_html: str) -> dict:
 async def fetch_schedule_data(phpsessid: str) -> dict | None:
     logger.info('Fetching schedule data via Playwright...')
     try:
-        async with async_playwright() as p:
-            browser = await p.chromium.connect(PLAYWRIGHT_WS)
-            try:
-                context_browser = await browser.new_context(user_agent=USER_AGENT)
-                await context_browser.add_cookies(
-                    [{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
-                )
-                page = await context_browser.new_page()
-
+        async with asyncio.timeout(60):
+            async with async_playwright() as p:
+                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
                 try:
-                    await page.goto(SCHEDULE_URL, wait_until='domcontentloaded')
-                    await page.wait_for_selector('table.schedule-table', timeout=10000)
-                    await page.wait_for_timeout(1500)
+                    context_browser = await browser.new_context(user_agent=USER_AGENT)
+                    await context_browser.add_cookies(
+                        [{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
+                    )
+                    page = await context_browser.new_page()
 
-                    table_html = await page.evaluate("""
-                        () => {
-                            const t = document.querySelector('table.schedule-table');
-                            return t ? t.outerHTML : null;
-                        }
-                    """)
-                    if not table_html:
-                        logger.error('table.schedule-table not found in DOM')
+                    try:
+                        await asyncio.wait_for(page.goto(SCHEDULE_URL, wait_until='domcontentloaded'), timeout=30)
+                        await page.wait_for_selector('table.schedule-table', timeout=10000)
+                        await page.wait_for_timeout(1500)
+
+                        table_html = await page.evaluate("""
+                            () => {
+                                const t = document.querySelector('table.schedule-table');
+                                return t ? t.outerHTML : null;
+                            }
+                        """)
+                        if not table_html:
+                            logger.error('table.schedule-table not found in DOM')
+                            return None
+
+                        debug_path = os.path.join(tempfile.gettempdir(), 'schedule_debug.html')
+                        with contextlib.suppress(Exception), open(debug_path, 'w', encoding='utf-8') as f:
+                            f.write(table_html)
+
+                        data = _parse_schedule_html(table_html)
+                        logger.info(list(data['weekdays'].keys()))
+                        logger.info(f'Schedule parsed: {len(data["weekdays"])} days, classes={data["classes"]}')
+
+                        return data
+
+                    except Exception as e:
+                        logger.error(f'Error parsing schedule: {e}')
                         return None
-
-                    debug_path = os.path.join(tempfile.gettempdir(), 'schedule_debug.html')
-                    with contextlib.suppress(Exception), open(debug_path, 'w', encoding='utf-8') as f:
-                        f.write(table_html)
-
-                    data = _parse_schedule_html(table_html)
-                    logger.info(list(data['weekdays'].keys()))
-                    logger.info(f'Schedule parsed: {len(data["weekdays"])} days, classes={data["classes"]}')
-
-                    return data
-
-                except Exception as e:
-                    logger.error(f'Error parsing schedule: {e}')
-                    return None
+                    finally:
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(page.close(), timeout=5)
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(context_browser.close(), timeout=5)
                 finally:
-                    await page.close()
-                    await context_browser.close()
-            finally:
-                await browser.close()
+                    with contextlib.suppress(Exception):
+                        await asyncio.wait_for(browser.close(), timeout=5)
+    except TimeoutError:
+        logger.error('Schedule fetch timed out (60s)')
+        return None
     except Exception as e:
         logger.error(f'Playwright error in schedule fetch: {e}')
         return None
@@ -1485,10 +1602,13 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
         int: Number of webinars found, or None if check failed
     """
     logger.info('Running webinar check...')
+    _t0 = time.time()
+    _metric_check_start()
 
     phpsessid = redis_client.get(KEY_PHPSESSID)
     if not phpsessid:
         logger.warning('PHPSESSID missing. Skipping check.')
+        _metric_check_fail(time.time() - _t0, phpsessid_missing=True)
         # --- DEBUG LOGGING ---
         try:
             with open('phpsessid_missing.log', 'a') as f:
@@ -1502,78 +1622,88 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
     content = ''
 
     try:
-        async with async_playwright() as p:
-            # Connect to remote Playwright service
-            browser = await p.chromium.connect(PLAYWRIGHT_WS)
-
-            try:
-                # Create browser context with user agent
-                context_browser = await browser.new_context(user_agent=USER_AGENT)
-
-                # Add PHPSESSID cookie
-                await context_browser.add_cookies(
-                    [{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
-                )
-
-                # Create new page
-                page = await context_browser.new_page()
+        async with asyncio.timeout(90):
+            async with async_playwright() as p:
+                # Connect to remote Playwright service
+                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
 
                 try:
-                    # Navigate to webinar page
-                    await page.goto(WEBINAR_URL, wait_until='domcontentloaded')
+                    # Create browser context with user agent
+                    context_browser = await browser.new_context(user_agent=USER_AGENT)
 
-                    # Wait for the table to load
-                    await page.wait_for_selector('#meetings table', timeout=10000)
-                    await page.wait_for_timeout(2000)
+                    # Add PHPSESSID cookie
+                    await context_browser.add_cookies(
+                        [{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
+                    )
 
-                    # Get page content
-                    content = await page.content()
+                    # Create new page
+                    page = await context_browser.new_page()
 
-                    # Check if "no webinar" message is present
-                    if NO_WEBINAR_MARKER not in content:
-                        logger.info('!!! WEBINAR FOUND !!!')
+                    try:
+                        # Navigate to webinar page
+                        await asyncio.wait_for(page.goto(WEBINAR_URL, wait_until='domcontentloaded'), timeout=30)
 
-                        # Extract webinar details from table rows
-                        rows = page.locator('#meetings table tbody tr')
-                        count = await rows.count()
+                        # Wait for the table to load
+                        await page.wait_for_selector('#meetings table', timeout=10000)
+                        await page.wait_for_timeout(2000)
 
-                        for i in range(count):
-                            row = rows.nth(i)
-                            text = await row.inner_text()
+                        # Get page content
+                        content = await page.content()
 
-                            if NO_WEBINAR_MARKER not in text:
-                                # Extract name (topic) from first column
-                                name_elem = row.locator('td').nth(0)
-                                name = await name_elem.inner_text()
-                                name = name.strip()
+                        # Check if "no webinar" message is present
+                        if NO_WEBINAR_MARKER not in content:
+                            logger.info('!!! WEBINAR FOUND !!!')
 
-                                # Extract join URL from fourth column
-                                url_elem = row.locator('td').nth(3).locator('a[href*="/webinar/join/"]').first
-                                url = await url_elem.get_attribute('href')
+                            # Extract webinar details from table rows
+                            rows = page.locator('#meetings table tbody tr')
+                            count = await rows.count()
 
-                                if name and url:
-                                    current_webinars.append({'name': name, 'url': url, 'text': text.strip()})
-                                    logger.info(f'Found webinar: {name} -> {url}')
-                    else:
-                        logger.info('No webinars found (expected message present)')
+                            for i in range(count):
+                                row = rows.nth(i)
+                                text = await row.inner_text()
 
-                except Exception as e:
-                    logger.error(f'Error checking page: {e}. Saving content for debug.')
-                    # If page content is available, save it on error
-                    with contextlib.suppress(Exception):
-                        if page and not content:
-                            content = await page.content()
+                                if NO_WEBINAR_MARKER not in text:
+                                    # Extract name (topic) from first column
+                                    name_elem = row.locator('td').nth(0)
+                                    name = await name_elem.inner_text()
+                                    name = name.strip()
+
+                                    # Extract join URL from fourth column
+                                    url_elem = row.locator('td').nth(3).locator('a[href*="/webinar/join/"]').first
+                                    url = await url_elem.get_attribute('href')
+
+                                    if name and url:
+                                        current_webinars.append({'name': name, 'url': url, 'text': text.strip()})
+                                        logger.info(f'Found webinar: {name} -> {url}')
+                        else:
+                            logger.info('No webinars found (expected message present)')
+
+                    except Exception as e:
+                        logger.error(f'Error checking page: {e}. Saving content for debug.')
+                        # If page content is available, save it on error
+                        with contextlib.suppress(Exception):
+                            if page and not content:
+                                content = await page.content()
+
+                        _metric_check_fail(time.time() - _t0)
+                        return None
+                    finally:
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(page.close(), timeout=5)
+                        with contextlib.suppress(Exception):
+                            await asyncio.wait_for(context_browser.close(), timeout=5)
 
-                    return None
                 finally:
-                    await page.close()
-                    await context_browser.close()
-
-            finally:
-                await browser.close()
+                    with contextlib.suppress(Exception):
+                        await asyncio.wait_for(browser.close(), timeout=5)
 
+    except TimeoutError:
+        logger.error('Webinar check timed out after 90s (playwright hang)')
+        _metric_check_fail(time.time() - _t0)
+        return None
     except Exception as e:
         logger.error(f'Playwright error: {e}')
+        _metric_check_fail(time.time() - _t0)
         return None
 
     # --- DEBUG LOGGING (Saving last response content) ---
@@ -1637,6 +1767,7 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
         else:
             logger.info(f'Found {len(current_webinars)} webinar(s), but all are already known')
 
+    _metric_check_ok(time.time() - _t0)
     return len(current_webinars)
 
 
@@ -1680,6 +1811,8 @@ def main():
     job_queue = app.job_queue
     job_queue.run_repeating(check_webinars_job, interval=WEBINAR_CHECK_INTERVAL, first=10)
 
+    start_metrics_server()
+
     logger.info('Bot started polling...')
     app.run_polling()
 

errorpages/compose.yaml

@@ -1,7 +1,7 @@
 services:
   errorpage:
     build: .
-    image: gcr.forust.xyz/forust/error-pages:latest
+    image: gcr.forust.xyz/forust/error-pages:prod
     pull_policy: build
     container_name: error-pages
     restart: unless-stopped

errorpages/k8s/error-pages.yaml

@@ -27,7 +27,21 @@ spec:
     spec:
       containers:
         - name: error-pages
-          image: gcr.forust.xyz/forust/error-pages:latest
+          image: gcr.forust.xyz/forust/error-pages:prod
+          # p95 6M, max 10M, no limit before.
+          resources:
+            requests:
+              cpu: "10m"
+              memory: "32Mi"
+            limits:
+              memory: "128Mi"
           ports:
             - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /404.html
+              port: 80
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
 ---

gitea/compose.yaml

@@ -1,6 +1,6 @@
 services:
   server:
-    image: docker.gitea.com/gitea:1.26
+    image: docker.gitea.com/gitea:1.27.3
     container_name: gitea
     restart: always
     environment:
@@ -61,7 +61,7 @@ services:
     depends_on:
       - db
   db:
-    image: docker.io/library/postgres:14
+    image: docker.io/library/postgres:14.24-alpine
     restart: always
     environment:
       - POSTGRES_USER=gitea

gitea/k8s/certificates.yaml

@@ -0,0 +1,29 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: gitea-prod-tls
+  namespace: gitea
+spec:
+  secretName: gitea-prod-tls
+  dnsNames:
+    - gcr.forust.xyz
+    - gitea.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: gitea
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

gitea/k8s/config.yaml

@@ -10,13 +10,19 @@ data:
   GITEA__server__SSH_PORT: "2221"
 
   GITEA__database__DB_TYPE: "postgres"
-  GITEA__database__HOST: "gitea-postgres-service:5432"
+  GITEA__database__HOST: "postgres.database.svc.cluster.local:5432"
   GITEA__database__NAME: "gitea"
   GITEA__security__REVERSE_PROXY_LIMIT: "1"
   GITEA__security__REVERSE_PROXY_TRUSTED_PROXIES: "*"
 
   GITEA__mailer__ENABLED: "false"
 
-  GITEA__log__logger__access__MODE: "console, file"
+  # No code/issue search needed: bleve reindexes the whole issue index on
+  # every pod restart (cron.rebuild_issue_indexer RUN_AT_START) and hammers
+  # the rotational disk for an hour. "db" serves issue search from postgres.
+  GITEA__indexer__ISSUE_INDEXER_TYPE: "db"
+  GITEA__indexer__REPO_INDEXER_ENABLED: "false"
+
+  GITEA__log__logger.access.MODE: "console, file"
   USER_UID: "1000"
   USER_GID: "1000"

gitea/k8s/gitea.yaml

@@ -31,7 +31,7 @@ spec:
     spec:
       containers:
         - name: gitea
-          image: docker.gitea.com/gitea:1.26
+          image: gitea/gitea:1.27.3
           envFrom:
             - configMapRef:
                 name: gitea-config
@@ -47,10 +47,10 @@ spec:
               mountPath: /data
           resources:
             requests:
-              memory: "512Mi"
+              memory: "320Mi"
               cpu: "300m"
             limits:
-              memory: "1.5Gi"
+              memory: "1Gi"
               cpu: "1300m"
       volumes:
         - name: gitea-data

gitea/k8s/ingress.yaml

@@ -9,16 +9,26 @@ spec:
   routes:
     - match: Host(`gitea.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: gitea-service
           port: 3000
+    # Registry route: NO crowdsec-bouncer.
+    # A deploy burst (runner Action API polls, `docker manifest inspect` per
+    # own image, containerd pulls, smoke probes) fires hundreds of parallel
+    # registry calls, and a ban on the runner breaks every later job. This
+    # route only serves authenticated OCI traffic - registry tokens and
+    # basic-auth are already handled by gitea - and scanners get nothing
+    # useful from /v2, so there is no bruteforce surface to protect here.
     - match: Host(`gcr.forust.xyz`) && PathPrefix(`/v2`)
       kind: Rule
       services:
         - name: gitea-service
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: gitea-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -39,6 +49,9 @@ spec:
       services:
         - name: gitea-service
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRouteTCP

gitea/k8s/postgres.yaml

@@ -1,62 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: gitea-postgres-service
-  namespace: gitea
-spec:
-  clusterIP: None
-  selector:
-    app: gitea-postgres
-  ports:
-    - port: 5432
-      targetPort: 5432
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: gitea-postgres-statefulset
-  namespace: gitea
-spec:
-  selector:
-    matchLabels:
-      app: gitea-postgres
-  serviceName: gitea-postgres-service
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: gitea-postgres
-    spec:
-      containers:
-        - name: gitea-postgres
-          image: postgres:14
-          env:
-            - name: POSTGRES_USER
-              valueFrom:
-                secretKeyRef:
-                  name: gitea-secrets
-                  key: GITEA__database__USER
-            - name: POSTGRES_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: gitea-secrets
-                  key: GITEA__database__PASSWD
-            - name: POSTGRES_DB
-              valueFrom:
-                secretKeyRef:
-                  name: gitea-secrets
-                  key: GITEA__database__USER
-          ports:
-            - containerPort: 5432
-              name: postgres
-          volumeMounts:
-            - name: postgres-data
-              mountPath: /var/lib/postgresql/data
-  volumeClaimTemplates:
-    - metadata:
-        name: postgres-data
-      spec:
-        accessModes: ["ReadWriteOnce"]
-        resources:
-          requests:
-            storage: 1Gi

gitea/k8s/secrets.yaml.example

@@ -7,3 +7,4 @@ type: Opaque
 stringData:
   GITEA__database__USER: "gitea"
   GITEA__database__PASSWD: "gitea"
+  GITEA__database__NAME: "gitea"

glance/compose.yaml

@@ -1,7 +1,7 @@
 services:
   glance:
     container_name: glance
-    image: glanceapp/glance
+    image: glanceapp/glance:v0.8.6
     restart: unless-stopped
     volumes:
       - ./config:/app/config:ro

glance/k8s/certificates.yaml

@@ -0,0 +1,29 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: glance-prod-tls
+  namespace: glance
+spec:
+  secretName: glance-prod-tls
+  dnsNames:
+    - forust.xyz
+    - www.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: glance
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

glance/k8s/glance.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
         - name: glance
-          image: glanceapp/glance
+          image: glanceapp/glance:v0.8.6
           envFrom:
             - secretRef:
                 name: glance-secrets
@@ -56,11 +56,11 @@ spec:
               readOnly: true
           resources:
             requests:
-              memory: "30Mi"
-              cpu: "20m"
-            limits:
-              memory: "100Mi"
               cpu: "50m"
+              memory: "32Mi"
+            limits:
+              cpu: "200m"
+              memory: "128Mi"
       volumes:
         - name: glance-config
           configMap:

glance/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: glance-service
           port: 8080
   tls:
-    certResolver: letsencrypt
+    secretName: glance-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -35,6 +35,9 @@ spec:
       services:
         - name: glance-service
           port: 8080
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: Middleware

headscale/compose.yaml

@@ -1,6 +1,6 @@
 services:
   headscale:
-    image: headscale/headscale:latest
+    image: headscale/headscale:v0.29.4
     restart: unless-stopped
     container_name: headscale-server
     command: serve
@@ -53,7 +53,7 @@ services:
       - "traefik.http.routers.headscale-metrics-dev.service=headscale-metrics"
       - "traefik.http.routers.headscale-metrics-dev.tls=true"
   headplane:
-    image: ghcr.io/tale/headplane:latest
+    image: ghcr.io/tale/headplane:0.7.1
     container_name: headplane
     restart: unless-stopped
     ports:
@@ -100,7 +100,7 @@ services:
       - "traefik.http.routers.headplane-dev.entrypoints=websecure"
       - "traefik.http.routers.headplane-dev.tls=true"
   web:
-    image: goodieshq/headscale-admin:latest
+    image: goodieshq/headscale-admin:0.28.0
     restart: unless-stopped
     ports:
       - 10080:80

headscale/k8s/routing/certificates.yaml

@@ -0,0 +1,41 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: headscale-prod-tls
+  namespace: headscale
+spec:
+  secretName: headscale-prod-tls
+  dnsNames:
+    - hs.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: headplane-prod-tls
+  namespace: headscale
+spec:
+  secretName: headplane-prod-tls
+  dnsNames:
+    - hp.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: headscale
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

headscale/k8s/routing/ingress.yaml

@@ -23,16 +23,22 @@ spec:
           port: 8080
     - match: Host(`hs.forust.xyz`) && PathPrefix(`/admin`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: headscale-ui-external
           port: 80
     - match: Host(`hs.forust.xyz`) && PathPrefix(`/metrics`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: headscale-server-external
           port: 9090
   tls:
-    certResolver: letsencrypt
+    secretName: headscale-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -47,6 +53,8 @@ spec:
       kind: Rule
       middlewares:
         - name: headplane-prefix
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: headplane-external
           port: 3000
@@ -56,7 +64,7 @@ spec:
         - name: headplane-external
           port: 3000
   tls:
-    certResolver: letsencrypt
+    secretName: headplane-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -82,6 +90,9 @@ spec:
       services:
         - name: headscale-server-external
           port: 9090
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -102,3 +113,5 @@ spec:
       services:
         - name: headplane-external
           port: 3000
+  tls:
+    secretName: internal-wildcard-tls

homepages/compose.yaml

@@ -3,7 +3,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.forust
-    image: gcr.forust.xyz/forust/forust-homepage:latest
+    image: gcr.forust.xyz/forust/forust-homepage:prod
     pull_policy: build
     # ports:
     #   - "8085:80"
@@ -35,7 +35,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.xdfnx
-    image: gcr.forust.xyz/forust/xdfnx-homepage:latest
+    image: gcr.forust.xyz/forust/xdfnx-homepage:prod
     pull_policy: build
     restart: unless-stopped
     # ports:

homepages/k8s/certificates.yaml

@@ -0,0 +1,42 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: forust-homepage-prod-tls
+  namespace: homepages
+spec:
+  secretName: forust-homepage-prod-tls
+  dnsNames:
+    - forust.xyz
+    - www.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: xdfnx-homepage-prod-tls
+  namespace: homepages
+spec:
+  secretName: xdfnx-homepage-prod-tls
+  dnsNames:
+    - xdfnx.cfd
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: homepages
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

homepages/k8s/homepages.yaml

@@ -27,16 +27,22 @@ spec:
     spec:
       containers:
         - name: forust-homepage
-          image: gcr.forust.xyz/forust/forust-homepage:latest
-          imagePullPolicy: Always
+          image: gcr.forust.xyz/forust/forust-homepage:prod
           ports:
             - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 80
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
           resources:
             requests:
-              memory: "10Mi"
+              memory: "32Mi"
               cpu: "20m"
             limits:
-              memory: "100Mi"
+              memory: "128Mi"
               cpu: "50m"
 ---
 apiVersion: v1
@@ -68,14 +74,20 @@ spec:
     spec:
       containers:
         - name: xdfnx-homepage
-          image: gcr.forust.xyz/forust/xdfnx-homepage:latest
-          imagePullPolicy: Always
+          image: gcr.forust.xyz/forust/xdfnx-homepage:prod
           ports:
             - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 80
+            periodSeconds: 10
+            timeoutSeconds: 2
+            failureThreshold: 3
           resources:
             requests:
-              memory: "10Mi"
+              memory: "32Mi"
               cpu: "20m"
             limits:
-              memory: "100Mi"
+              memory: "128Mi"
               cpu: "50m"

homepages/k8s/ingress.yaml

@@ -9,12 +9,15 @@ spec:
   routes:
     - match: Host(`forust.xyz`) || Host(`www.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       priority: 10
       services:
         - name: forust-homepage-service
           port: 80
   tls:
-    certResolver: letsencrypt
+    secretName: forust-homepage-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -31,6 +34,9 @@ spec:
       services:
         - name: forust-homepage-service
           port: 80
+  tls:
+    secretName: internal-wildcard-tls
+
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -43,11 +49,14 @@ spec:
   routes:
     - match: Host(`xdfnx.cfd`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: xdfnx-homepage-service
           port: 80
   tls:
-    certResolver: letsencrypt
+    secretName: xdfnx-homepage-prod-tls
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -63,3 +72,5 @@ spec:
       services:
         - name: xdfnx-homepage-service
           port: 80
+  tls:
+    secretName: internal-wildcard-tls

immich/.env.example

@@ -0,0 +1,24 @@
+# You can find documentation for all the supported env variables at https://docs.immich.app/install/environment-variables
+
+# The location where your uploaded files are stored. The k8s manifests bind
+# mount /mnt/immich/library, which is the sdc9 partition - the same place, so
+# the two deployment paths look at one library.
+UPLOAD_LOCATION=/mnt/immich/library
+
+# The location where your database files are stored. Network shares are not supported for the database
+DB_DATA_LOCATION=./postgres
+
+# To set a timezone, uncomment the next line and change Etc/UTC to a TZ identifier from this list: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List
+# TZ=Etc/UTC
+
+# The Immich version to use. You can pin this to a specific version like "v2.1.0"
+IMMICH_VERSION=v3
+
+# Connection secret for postgres. You should change it to a random password
+# Please use only the characters `A-Za-z0-9`, without special characters or spaces
+DB_PASSWORD=postgres
+
+# The values below this line do not need to be changed
+###################################################################################
+DB_USERNAME=postgres
+DB_DATABASE_NAME=immich

immich/compose.yaml

@@ -0,0 +1,63 @@
+name: immich
+
+services:
+  immich-server:
+    container_name: immich_server
+    image: ghcr.io/immich-app/immich-server:v3
+    volumes:
+      - ${UPLOAD_LOCATION}:/data
+      - /etc/localtime:/etc/localtime:ro
+    env_file:
+      - .env
+    ports:
+      - "2283:2283"
+    depends_on:
+      - redis
+      - database
+    restart: always
+    healthcheck:
+      disable: false
+
+  immich-machine-learning:
+    container_name: immich_machine_learning
+    # For hardware acceleration, add one of -[armnn, cuda, rocm, openvino, rknn] to the image tag.
+    # Example tag: ${IMMICH_VERSION:-release}-cuda
+    image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release}
+    # extends: # uncomment this section for hardware acceleration - see https://docs.immich.app/features/ml-hardware-acceleration
+    #   file: hwaccel.ml.yml
+    #   service: cpu # set to one of [armnn, cuda, rocm, openvino, openvino-wsl, rknn] for accelerated inference - use the `-wsl` version for WSL2 where applicable
+    volumes:
+      - model-cache:/cache
+    env_file:
+      - .env
+    restart: always
+    healthcheck:
+      disable: false
+
+  redis:
+    container_name: immich_redis
+    image: docker.io/valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd
+    healthcheck:
+      test: redis-cli ping | grep -q PONG || exit 1
+    restart: always
+
+  database:
+    container_name: immich_postgres
+    image: ghcr.io/immich-app/postgres:14-vectorchord0.4.3-pgvectors0.2.0@sha256:bcf63357191b76a916ae5eb93464d65c07511da41e3bf7a8416db519b40b1c23
+    environment:
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_USER: ${DB_USERNAME}
+      POSTGRES_DB: ${DB_DATABASE_NAME}
+      POSTGRES_INITDB_ARGS: "--data-checksums"
+    # Uncomment the DB_STORAGE_TYPE: 'HDD' var if your database isn't stored on SSDs
+    # DB_STORAGE_TYPE: 'HDD'
+    volumes:
+      # Do not edit the next line. If you want to change the database storage location on your system, edit the value of DB_DATA_LOCATION in the .env file
+      - ${DB_DATA_LOCATION}:/var/lib/postgresql/data
+    shm_size: 128mb
+    restart: always
+    healthcheck:
+      disable: false
+
+volumes:
+  model-cache:

immich/k8s/certificates.yaml

@@ -0,0 +1,28 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: immich-prod-tls
+  namespace: immich
+spec:
+  secretName: immich-prod-tls
+  dnsNames:
+    - immich.forust.xyz
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: internal-wildcard-tls
+  namespace: immich
+spec:
+  secretName: internal-wildcard-tls
+  dnsNames:
+    - "*.workstation.internal"
+    - "*.gigaforust.internal"
+    - workstation.internal
+    - gigaforust.internal
+  issuerRef:
+    name: internal-ca
+    kind: ClusterIssuer

immich/k8s/config.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: immich-config
+  namespace: immich
+data:
+  TZ: "Europe/Bratislava"
+
+  # The database in this namespace, not the shared one in the database
+  # namespace: v3 needs VectorChord, and only the dedicated image carries it.
+  DB_HOSTNAME: "immich-postgres"
+  DB_PORT: "5432"
+  DB_USERNAME: "immich"
+  DB_DATABASE_NAME: "immich"
+  DB_SSL_MODE: "disable"
+  DB_VECTOR_EXTENSION: "vectorchord"
+
+  REDIS_HOSTNAME: "immich-valkey"
+  REDIS_PORT: "6379"
+
+  # Traefik is the only client of the server, and it is a pod: the address immich
+  # sees is inside the node's pod CIDR. Without this the server does not trust
+  # X-Forwarded-For and every request looks like it came from Traefik itself.
+  IMMICH_TRUSTED_PROXIES: "10.244.0.0/24"

immich/k8s/immich.yaml

@@ -0,0 +1,95 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-service
+  namespace: immich
+spec:
+  selector:
+    app: immich
+  ports:
+    - name: http
+      port: 2283
+      targetPort: 2283
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-deployment
+  namespace: immich
+  labels:
+    app: immich
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: immich
+  template:
+    metadata:
+      labels:
+        app: immich
+    spec:
+      containers:
+        - name: immich
+          image: ghcr.io/immich-app/immich-server:v3
+          envFrom:
+            - configMapRef:
+                name: immich-config
+            - secretRef:
+                name: immich-secrets
+          ports:
+            - name: http
+              containerPort: 2283
+          volumeMounts:
+            - name: immich-data
+              mountPath: /data
+          # The first boot runs migrations and warms the transcoder, which can
+          # take minutes, so liveness has to wait on the startup probe.
+          startupProbe:
+            httpGet:
+              path: /api/server/ping
+              port: http
+            failureThreshold: 60
+            periodSeconds: 10
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /api/server/ping
+              port: http
+            periodSeconds: 10
+            timeoutSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /api/server/ping
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          # Only the request is scheduled against, and the node is already
+          # oversubscribed (5.58 of 6 cores requested) while actually running
+          # at about 1.5. So the request states what this sits at while idle -
+          # tens of millicores - and the limit leaves room for the burst that
+          # matters: thumbnails, transcodes and metadata extraction.
+          #
+          # The limit used to be 2Gi, but the server OOMKilled on boot while
+          # chewing through a backlog of unprocessed assets (API +Workers in
+          # one container spike well past idle).
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "512Mi"
+            limits:
+              cpu: "1500m"
+              memory: "4Gi"
+      volumes:
+        - name: immich-data
+          # The library lives on the node's own disk, not in a PVC. A PVC here
+          # meant declaring a size up front for data that does not exist yet,
+          # on a provisioner that cannot grow it, and the only copy of the
+          # photos was one `kubectl delete namespace` away.
+          #
+          # Directory, not DirectoryOrCreate, on purpose: if sdc9 is not
+          # mounted, this must fail loudly instead of quietly writing the
+          # library onto the root filesystem.
+          hostPath:
+            path: /mnt/immich/library
+            type: Directory

immich/k8s/ingress.yaml

@@ -0,0 +1,36 @@
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: immich-prod
+  namespace: immich
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`immich.forust.xyz`)
+      kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
+      services:
+        - name: immich-service
+          port: 2283
+  tls:
+    secretName: immich-prod-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: immich-local
+  namespace: immich
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`immich.workstation.internal`) || Host(`immich.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: immich-service
+          port: 2283
+  tls:
+    secretName: internal-wildcard-tls