chore(deps): update container patch updates

.gitea/actionlint.yaml

@@ -1,10 +0,0 @@
-# actionlint configuration. Passed explicitly from the ci workflow:
-#   actionlint -config-file .gitea/actionlint.yaml .gitea/workflows/*.yaml
-#
-# The self-hosted act_runner registers custom labels that actionlint cannot know
-# about, so declare them here instead of silencing the whole runner-label check.
-self-hosted-runner:
-  labels:
-    - arch
-    - homelab
-    - prod

.gitea/workflows/ci.yaml

@@ -7,112 +7,19 @@ on:
   pull_request:
   workflow_dispatch:
 
-# Every job here is checkout plus local tools. The token needs to read the tree
-# and nothing else, and saying so keeps a future step that reaches for the API
-# from quietly holding a token that can write to the repository.
-permissions:
-  contents: read
-
-concurrency:
-  group: ci-${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
-
 env:
   REGISTRY: gcr.forust.xyz
 
 jobs:
-  lint-compose:
-    runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 10
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-
-      # Structure check for every committed Compose file, active or not.
-      # Interpolation, env-file and bind-mount resolution are all switched off,
-      # because inactive stacks have no .env here and would only fail on their
-      # ${VAR:?} guards. Active stacks get the full check with interpolation in
-      # the deploy workflow, where the real .env files live.
-      - name: Validate Compose files
-        shell: bash
-        run: |
-          set -euo pipefail
-          source .gitea/workflows/compose-lint.sh
-
-          mapfile -t safe_flags < <(compose_safe_flags)
-          echo "docker compose config ${safe_flags[*]-}"
-
-          mapfile -t files < <(compose_files)
-          if [ "${#files[@]}" -eq 0 ]; then
-            echo "No Compose files found."
-            exit 0
-          fi
-
-          failed=0
-          for f in "${files[@]}"; do
-            if ! out="$(validate_compose_file "$f" ${safe_flags[@]+"${safe_flags[@]}"} 2>&1)"; then
-              failed=1
-              echo "::error file=${f}::$(printf '%s' "$out" | head -1)"
-            fi
-          done
-
-          if [ "$failed" -ne 0 ]; then
-            echo "Compose validation failed."
-            exit 1
-          fi
-          echo "checked ${#files[@]} Compose file(s)"
-
-  lint-actionlint:
-    runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 10
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-
-      - name: Lint Gitea Actions workflows with actionlint
-        shell: bash
-        run: |
-          set -euo pipefail
-          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh actionlint)"
-          export PATH="$tools_dir:$PATH"
-          actionlint -config-file .gitea/actionlint.yaml -color .gitea/workflows/*.yaml
-
-  lint-shellcheck:
-    runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 10
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-
-      - name: Lint shell scripts with ShellCheck
-        shell: bash
-        run: |
-          set -euo pipefail
-          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh shellcheck)"
-          export PATH="$tools_dir:$PATH"
-          mapfile -t scripts < <(
-            git ls-files '*.sh' ':(glob)**/*.bash'
-          )
-          if [ "${#scripts[@]}" -eq 0 ]; then
-            echo "No shell scripts found."
-            exit 0
-          fi
-          shellcheck --external-sources --source-path=SCRIPTDIR --severity=style "${scripts[@]}"
-
   lint-prettier:
     runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
 
       - name: Check formatting with Prettier
         shell: bash
         run: |
-          set -euo pipefail
-          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh prettier)"
-          export PATH="$tools_dir:$PATH"
-
           mapfile -t prettier_files < <(
             git ls-files \
               | grep -E '\.(md|json|ya?ml|html|css)$' \
@@ -124,38 +31,36 @@ jobs:
             exit 0
           fi
 
-          prettier --check --ignore-unknown "${prettier_files[@]}"
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            node:22-alpine \
+            sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
 
   lint-ruff:
     runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
 
-      - name: Lint and format-check Python with Ruff
+      - name: Lint Python with Ruff
         shell: bash
         run: |
-          set -euo pipefail
-          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh ruff)"
-          export PATH="$tools_dir:$PATH"
-          ruff check .
-          ruff format --check .
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            ghcr.io/astral-sh/ruff:latest \
+            check .
 
   lint-yaml:
     runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
 
       - name: Lint YAML syntax
         shell: bash
         run: |
-          set -euo pipefail
-          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh yamllint)"
-          export PATH="$tools_dir:$PATH"
-
           mapfile -t yaml_files < <(
             git ls-files '*.yaml' '*.yml' \
               ':!node_modules/**' \
@@ -167,22 +72,21 @@ jobs:
             exit 0
           fi
 
-          yamllint -c .yamllint "${yaml_files[@]}"
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            cytopia/yamllint:latest \
+            -c .yamllint "${yaml_files[@]}"
 
   lint-dockerfiles:
     runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
 
       - name: Lint Dockerfiles
         shell: bash
         run: |
-          set -euo pipefail
-          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh hadolint)"
-          export PATH="$tools_dir:$PATH"
-
           mapfile -t dockerfiles < <(
             git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
           )
@@ -192,22 +96,22 @@ jobs:
             exit 0
           fi
 
-          hadolint -c .hadolint.yaml "${dockerfiles[@]}"
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            --entrypoint hadolint \
+            hadolint/hadolint:latest-debian \
+            -c .hadolint.yaml "${dockerfiles[@]}"
 
   validate:
     runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 20
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
 
-      - name: Validate Kubernetes manifests against JSON schemas
+      - name: Validate Kubernetes manifests
         shell: bash
         run: |
-          set -euo pipefail
-          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh kubeconform)"
-          export PATH="$tools_dir:$PATH"
-
           mapfile -t manifests < <(
             git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
               | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
@@ -218,113 +122,24 @@ jobs:
             exit 0
           fi
 
-          kubeconform \
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            ghcr.io/yannh/kubeconform:latest \
             -strict \
             -ignore-missing-schemas \
             -summary \
             "${manifests[@]}"
 
-      # kubeconform has no schemas for CRDs, so every IngressRoute, Certificate,
-      # PrometheusRule, Middleware, ServersTransport and ServiceMonitor is silently
-      # skipped above. The live API server knows the real CRD schemas (and runs the
-      # cert-manager / Traefik admission webhooks), so validate there too.
-      #
-      # Only services marked with a k8s/active marker are checked: server-side
-      # dry-run needs the target namespace to exist, and inactive services are not
-      # deployed. Services being enabled for the first time are still covered by
-      # the JSON-schema pass above.
-      #
-      # Main pushes only. `--dry-run=server` persists nothing, but it does execute
-      # the admission webhooks of the production API server, so anyone able to open
-      # a pull request would be able to run arbitrary manifest content through
-      # cert-manager and Traefik. A pull request has nothing to gain from it either:
-      # only main is ever deployed, and this job runs to completion before the
-      # deploy workflow is allowed to start, so a bad CRD is still caught before
-      # anything reaches the cluster -- just on the push rather than on the PR.
-      - name: Note the server-side check is not running here
-        if: github.event_name == 'pull_request' || github.ref != 'refs/heads/main'
-        shell: bash
-        run: |
-          echo "::notice::Skipping the server-side dry-run. It executes the cert-manager and" \
-            "Traefik admission webhooks against the production API server, so it is limited" \
-            "to pushes to main. CRDs are still schema-checked by kubeconform above, and the" \
-            "server-side pass still runs on main before the deploy."
-
-      - name: Validate active manifests against the live API server
-        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
-        shell: bash
-        run: |
-          set -euo pipefail
-
-          if ! kubectl get --raw='/readyz' --request-timeout=10s >/dev/null 2>&1; then
-            echo "::warning::Cluster unreachable — skipped server-side validation of CRDs (IngressRoute, Certificate, PrometheusRule). Review manifest changes manually."
-            exit 0
-          fi
-
-          mapfile -t k8s_dirs < <(
-            git ls-files '*.yaml' '*.yml' \
-              | grep -E '(^|/)k8s/' \
-              | sed -E 's#((^|.*/)k8s)/.*#\1#' \
-              | sort -u
-          )
-
-          manifests=()
-          kustomize_apps=()
-          for dir in "${k8s_dirs[@]}"; do
-            if [ ! -f "${dir}/active" ]; then
-              echo "skip (no k8s/active): ${dir}"
-              continue
-            fi
-            if [ -f "${dir}/overlays/prod/kustomization.yaml" ]; then
-              kustomize_apps+=("${dir}/overlays/prod")
-            elif [ -f "${dir}/base/kustomization.yaml" ]; then
-              kustomize_apps+=("${dir}/base")
-            else
-              while IFS= read -r f; do
-                [ -n "$f" ] && manifests+=("$f")
-              done < <(
-                git ls-files "${dir}/*.yaml" "${dir}/*.yml" \
-                  | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
-              )
-            fi
-          done
-
-          echo "server-side dry-run: ${#manifests[@]} manifests, ${#kustomize_apps[@]} kustomize apps"
-          failed=0
-          for m in ${manifests[@]+"${manifests[@]}"}; do
-            if ! out="$(kubectl apply --dry-run=server -f "$m" 2>&1)"; then
-              failed=1
-              echo "::error file=${m}::$(printf '%s' "$out" | head -1)"
-            fi
-          done
-          for k in ${kustomize_apps[@]+"${kustomize_apps[@]}"}; do
-            if ! out="$(kubectl apply -k "$k" --dry-run=server 2>&1)"; then
-              failed=1
-              echo "::error file=${k}::$(printf '%s' "$out" | head -1)"
-            fi
-          done
-
-          if [ "$failed" -ne 0 ]; then
-            echo "Server-side validation failed. The API server (or an admission webhook) rejected these manifests."
-            exit 1
-          fi
-          echo "server-side dry-run: all active manifests accepted by the API server"
-
   build:
-    needs:
-      # The panel's scan-deps/test-backend/test-frontend jobs gated here until
-      # userbot moved to its own repo; upstream's code is upstream's gate now.
-      # The rule is unchanged: publishing and passing the checks are the same
-      # gate, so a commit that fails any of these still cannot move :prod.
-      [lint-actionlint, lint-shellcheck, lint-compose, lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
-    if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev') && !startsWith(github.ref_name, 'renovate/')
+    needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
+    if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
     runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 60
     outputs:
       services: ${{ steps.services.outputs.services }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -332,20 +147,12 @@ jobs:
         id: services
         shell: bash
         run: |
-          set -euo pipefail
           base="${{ github.event.before }}"
           if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
             base="$(git rev-list --max-parents=0 HEAD)"
           fi
 
-          # A failed diff used to leave changed_files empty, which reads exactly
-          # like "nothing to build": the job went green having built nothing and
-          # the tag never moved. The status is checked, not assumed.
-          if ! changed="$(git diff --name-only "$base" "${GITHUB_SHA}")"; then
-            echo "::error::cannot diff ${base}..${GITHUB_SHA}"
-            exit 1
-          fi
-          mapfile -t changed_files <<<"$changed"
+          mapfile -t changed_files < <(git diff --name-only "$base" "${GITHUB_SHA}")
 
           services=()
 
@@ -365,9 +172,15 @@ jobs:
 
           for file in "${changed_files[@]}"; do
             case "$file" in
+              dtek_notif/*)
+                add_service dtek_notif
+                ;;
               errorpages/*)
                 add_service errorpages
                 ;;
+              userbot/*)
+                add_service userbot
+                ;;
               homepages/*)
                 add_service homepages
                 ;;
@@ -387,78 +200,95 @@ jobs:
           echo "services=$(paste -sd, /tmp/services.txt)" >> "$GITHUB_OUTPUT"
 
       - name: Log in to registry
-        # The pin step below also writes (manifest PUTs), and it runs on every
-        # main push — including manifest-only ones where services is empty. A
-        # stale persistent login on the old runner used to mask this; a clean
-        # runner pushes anonymously and gets 401.
-        if: steps.services.outputs.services != '' || github.ref_name == 'main'
+        if: steps.services.outputs.services != ''
         shell: bash
-        # Through env, not by substitution into the script. A secret written
-        # into a run: block is pasted into the shell source before bash parses
-        # it, so a password containing a quote, a backtick or $(...) becomes
-        # code that runs. Masking the value in the log does not prevent that.
-        env:
-          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
         run: |
-          set -euo pipefail
-          printf '%s' "$REGISTRY_PASSWORD" | docker login "${REGISTRY}" \
-            -u "$REGISTRY_USERNAME" \
+          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
+            -u "${{ secrets.REGISTRY_USERNAME }}" \
             --password-stdin
 
       - name: Build and push changed images
         if: steps.services.outputs.services != ''
         shell: bash
         run: |
-          # This step was the one run: block in the workflow without it, and it
-          # is the one that cannot afford it: a docker push that failed partway
-          # through the loop used to be followed by more pushes, the loop's exit
-          # status came from the last one, and the job went green with half the
-          # images missing from the registry.
-          set -euo pipefail
           IFS=, read -r -a services <<< "${{ steps.services.outputs.services }}"
 
-          # Tags for this push. The commit-pinned name is the point of this
-          # step: the deploy resolves it in preference to :prod, so a deploy
-          # that sat in the queue behind a later push still gets the build of
-          # the commit CI validated, instead of whatever :prod points at by the
-          # time it runs. See render_pinned in deploy-lib.sh.
-          commit_tag=""
-          if [ "${GITHUB_REF_NAME}" = "main" ]; then
-            commit_tag="sha-${GITHUB_SHA:0:12}"
-          fi
-
-          set_tags() {
-            tags=()
-            case "${GITHUB_REF_NAME}" in
-              main) tags+=("main" "prod") ;;
-              dev) tags+=("dev") ;;
-            esac
-            if [ -n "$commit_tag" ]; then
-              tags+=("$commit_tag")
-            fi
-          }
-
           for service in "${services[@]}"; do
             case "$service" in
-              errorpages)
-                image="${REGISTRY}/forust/error-pages"
-                set_tags
+              dtek_notif)
+                image="${REGISTRY}/forust/dtek-notif"
+                tags=("latest")
+                case "${GITHUB_REF_NAME}" in
+                  main)
+                    tags+=("main" "prod")
+                    ;;
+                  dev)
+                    tags+=("dev")
+                    ;;
+                esac
                 build_args=()
                 for tag in "${tags[@]}"; do
                   build_args+=(-t "${image}:${tag}")
                 done
-                docker build \
-                  --cache-from "type=registry,ref=${image}:buildcache" \
-                  --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
-                  "${build_args[@]}" errorpages
+                docker build "${build_args[@]}" dtek_notif
                 for tag in "${tags[@]}"; do
                   docker push "${image}:${tag}"
                 done
                 ;;
+              errorpages)
+                image="${REGISTRY}/forust/error-pages"
+                tags=("latest")
+                case "${GITHUB_REF_NAME}" in
+                  main)
+                    tags+=("main" "prod")
+                    ;;
+                  dev)
+                    tags+=("dev")
+                    ;;
+                esac
+                build_args=()
+                for tag in "${tags[@]}"; do
+                  build_args+=(-t "${image}:${tag}")
+                done
+                docker build "${build_args[@]}" errorpages
+                for tag in "${tags[@]}"; do
+                  docker push "${image}:${tag}"
+                done
+                ;;
+              userbot)
+                tags=("latest")
+                case "${GITHUB_REF_NAME}" in
+                  main)
+                    tags+=("main" "prod")
+                    ;;
+                  dev)
+                    tags+=("dev")
+                    ;;
+                esac
+                for target in runtime panel; do
+                  case "$target" in
+                    runtime)
+                      context="userbot"
+                      image="${REGISTRY}/forust/userbot"
+                      ;;
+                    panel)
+                      context="userbot/panel"
+                      image="${REGISTRY}/forust/userbot-panel"
+                      ;;
+                  esac
+                  build_args=()
+                  for tag in "${tags[@]}"; do
+                    build_args+=(-t "${image}:${tag}")
+                  done
+                  docker build "${build_args[@]}" "$context"
+                  for tag in "${tags[@]}"; do
+                    docker push "${image}:${tag}"
+                  done
+                done
+                ;;
               homepages)
-                for variant in forust xdfnx; do
-                  case "$variant" in
+                for service in forust xdfnx; do
+                  case "$service" in
                     forust)
                       image="${REGISTRY}/forust/forust-homepage"
                       ;;
@@ -466,23 +296,28 @@ jobs:
                       image="${REGISTRY}/forust/xdfnx-homepage"
                       ;;
                   esac
-                  set_tags
+                  tags=("latest")
+                  case "${GITHUB_REF_NAME}" in
+                    main)
+                      tags+=("main" "prod")
+                      ;;
+                    dev)
+                      tags+=("dev")
+                      ;;
+                  esac
                   build_args=()
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
                   done
-                  docker build \
-                    --cache-from "type=registry,ref=${image}:buildcache" \
-                    --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
-                    "${build_args[@]}" -f "homepages/Dockerfile.${variant}" homepages
+                  docker build "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
                 done
                 ;;
               edu_master)
-                for variant in session-keeper webinar-checker; do
-                  case "$variant" in
+                for service in session-keeper webinar-checker; do
+                  case "$service" in
                     session-keeper)
                       context="edu_master/phpsessid-bot"
                       image="${REGISTRY}/forust/session-keeper"
@@ -492,15 +327,20 @@ jobs:
                       image="${REGISTRY}/forust/webinar-checker"
                       ;;
                   esac
-                  set_tags
+                  tags=("latest")
+                  case "${GITHUB_REF_NAME}" in
+                    main)
+                      tags+=("main" "prod")
+                      ;;
+                    dev)
+                      tags+=("dev")
+                      ;;
+                  esac
                   build_args=()
                   for tag in "${tags[@]}"; do
                     build_args+=(-t "${image}:${tag}")
                   done
-                  docker build \
-                    --cache-from "type=registry,ref=${image}:buildcache" \
-                    --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
-                    "${build_args[@]}" "$context"
+                  docker build "${build_args[@]}" "$context"
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
@@ -509,41 +349,22 @@ jobs:
             esac
           done
 
-      # Every image the tree names has to carry the commit-pinned name, not only
-      # the ones this push rebuilt. A push that touches nothing but manifests
-      # builds nothing, and its deploy would then find no commit-pinned tag to
-      # resolve and quietly fall back to the moving :prod - which is the whole
-      # failure the commit-pinned name exists to remove.
-      #
-      # Re-tagging copies the manifest list and transfers no layers, so pinning
-      # six images that already exist costs six registry writes.
-      #
-      # The list is derived from the tree rather than written out here, so an
-      # image added to a manifest is covered without a second place to update.
-      - name: Pin the commit name on the images this push did not rebuild
-        if: github.ref_name == 'main'
+  deploy-userbot-panel:
+    needs: build
+    if: github.ref_name == 'main' && contains(needs.build.outputs.services, 'userbot')
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Apply and roll out userbot panel
         shell: bash
         run: |
-          set -euo pipefail
-          commit_tag="sha-${GITHUB_SHA:0:12}"
-          mapfile -t repos < <(
-            git grep -hoE 'gcr\.forust\.xyz/forust/[A-Za-z0-9._-]+' -- '*.yaml' '*.yml' \
-              | sort -u
-          )
-          if [ "${#repos[@]}" -eq 0 ]; then
-            echo "No own images referenced by the tree."
-            exit 0
-          fi
-          echo "pinning ${#repos[@]} image(s) to $commit_tag"
-          for repo in "${repos[@]}"; do
-            if docker buildx imagetools inspect "$repo:$commit_tag" >/dev/null 2>&1; then
-              echo "  already built by this push: ${repo##*/}"
-              continue
-            fi
-            if ! docker buildx imagetools inspect "$repo:prod" >/dev/null 2>&1; then
-              echo "  WARNING: ${repo##*/} has no :prod to pin and no build produced it"
-              continue
-            fi
-            docker buildx imagetools create --tag "$repo:$commit_tag" "$repo:prod"
-            echo "  pinned ${repo##*/}"
-          done
+          kubectl apply -f userbot/k8s/base/panel.yaml
+          kubectl get secret userbot-common-secrets -n default -o json \
+            | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
+            | kubectl apply -f -
+          # Keep legacy deployments (forust/anna) in sync with manifests; they have no replicas field, so apply leaves scaling to the user manager only.
+          kubectl apply -f userbot/k8s/base/userbots.yaml
+          kubectl rollout restart deployment/userbot-panel -n userbot
+          kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s

.gitea/workflows/compose-lint.sh

@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-# Shared helpers for validating Compose files. Sourced both by steps in
-# .gitea/workflows/ci.yaml and by deploy-lib.sh on the workstation.
-#
-# Two levels of checking, matching how the repo is structured:
-#
-#   general  every committed Compose file, active or not. Pure structure check:
-#            no ${VAR} interpolation, no .env lookup, no bind-mount path
-#            resolution. Disabled stacks deliberately have no .env in the repo
-#            and no values on the CI runner, so a full `config` run would fail on
-#            their `${VAR:?}` guards for reasons that have nothing to do with the
-#            change under review.
-#
-#   full     active stacks only, with interpolation and env-file resolution, so
-#            required variables and referenced files are actually resolved. Needs
-#            the gitignored .env files, so this only runs in the deploy workflow
-#            on the workstation.
-#
-# This file is meant to be sourced, not executed.
-
-# All committed Compose files, including the ones deploy never starts.
-compose_files() {
-  git ls-files \
-    '*/compose.yaml' '*/compose.yml' 'compose.yaml' 'compose.yml' \
-    '*/docker-compose.yaml' '*/docker-compose.yml'
-}
-
-# Prints the flags that turn `docker compose config` into the general check.
-# Probed rather than hardcoded so an older Compose without --no-env-resolution
-# still gets the flags it does support.
-compose_safe_flags() {
-  local help flag
-  help="$(docker compose config --help 2>/dev/null || true)"
-  for flag in --no-interpolate --no-env-resolution --no-path-resolution; do
-    if printf '%s' "$help" | grep -q -- "$flag"; then
-      printf '%s\n' "$flag"
-    fi
-  done
-}
-
-# validate_compose_file <file> [extra docker compose config flags...]
-validate_compose_file() {
-  local file="$1"
-  shift
-  docker compose -f "$file" config --quiet "$@"
-}

.gitea/workflows/deploy.yaml

@@ -1,197 +1,152 @@
 name: deploy
 
 on:
-  # Deploy only what CI already validated. workflow_run is used instead of
-  # workflow_dispatch so a red lint/validate run can never reach the cluster.
-  workflow_run:
-    workflows: [ci]
-    types: [completed]
   workflow_dispatch:
 
-# The deploy jobs read the tree, then reach the cluster over SSH with the
-# deploy key. The Actions token itself is not part of that path, so it gets
-# read-only contents and no more.
-permissions:
-  contents: read
-
 concurrency:
   group: deploy-main
-  # Queue instead of cancelling. Cancelling a run kills the apply job mid-loop and
-  # takes the verify job down with it, so a superseded deploy would leave the
-  # cluster half-applied and unchecked — the exact failure the verify job exists
-  # to catch. kubectl apply and docker compose up are both idempotent, so letting
-  # the older run finish and then deploying the newer commit costs little.
   cancel-in-progress: false
 
-env:
+jobs:
+  redeploy:
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    steps:
+      - name: Redeploy workstation
+        shell: bash
+        env:
           DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
           DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
           DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
           DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
           DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+          # Set APPLY_PRUNE=true to enable kubectl apply --prune. Requires every
+          # manifest to carry label app.kubernetes.io/managed-by=homelab-deploy,
+          # otherwise previously applied resources get deleted on the next run.
           APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
-  # workflow_run's own GITHUB_SHA points at the branch head, not at the commit the
-  # finished ci run checked. Pin the exact validated commit instead, so a push
-  # landing mid-deploy cannot make the workstation deploy something else. Also
-  # what the verify job checks the snapshot against. Empty for workflow_dispatch,
-  # which falls back to the current origin/main.
-  DEPLOY_SHA: ${{ github.event.workflow_run.head_sha }}
-
-jobs:
-  preflight:
-    # Autodeploy defaults to OFF: pushes deploy only when the AUTODEPLOY repo
-    # variable is set to 'true' (Settings -> Actions -> Variables). A manual
-    # Run workflow always bypasses the switch: dispatching it is the explicit
-    # intent to deploy.
-    if: >-
-      (vars.AUTODEPLOY == 'true' || github.event_name == 'workflow_dispatch') &&
-      (github.event_name != 'workflow_run' ||
-      (github.event.workflow_run.conclusion == 'success' &&
-       github.event.workflow_run.head_branch == 'main'))
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    timeout-minutes: 10
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-
-      - name: Fetch and reset workstation
-        shell: bash
         run: |
           set -euo pipefail
-          ./.gitea/workflows/ssh-run.sh preflight
 
-  validate:
-    needs: [preflight]
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    timeout-minutes: 20
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+          : "${DEPLOY_HOST:?missing DEPLOY_HOST}"
+          : "${DEPLOY_USER:?missing DEPLOY_USER}"
+          : "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
 
-      - name: Dry-run manifests and check Secrets
-        shell: bash
-        run: |
+          deploy_port="${DEPLOY_PORT:-22}"
+          deploy_path="${DEPLOY_PATH:-/srv/homelab}"
+
+          ssh_key="$RUNNER_TEMP/deploy_key"
+          mkdir -p "$RUNNER_TEMP"
+          printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
+          chmod 600 "$ssh_key"
+
+          ssh_opts=(
+            -i "$ssh_key"
+            -p "$deploy_port"
+            -o BatchMode=yes
+            -o StrictHostKeyChecking=accept-new
+          )
+
+          ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
+            "DEPLOY_PATH=$(printf '%q' \"$deploy_path\") APPLY_PRUNE=$(printf '%q' \"${APPLY_PRUNE:-false}\") bash -se" <<'EOF'
           set -euo pipefail
-          ./.gitea/workflows/ssh-run.sh validate
 
-  apply-k8s:
-    needs: [validate]
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    # Apply only, no verification, so this is just the work itself: snapshot,
-    # then sequential `helm upgrade --install --wait --rollback-on-failure --timeout 10m`, then the apply loop.
-    # Verification has its own job and its own budget.
-    #
-    # 45 is roughly four times the measured cost of the stage, which is
-    # deliberately not raised on a theory:
-    #
-    #   helm, healthy          3 no-op upgrades              ~3-5 min
-    #   helm, one release bad  rollback-on-failure spends its 10m,      ~10-15 min
-    #                            then rolls that one back
-    #   apply loop             ~40 manifests, 4 of which     ~1 min
-    #                           resolve an image digest
-    #   restart_stale_images   7.6s to find 8 workloads,     ~0.5 min
-    #                           9.8s to resolve their digests
-    #
-    # The helm figure is one release, not three: `set -e` aborts
-    # upgrade_helm_releases on the first failure, so a broken release costs
-    # 10m and the other two are never attempted. Multiplying 10m by three
-    # overstates the worst case by 20 minutes.
-    #
-    # The 45 minutes this was last raised to 45 were still not enough, and the
-    # job logs for those runs no longer exist, so what actually consumed the
-    # budget is not known - the two measurable candidates above account for
-    # ~15 of it. The unbounded `docker manifest inspect` against the registry's
-    # known hang mode is now bounded inside registry_digest (25s timeout, 3
-    # attempts): a dead registry fails each owned image after ~85s instead of
-    # hanging the stage, and a blinking one is retried instead of failing the
-    # whole apply file. Still open: make the stage announce which manifest it
-    # is working on, so a killed run leaves a diagnosable last line.
-    timeout-minutes: 45
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+          repo="${DEPLOY_PATH:-/srv/homelab}"
 
-      - name: Apply Kubernetes manifests
-        shell: bash
-        run: |
-          set -euo pipefail
-          ./.gitea/workflows/ssh-run.sh apply-k8s
+          if [ ! -d "$repo/.git" ]; then
+            echo "Repository not found at $repo"
+            exit 1
+          fi
 
-  apply-compose:
-    needs: [validate]
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    timeout-minutes: 30
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+          git -C "$repo" fetch origin main
+          git -C "$repo" reset --hard origin/main
 
-      - name: Redeploy docker compose stacks
-        shell: bash
-        run: |
-          set -euo pipefail
-          ./.gitea/workflows/ssh-run.sh apply-compose
+          # Runtime selection: a service is k8s-managed when $SERVICE/k8s/active
+          # exists. Otherwise it is compose-managed, and only k8s/routing/*
+          # manifests (external Services / EndpointSlices / ServersTransport /
+          # Ingresses that route to docker backends) are applied.
+          #   migrate:   touch SERVICE/k8s/active   (+ move routing files up)
+          #   rollback:  rm SERVICE/k8s/active
+          collect_k8s() {
+            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
+              ! -path '*/routing/*' ! -path '*/overlays/*' \
+              ! -name 'kustomization.y*ml' ! -name '*.example.y*ml' \
+              ! -name '*values.y*ml' ! -name 'patch-*.y*ml' \
+              | sort
+          }
 
-  # Watches the workloads this deploy changed and rolls back the ones that never
-  # became healthy. Runs even when the apply jobs failed, timed out or were
-  # cancelled — that is the whole point of splitting it out. `always()` is what
-  # lets it start after a failed dependency; the needs on apply-compose are a
-  # barrier, so verification begins only once both applies are done.
-  verify-k8s:
-    needs: [apply-k8s, apply-compose]
-    if: >-
-      always() &&
-      needs.apply-k8s.result != 'skipped' &&
-      needs.apply-compose.result != 'skipped'
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    # Not raised, because the arithmetic does not close.
-    #
-    # 32 workloads are under management and the wave width is 8, so the verify
-    # itself is 4 waves of ROLLOUT_TIMEOUT (300s) = 20 minutes worst case, when
-    # every rollout times out rather than converging. That is already 20 of 30.
-    #
-    # The other 10 would have to absorb rollback, and rollback_workloads is a
-    # serial `while read` loop at 300s per failed workload. 10 minutes buys two.
-    # Any larger number is buying a bigger multiple of an unbounded term rather
-    # than covering a known cost: 60 minutes buys eight, and 60 minutes is
-    # therefore not a bound, it is a guess with two digits.
-    #
-    # The number becomes derivable the moment rollback uses the same wave width
-    # as the verify: 32 failures then cost 4 waves = 20 minutes instead of 160,
-    # and 45 covers verify plus rollback at full width. That change is to the
-    # recovery path and is not folded into a timeout edit.
-    timeout-minutes: 30
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+          collect_k8s_inactive() {
+            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
+              \( -name 'namespace.y*ml' -o -path '*/routing/*' \) \
+              ! -path '*/overlays/*' ! -name '*.example.y*ml' \
+              | sort
+          }
 
-      - name: Verify workloads and roll back on failure
-        shell: bash
-        run: |
-          set -euo pipefail
-          ./.gitea/workflows/ssh-run.sh verify-k8s
+          mapfile -t compose_stacks < <(
+            find "$repo" -type f \( -name 'compose.yaml' -o -name 'compose.yml' \) | sort
+          )
 
-  # Asks the public route of every active service whether it is actually
-  # serving, which the rollout check above structurally cannot: a pod can
-  # converge and still be crash-looping, or be listening on a port no Service
-  # points at, or answer 500.
-  #
-  # `always()` for the same reason verify-k8s has it, and it runs after that job
-  # specifically because a rollback is when a route most needs re-checking. The
-  # needs is a barrier, not a filter: whether verify-k8s passed, failed or was
-  # cancelled, the probes are what say whether the cluster is serving, and
-  # suppressing them on a rollback would hide the one run where the answer
-  # matters most.
-  smoke:
-    needs: [verify-k8s]
-    if: always() && needs.verify-k8s.result != 'skipped'
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    timeout-minutes: 10
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+          mapfile -t k8s_manifests < <(
+            for kd in $(find "$repo" -type d -name k8s ! -path '*/.git/*' | sort); do
+              if [ -f "$kd/active" ]; then
+                collect_k8s "$kd"
+              else
+                collect_k8s_inactive "$kd"
+              fi
+            done
+          )
 
-      - name: Probe the public route of every active service
-        shell: bash
-        run: |
-          set -euo pipefail
-          ./.gitea/workflows/ssh-run.sh smoke
+          echo "== Validate compose stacks =="
+          for cf in "${compose_stacks[@]}"; do
+            dir=$(dirname "$cf")
+            if [ -f "$dir/k8s/active" ]; then
+              echo "  skip (k8s-managed): $dir"
+              continue
+            fi
+            echo "  config: $cf"
+            docker compose -f "$cf" config --quiet
+          done
+
+          echo "== Validate k8s manifests (kubectl dry-run) =="
+          for m in "${k8s_manifests[@]}"; do
+            echo "  apply --dry-run=client $m"
+            kubectl apply --dry-run=client -f "$m" >/dev/null
+          done
+
+          echo "== Applying Kubernetes manifests =="
+          ns_files=()
+          other_files=()
+          for m in "${k8s_manifests[@]}"; do
+            case "$m" in
+              */namespace.y?ml) ns_files+=("$m") ;;
+              *) other_files+=("$m") ;;
+            esac
+          done
+
+          prune_opts=()
+          if [ "${APPLY_PRUNE:-false}" = "true" ]; then
+            prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
+          fi
+
+          if [ "${#ns_files[@]}" -gt 0 ]; then
+            echo "  namespaces first: ${ns_files[*]}"
+            kubectl apply -f "${ns_files[@]}"
+          fi
+          if [ "${#other_files[@]}" -gt 0 ]; then
+            echo "  resources: ${other_files[*]}"
+            kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
+          fi
+
+          echo "== Redeploying docker compose stacks =="
+          for cf in "${compose_stacks[@]}"; do
+            dir=$(dirname "$cf")
+            if [ -f "$dir/k8s/active" ]; then
+              echo "  skip (k8s-managed): $dir"
+              continue
+            fi
+            echo "  compose: $dir"
+            if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
+              docker compose -f "$cf" build
+              docker compose -f "$cf" push
+            fi
+            docker compose -f "$cf" up -d --pull always --remove-orphans
+          done
+          EOF

.gitea/workflows/install-ci-tools.sh

@@ -1,254 +0,0 @@
-#!/usr/bin/env bash
-# Installs the pinned CI tools into "$TOOLS_DIR/bin" and echoes that directory
-# on stdout, so callers can do:
-#
-#   export PATH="$(bash .gitea/workflows/install-ci-tools.sh kubeconform shellcheck):$PATH"
-#
-# Versions come from tool-versions.env next to this script and are kept fresh by
-# Renovate. Re-running is cheap: an already-installed tool at the pinned version
-# is left alone.
-set -euo pipefail
-
-here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=tool-versions.env
-. "$here/tool-versions.env"
-
-TOOLS_DIR="${TOOLS_DIR:-${RUNNER_TEMP:-/tmp}/homelab-tools}"
-BIN_DIR="$TOOLS_DIR/bin"
-mkdir -p "$BIN_DIR"
-# The just-installed tools must resolve inside this script too: callers only
-# prepend BIN_DIR to PATH after the script exits, so a bare `uv` below would
-# miss the binary install_uv just placed (exit 127 on a clean runner).
-export PATH="$BIN_DIR:$PATH"
-
-arch="$(uname -m)"
-# Upstream projects disagree on arch spelling: kubeconform and actionlint use
-# Go names (amd64/arm64), shellcheck uses uname names (x86_64/aarch64), node
-# uses neither (x64/arm64), and hadolint mixes the two in a single release
-# (x86_64 but arm64).
-case "$arch" in
-  x86_64 | amd64)
-    goarch=amd64
-    sharch=x86_64
-    nodearch=x64
-    hadolintarch=x86_64
-    ;;
-  aarch64 | arm64)
-    goarch=arm64
-    sharch=aarch64
-    nodearch=arm64
-    hadolintarch=arm64
-    ;;
-  *)
-    echo "install-ci-tools: unsupported architecture: $arch" >&2
-    exit 1
-    ;;
-esac
-
-fetch() {
-  # fetch <url> <dest>
-  if command -v curl >/dev/null 2>&1; then
-    curl -sSLf --retry 3 -o "$2" "$1"
-  elif command -v wget >/dev/null 2>&1; then
-    wget -q -O "$2" "$1"
-  else
-    echo "install-ci-tools: neither curl nor wget is available" >&2
-    exit 1
-  fi
-}
-
-# resolve <command>
-# Absolute path to use for invoking a tool: the copy in BIN_DIR when present,
-# otherwise the name for PATH lookup. Every version check and every in-script
-# invocation goes through this, so a tool missing from both places reads as
-# "not installed" instead of dying with 127 under `set -e`.
-resolve() {
-  if [ -x "$BIN_DIR/$1" ]; then
-    printf '%s' "$BIN_DIR/$1"
-  else
-    printf '%s' "$1"
-  fi
-}
-
-# installed_version <command>
-# Prints the version of an already-installed tool, or nothing. Each tool spells
-# its version flag differently, hence the case.
-installed_version() {
-  local bin out
-  bin="$(resolve "$1")"
-  if ! command -v "$bin" >/dev/null 2>&1; then
-    return 0
-  fi
-  case "$1" in
-    kubeconform) out="$("$bin" -v 2>/dev/null | head -1 || true)" ;;
-    *) out="$("$bin" --version 2>/dev/null | head -1 || true)" ;;
-  esac
-  printf '%s' "$out"
-}
-
-# at_version <command> <expected>
-at_version() {
-  case "$(installed_version "$1")" in
-    *"$2"*) return 0 ;;
-    *) return 1 ;;
-  esac
-}
-
-install_kubeconform() {
-  if at_version kubeconform "v${KUBECONFORM_VERSION}"; then
-    return 0
-  fi
-  local tmp
-  tmp="$(mktemp -d)"
-  fetch "https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${goarch}.tar.gz" \
-    "$tmp/kubeconform.tar.gz"
-  tar -xzf "$tmp/kubeconform.tar.gz" -C "$tmp" kubeconform
-  install -m 0755 "$tmp/kubeconform" "$BIN_DIR/kubeconform"
-  rm -rf "$tmp"
-}
-
-install_shellcheck() {
-  if at_version shellcheck "${SHELLCHECK_VERSION}"; then
-    return 0
-  fi
-  local tmp
-  tmp="$(mktemp -d)"
-  fetch "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.${sharch}.tar.xz" \
-    "$tmp/shellcheck.tar.xz"
-  tar -xJf "$tmp/shellcheck.tar.xz" -C "$tmp" --strip-components=1 "shellcheck-v${SHELLCHECK_VERSION}/shellcheck"
-  install -m 0755 "$tmp/shellcheck" "$BIN_DIR/shellcheck"
-  rm -rf "$tmp"
-}
-
-install_uv() {
-  if at_version uv "${UV_VERSION}"; then
-    return 0
-  fi
-  local tmp
-  tmp="$(mktemp -d)"
-  # uv release tags carry no leading v, unlike every other tool installed here.
-  fetch "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-${sharch}-unknown-linux-gnu.tar.gz" \
-    "$tmp/uv.tar.gz"
-  tar -xzf "$tmp/uv.tar.gz" -C "$tmp" --strip-components=1 "uv-${sharch}-unknown-linux-gnu/uv"
-  install -m 0755 "$tmp/uv" "$BIN_DIR/uv"
-  rm -rf "$tmp"
-}
-
-install_hadolint() {
-  if at_version hadolint "${HADOLINT_VERSION}"; then
-    return 0
-  fi
-  # A bare binary, no archive: hadolint ships one file per platform.
-  fetch "https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VERSION}/hadolint-linux-${hadolintarch}" \
-    "$BIN_DIR/hadolint"
-  chmod 0755 "$BIN_DIR/hadolint"
-}
-
-# ruff and yamllint both come from PyPI as wheels, which uv unpacks for us.
-install_uv_tool() {
-  # <package> <pinned version>
-  if at_version "$1" "$2"; then
-    return 0
-  fi
-  install_uv
-  UV_TOOL_BIN_DIR="$BIN_DIR" "$BIN_DIR/uv" tool install --force "$1==$2" >/dev/null
-}
-
-install_ruff() {
-  install_uv_tool ruff "${RUFF_VERSION}"
-}
-
-install_yamllint() {
-  install_uv_tool yamllint "${YAMLLINT_VERSION}"
-}
-
-install_pip_audit() {
-  install_uv_tool pip-audit "${PIP_AUDIT_VERSION}"
-}
-
-install_prettier() {
-  if at_version prettier "${PRETTIER_VERSION}"; then
-    return 0
-  fi
-  # Not a standalone binary: prettier's entry point requires ../package.json
-  # relative to its own real path, so the package directory has to survive
-  # next to it. Hence a versioned directory plus a relative symlink, rather
-  # than copying the one file out as the other installers do.
-  local dir="$BIN_DIR/prettier-${PRETTIER_VERSION}"
-  if [ ! -f "$dir/package/package.json" ]; then
-    rm -rf "$dir"
-    mkdir -p "$dir"
-    fetch "https://registry.npmjs.org/prettier/-/prettier-${PRETTIER_VERSION}.tgz" "$dir/prettier.tgz"
-    tar -xzf "$dir/prettier.tgz" -C "$dir"
-    rm -f "$dir/prettier.tgz"
-    # npm strips the exec bit from bin/ on the way into the tarball.
-    chmod 0755 "$dir/package/bin/prettier.cjs"
-  fi
-  # Relative, so the whole tree stays valid if TOOLS_DIR is relocated.
-  ln -sfn "prettier-${PRETTIER_VERSION}/package/bin/prettier.cjs" "$BIN_DIR/prettier"
-}
-
-install_node() {
-  # npm gets checked by running it, not by looking it up: what matters is that
-  # it answers, so a stub, a half-removed Arch package or a name that resolves
-  # to something broken all have to read as "not installed". The runner's npm
-  # is a symlink into /usr/lib/node_modules/npm, which is exactly the kind of
-  # thing that disappears between runs.
-  if at_version node "v${NODE_VERSION}" && [ -n "$(installed_version npm)" ]; then
-    return 0
-  fi
-  # Same shape as prettier above: the tarball's bin/npm and bin/npx are links
-  # into lib/node_modules, so the whole tree has to survive next to them.
-  local dir="$BIN_DIR/node-${NODE_VERSION}"
-  if [ ! -x "$dir/bin/node" ]; then
-    rm -rf "$dir"
-    mkdir -p "$dir"
-    fetch "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${nodearch}.tar.xz" \
-      "$dir/node.tar.xz"
-    tar -xJf "$dir/node.tar.xz" -C "$dir" --strip-components=1 "node-v${NODE_VERSION}-linux-${nodearch}"
-    rm -f "$dir/node.tar.xz"
-  fi
-  # Relative, so the whole tree stays valid if TOOLS_DIR is relocated.
-  for bin in node npm npx; do
-    ln -sfn "node-${NODE_VERSION}/bin/${bin}" "$BIN_DIR/${bin}"
-  done
-}
-
-install_actionlint() {
-  if at_version actionlint "${ACTIONLINT_VERSION}"; then
-    return 0
-  fi
-  local tmp
-  tmp="$(mktemp -d)"
-  fetch "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_${goarch}.tar.gz" \
-    "$tmp/actionlint.tar.gz"
-  tar -xzf "$tmp/actionlint.tar.gz" -C "$tmp" actionlint
-  install -m 0755 "$tmp/actionlint" "$BIN_DIR/actionlint"
-  rm -rf "$tmp"
-}
-
-wanted=("$@")
-if [ "${#wanted[@]}" -eq 0 ]; then
-  wanted=(kubeconform shellcheck actionlint prettier ruff yamllint hadolint)
-fi
-
-for tool in "${wanted[@]}"; do
-  case "$tool" in
-    kubeconform) install_kubeconform ;;
-    shellcheck) install_shellcheck ;;
-    actionlint) install_actionlint ;;
-    prettier) install_prettier ;;
-    ruff) install_ruff ;;
-    yamllint) install_yamllint ;;
-    pip-audit) install_pip_audit ;;
-    hadolint) install_hadolint ;;
-    node) install_node ;;
-    uv) install_uv ;;
-    *)
-      echo "install-ci-tools: unknown tool: $tool" >&2
-      exit 1
-      ;;
-  esac
-done
-
-printf '%s\n' "$BIN_DIR"

.gitea/workflows/renovate-ci.yaml

@@ -7,58 +7,33 @@ on:
       - main
   workflow_dispatch:
 
-permissions:
-  contents: read
-
 jobs:
   validate-renovate:
     runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 20
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
 
-      # renovate/k8s/cronjob.yaml is the single source of truth for the image tag,
-      # so the same version that runs in the cluster is the one validated here.
-      - name: Resolve the deployed Renovate image
-        id: image
+      - name: Validate Renovate Compose draft
         shell: bash
         run: |
           set -euo pipefail
-          image="$(sed -n 's|.*image:[[:space:]]*\(renovate/renovate:[^[:space:]]*\).*|\1|p' \
-            renovate/k8s/cronjob.yaml | head -1)"
-          if [ -z "$image" ]; then
-            echo "::error::no renovate/renovate image found in renovate/k8s/cronjob.yaml"
-            exit 1
-          fi
-          echo "using $image"
-          echo "image=$image" >> "$GITHUB_OUTPUT"
+          trap 'rm -f renovate/.env' EXIT
+          printf '%s\n' \
+            'RENOVATE_ENDPOINT=https://gitea.example/api/v1' \
+            'RENOVATE_TOKEN=test-token' \
+            'RENOVATE_REPOSITORIES=forust/homelab' \
+            > renovate/.env
+          docker compose -f renovate/renovate-compose.yaml config --quiet
 
-      - name: Validate Renovate repository config
+      - name: Validate Kubernetes manifests
         shell: bash
         run: |
           set -euo pipefail
           docker run --rm \
-            -v "$PWD/renovate:/opt/renovate:ro" \
-            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
-            "${{ steps.image.outputs.image }}" \
-            renovate-config-validator /opt/renovate/renovate.json
-
-      # The CronJob cannot read the repository, so renovate/k8s/configmap.yaml
-      # carries an inlined copy of the config. Fail if it no longer matches.
-      - name: Check the generated Renovate ConfigMap
-        shell: bash
-        run: |
-          set -euo pipefail
-          ./.gitea/workflows/sync-renovate-configmap.sh --check
-
-      - name: Validate Renovate Kubernetes manifests
-        shell: bash
-        run: |
-          set -euo pipefail
-          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh kubeconform)"
-          export PATH="$tools_dir:$PATH"
-          kubeconform \
+            -v "$PWD:/work" \
+            -w /work \
+            ghcr.io/yannh/kubeconform:latest \
             -strict \
             -ignore-missing-schemas \
             -summary \
@@ -66,11 +41,12 @@ jobs:
             renovate/k8s/configmap.yaml \
             renovate/k8s/cronjob.yaml
 
-      - name: Validate Renovate Compose file
+      - name: Validate Renovate repository config
         shell: bash
         run: |
           set -euo pipefail
-          source .gitea/workflows/compose-lint.sh
-          mapfile -t safe_flags < <(compose_safe_flags)
-          validate_compose_file renovate/renovate-compose.yaml \
-            ${safe_flags[@]+"${safe_flags[@]}"}
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            renovate/renovate:44.83.2 \
+            renovate-config-validator renovate.json

.gitea/workflows/renovate-run.yaml

@@ -1,92 +0,0 @@
-name: renovate-run
-
-on:
-  workflow_dispatch:
-    inputs:
-      repositories:
-        description: "Repositories to scan (comma-separated)"
-        required: false
-        default: "forust/homelab"
-      log_level:
-        description: "Renovate log level"
-        required: false
-        default: "info"
-        type: choice
-        options:
-          - info
-          - debug
-      dry_run:
-        description: "Plan only, do not open or update PRs"
-        required: false
-        default: false
-        type: boolean
-
-# Renovate writes through its own bot PAT, passed in as RENOVATE_TOKEN, so the
-# Actions token is only ever used to read the checkout.
-permissions:
-  contents: read
-
-concurrency:
-  group: renovate-run
-  cancel-in-progress: false
-
-jobs:
-  run-renovate:
-    runs-on: [self-hosted, linux, arch, homelab]
-    timeout-minutes: 60
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-
-      # renovate/k8s/cronjob.yaml is the single source of truth for the image tag.
-      # Reading it here means this workflow validates and runs the exact version
-      # that is deployed, instead of a copy that silently goes stale.
-      - name: Resolve the deployed Renovate image
-        id: image
-        shell: bash
-        run: |
-          set -euo pipefail
-          image="$(sed -n 's|.*image:[[:space:]]*\(renovate/renovate:[^[:space:]]*\).*|\1|p' \
-            renovate/k8s/cronjob.yaml | head -1)"
-          if [ -z "$image" ]; then
-            echo "::error::no renovate/renovate image found in renovate/k8s/cronjob.yaml"
-            exit 1
-          fi
-          echo "using $image"
-          echo "image=$image" >> "$GITHUB_OUTPUT"
-
-      - name: Validate Renovate config
-        shell: bash
-        run: |
-          set -euo pipefail
-          docker run --rm \
-            -v "$PWD/renovate/renovate.json:/opt/renovate/renovate.json:ro" \
-            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
-            "${{ steps.image.outputs.image }}" \
-            renovate-config-validator
-
-      - name: Run Renovate
-        shell: bash
-        env:
-          RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
-          RENOVATE_GITHUB_COM_TOKEN: ${{ secrets.RENOVATE_GITHUB_COM_TOKEN }}
-          RENOVATE_REPOSITORIES: ${{ inputs.repositories }}
-          RENOVATE_DRY_RUN: ${{ inputs.dry_run && 'full' || '' }}
-          LOG_LEVEL: ${{ inputs.log_level }}
-        run: |
-          set -euo pipefail
-
-          : "${RENOVATE_TOKEN:?missing RENOVATE_TOKEN secret — add a renovate-bot PAT in repo/org Actions secrets}"
-
-          docker run --rm \
-            -v "$PWD/renovate/renovate.json:/opt/renovate/renovate.json:ro" \
-            -e RENOVATE_PLATFORM=gitea \
-            -e RENOVATE_ENDPOINT=https://git.forust.xyz/api/v1 \
-            -e RENOVATE_TOKEN="$RENOVATE_TOKEN" \
-            -e RENOVATE_GITHUB_COM_TOKEN="${RENOVATE_GITHUB_COM_TOKEN:-}" \
-            -e RENOVATE_REPOSITORIES="${RENOVATE_REPOSITORIES:-forust/homelab}" \
-            -e RENOVATE_DRY_RUN="${RENOVATE_DRY_RUN:-}" \
-            -e RENOVATE_CONFIG_FILE=/opt/renovate/renovate.json \
-            -e RENOVATE_BASE_DIR=/tmp/renovate \
-            -e LOG_LEVEL="${LOG_LEVEL:-info}" \
-            "${{ steps.image.outputs.image }}"

.gitea/workflows/ssh-run.sh

@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# usage: ssh-run.sh <stage>
-# Runs one deploy-lib.sh stage on the workstation over SSH.
-set -euo pipefail
-
-: "${DEPLOY_HOST:?missing DEPLOY_HOST}"
-: "${DEPLOY_USER:?missing DEPLOY_USER}"
-: "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
-
-deploy_port="${DEPLOY_PORT:-22}"
-deploy_path="${DEPLOY_PATH:-/srv/homelab}"
-deploy_path="$(printf '%s' "$deploy_path" | tr -d '\"' | tr -d '\r' | xargs)"
-
-# The private key is written to a per-run directory that is removed on exit, so a
-# failed or cancelled job cannot leave deploy credentials in the runner's temp
-# directory. Do not use a fixed path: apply-k8s and apply-compose run in parallel.
-key_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/homelab-deploy-key.XXXXXXXX")"
-trap 'rm -rf "$key_dir"' EXIT INT TERM
-
-ssh_key="$key_dir/deploy_key"
-printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
-chmod 600 "$ssh_key"
-
-# A connection that died silently used to hang until the job timeout, and the
-# stage was never re-run: one flaky TCP session cost a whole 45-minute apply.
-# ServerAlive* bounds how long a dead peer goes unnoticed, ConnectTimeout bounds
-# setup. Only exit 255 - ssh's own transport failures - is retried. A stage that
-# fails on its own merits exits with the remote's status, so a real failure
-# still surfaces its own log instead of burning three attempts. The stages are
-# declarative applies, so re-running one that had already committed is harmless.
-ssh_opts=(
-  -i "$ssh_key" -p "$deploy_port"
-  -o BatchMode=yes -o StrictHostKeyChecking=accept-new
-  -o ConnectTimeout=15
-  -o ServerAliveInterval=15 -o ServerAliveCountMax=4
-)
-
-rc=0
-# apply-k8s and apply-compose are separate workflow jobs so the graph stays
-# intact for the verify job, but on a single node they must not run at once:
-# host docker churn on top of cluster churn is what melts the node (load 40+,
-# netbird/ssh die, helm is left pending-*). Serialize them on the workstation
-# with a shared lock; whoever arrives second waits.
-remote_cmd=(bash -se)
-case "$1" in
-  apply-k8s | apply-compose)
-    remote_cmd=(flock -w 5400 /tmp/homelab-apply.lock bash -se)
-    ;;
-esac
-for attempt in 1 2 3; do
-  if [ "$attempt" -gt 1 ]; then
-    echo ":: warning::ssh transport failed, retrying (${attempt}/3)"
-    sleep $((attempt * 5))
-  fi
-  rc=0
-  # shellcheck disable=SC2029 # remote_cmd/ssh_opts expand on the client on purpose: they select the local ssh invocation, only the heredoc runs remotely.
-  ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
-    env "REPO=$deploy_path" "APPLY_PRUNE=${APPLY_PRUNE:-false}" \
-    "DEPLOY_SHA=${DEPLOY_SHA:-}" "DEPLOY_SNAPSHOT_DIR=${DEPLOY_SNAPSHOT_DIR:-}" \
-    "STAGE=$1" "${remote_cmd[@]}" <<'EOF' || rc=$?
-source "$REPO/.gitea/workflows/deploy-lib.sh"
-run_stage "$STAGE"
-EOF
-  [ "$rc" -eq 0 ] && break
-  [ "$rc" -ne 255 ] && break
-done
-
-if [ "$rc" -ne 0 ]; then
-  echo ":: error::stage $1 failed over ssh (exit $rc)"
-fi
-exit "$rc"

.gitea/workflows/sync-renovate-configmap.sh

@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-# Regenerates renovate/k8s/configmap.yaml from renovate/renovate.json.
-#
-# renovate/renovate.json is the single source of truth: the CronJob, the Compose
-# file and the renovate-run workflow all mount that exact file. A ConfigMap cannot
-# read a file from the repository, so the same bytes are inlined here as a literal
-# block. This script keeps the copy honest:
-#
-#   .gitea/workflows/sync-renovate-configmap.sh          # rewrite in place
-#   .gitea/workflows/sync-renovate-configmap.sh --check  # fail if out of date
-#
-# renovate-ci runs the --check form on every PR and push, so a config change that
-# forgets to regenerate the ConfigMap cannot be merged.
-set -euo pipefail
-
-here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-repo="$(git -C "$here" rev-parse --show-toplevel)"
-
-src="$repo/renovate/renovate.json"
-dst="$repo/renovate/k8s/configmap.yaml"
-[ -f "$src" ] || {
-  echo "missing $src" >&2
-  exit 1
-}
-
-render() {
-  cat <<'HEADER'
-# GENERATED FILE - do not edit by hand.
-# Source: renovate/renovate.json
-# Regenerate: .gitea/workflows/sync-renovate-configmap.sh
-# Verify:     .gitea/workflows/sync-renovate-configmap.sh --check
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: renovate-config
-  namespace: renovate
-data:
-  renovate.json: |
-HEADER
-  sed 's/^/    /' "$src"
-}
-
-if [ "${1:-}" = "--check" ]; then
-  if ! diff -u "$dst" <(render) >/dev/null 2>&1; then
-    echo "ERROR: $dst is out of sync with renovate/renovate.json"
-    echo "Run: .gitea/workflows/sync-renovate-configmap.sh"
-    diff -u "$dst" <(render) || true
-    exit 1
-  fi
-  echo "renovate/k8s/configmap.yaml is in sync with renovate/renovate.json"
-  exit 0
-fi
-
-render >"$dst"
-echo "wrote $dst"

.gitea/workflows/tool-versions.env

@@ -1,33 +0,0 @@
-# Pinned versions of the CI tools installed by install-ci-tools.sh.
-# Renovate keeps these up to date (see customManagers in renovate/renovate.json).
-#
-# Every version here except NODE_VERSION matches what was already installed on
-# the runner, so pinning them changes what CI does not at all. It changes what
-# CI does when the runner is rebuilt with something else: today
-# install-ci-tools.sh finds the pinned version already on PATH and installs
-# nothing, and a runner that drifts gets the pinned one installed over it.
-#
-# The renovate image version is NOT pinned here: renovate/k8s/cronjob.yaml is the
-# single source of truth and the workflows read the tag from it, so there is
-# nothing to drift.
-ACTIONLINT_VERSION="1.7.7"
-SHELLCHECK_VERSION="0.11.0"
-KUBECONFORM_VERSION="0.8.0"
-PRETTIER_VERSION="3.8.1"
-RUFF_VERSION="0.16.8"
-YAMLLINT_VERSION="1.38.0"
-HADOLINT_VERSION="2.14.0"
-# pip-audit reads the advisory database over the network, so a floating version
-# would make the same commit report different things on different days. Pin it
-# like the rest: the advisories themselves are the moving part, not the tool.
-PIP_AUDIT_VERSION="2.10.1"
-# uv builds the throwaway venv the pytest job runs in, and unpacks the PyPI
-# wheels for ruff, yamllint and pip-audit.
-UV_VERSION="0.12.17"
-# node runs `npm ci` for the frontend tests and the npm audit, and it is the one
-# pin here that does NOT come from the runner: the runner's system node is a
-# rolling Arch package (it was node 26 with no npm at all when this was pinned),
-# and the panel image is node:22-alpine. Pinned to the image's major on purpose,
-# so the tree that gets tested is the tree that gets built. Renovate keeps this
-# in step with the Dockerfile's node: tag via the "node runtime" group.
-NODE_VERSION="22.23.3"

.github/workflows/ci.yaml

@@ -0,0 +1,370 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - "**"
+  pull_request:
+  workflow_dispatch:
+
+env:
+  REGISTRY: gcr.forust.xyz
+
+jobs:
+  lint-prettier:
+    runs-on: [self-hosted, linux, arch, homelab]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Check formatting with Prettier
+        shell: bash
+        run: |
+          mapfile -t prettier_files < <(
+            git ls-files \
+              | grep -E '\.(md|json|ya?ml|html|css)$' \
+              | grep -Ev '^(\.docs/|\.zed/|errorpages/html/|homepages/(forust_files|xdfnx_files)/)'
+          )
+
+          if [ "${#prettier_files[@]}" -eq 0 ]; then
+            echo "No Prettier-managed files found."
+            exit 0
+          fi
+
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            node:22-alpine \
+            sh -lc 'npx --yes prettier@3 --check --ignore-unknown "$@"' sh "${prettier_files[@]}"
+
+  lint-ruff:
+    runs-on: [self-hosted, linux, arch, homelab]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Lint Python with Ruff
+        shell: bash
+        run: |
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            ghcr.io/astral-sh/ruff:latest \
+            check .
+
+  lint-yaml:
+    runs-on: [self-hosted, linux, arch, homelab]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Lint YAML syntax
+        shell: bash
+        run: |
+          mapfile -t yaml_files < <(
+            git ls-files '*.yaml' '*.yml' \
+              ':!node_modules/**' \
+              ':!**/.venv/**'
+          )
+
+          if [ "${#yaml_files[@]}" -eq 0 ]; then
+            echo "No YAML files found."
+            exit 0
+          fi
+
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            cytopia/yamllint:latest \
+            -c .yamllint "${yaml_files[@]}"
+
+  lint-dockerfiles:
+    runs-on: [self-hosted, linux, arch, homelab]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Lint Dockerfiles
+        shell: bash
+        run: |
+          mapfile -t dockerfiles < <(
+            git ls-files ':(glob)**/Dockerfile' ':(glob)**/Dockerfile.*'
+          )
+
+          if [ "${#dockerfiles[@]}" -eq 0 ]; then
+            echo "No Dockerfiles found."
+            exit 0
+          fi
+
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            --entrypoint hadolint \
+            hadolint/hadolint:latest-debian \
+            -c .hadolint.yaml "${dockerfiles[@]}"
+
+  validate:
+    runs-on: [self-hosted, linux, arch, homelab]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Validate Kubernetes manifests
+        shell: bash
+        run: |
+          mapfile -t manifests < <(
+            git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
+              | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
+          )
+
+          if [ "${#manifests[@]}" -eq 0 ]; then
+            echo "No Kubernetes manifests found."
+            exit 0
+          fi
+
+          docker run --rm \
+            -v "$PWD:/work" \
+            -w /work \
+            ghcr.io/yannh/kubeconform:latest \
+            -strict \
+            -ignore-missing-schemas \
+            -summary \
+            "${manifests[@]}"
+
+  build:
+    needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
+    if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
+    runs-on: [self-hosted, linux, arch, homelab]
+    outputs:
+      services: ${{ steps.services.outputs.services }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Detect changed docker-built services
+        id: services
+        shell: bash
+        run: |
+          base="${{ github.event.before }}"
+          if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
+            base="$(git rev-list --max-parents=0 HEAD)"
+          fi
+
+          mapfile -t changed_files < <(git diff --name-only "$base" "${GITHUB_SHA}")
+
+          services=()
+
+          add_service() {
+            local name="$1"
+            local seen=0
+            for existing in "${services[@]}"; do
+              if [ "$existing" = "$name" ]; then
+                seen=1
+                break
+              fi
+            done
+            if [ "$seen" -eq 0 ]; then
+              services+=("$name")
+            fi
+          }
+
+          for file in "${changed_files[@]}"; do
+            case "$file" in
+              dtek_notif/*)
+                add_service dtek_notif
+                ;;
+              errorpages/*)
+                add_service errorpages
+                ;;
+              userbot/*)
+                add_service userbot
+                ;;
+              homepages/*)
+                add_service homepages
+                ;;
+              edu_master/phpsessid-bot/*|edu_master/webinar-checker/*|edu_master/compose.yaml)
+                add_service edu_master
+                ;;
+            esac
+          done
+
+          if [ "${#services[@]}" -eq 0 ]; then
+            echo "No docker-built services changed."
+            echo "services=" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          printf '%s\n' "${services[@]}" | tee /tmp/services.txt
+          echo "services=$(paste -sd, /tmp/services.txt)" >> "$GITHUB_OUTPUT"
+
+      - name: Log in to registry
+        if: steps.services.outputs.services != ''
+        shell: bash
+        run: |
+          echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login "${REGISTRY}" \
+            -u "${{ secrets.REGISTRY_USERNAME }}" \
+            --password-stdin
+
+      - name: Build and push changed images
+        if: steps.services.outputs.services != ''
+        shell: bash
+        run: |
+          IFS=, read -r -a services <<< "${{ steps.services.outputs.services }}"
+
+          for service in "${services[@]}"; do
+            case "$service" in
+              dtek_notif)
+                image="${REGISTRY}/forust/dtek-notif"
+                tags=("latest")
+                case "${GITHUB_REF_NAME}" in
+                  main)
+                    tags+=("main" "prod")
+                    ;;
+                  dev)
+                    tags+=("dev")
+                    ;;
+                esac
+                build_args=()
+                for tag in "${tags[@]}"; do
+                  build_args+=(-t "${image}:${tag}")
+                done
+                docker build "${build_args[@]}" dtek_notif
+                for tag in "${tags[@]}"; do
+                  docker push "${image}:${tag}"
+                done
+                ;;
+              errorpages)
+                image="${REGISTRY}/forust/error-pages"
+                tags=("latest")
+                case "${GITHUB_REF_NAME}" in
+                  main)
+                    tags+=("main" "prod")
+                    ;;
+                  dev)
+                    tags+=("dev")
+                    ;;
+                esac
+                build_args=()
+                for tag in "${tags[@]}"; do
+                  build_args+=(-t "${image}:${tag}")
+                done
+                docker build "${build_args[@]}" errorpages
+                for tag in "${tags[@]}"; do
+                  docker push "${image}:${tag}"
+                done
+                ;;
+              userbot)
+                tags=("latest")
+                case "${GITHUB_REF_NAME}" in
+                  main)
+                    tags+=("main" "prod")
+                    ;;
+                  dev)
+                    tags+=("dev")
+                    ;;
+                esac
+                for target in runtime panel; do
+                  case "$target" in
+                    runtime)
+                      context="userbot"
+                      image="${REGISTRY}/forust/userbot"
+                      ;;
+                    panel)
+                      context="userbot/panel"
+                      image="${REGISTRY}/forust/userbot-panel"
+                      ;;
+                  esac
+                  build_args=()
+                  for tag in "${tags[@]}"; do
+                    build_args+=(-t "${image}:${tag}")
+                  done
+                  docker build "${build_args[@]}" "$context"
+                  for tag in "${tags[@]}"; do
+                    docker push "${image}:${tag}"
+                  done
+                done
+                ;;
+              homepages)
+                for service in forust xdfnx; do
+                  case "$service" in
+                    forust)
+                      image="${REGISTRY}/forust/forust-homepage"
+                      ;;
+                    xdfnx)
+                      image="${REGISTRY}/forust/xdfnx-homepage"
+                      ;;
+                  esac
+                  tags=("latest")
+                  case "${GITHUB_REF_NAME}" in
+                    main)
+                      tags+=("main" "prod")
+                      ;;
+                    dev)
+                      tags+=("dev")
+                      ;;
+                  esac
+                  build_args=()
+                  for tag in "${tags[@]}"; do
+                    build_args+=(-t "${image}:${tag}")
+                  done
+                  docker build "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
+                  for tag in "${tags[@]}"; do
+                    docker push "${image}:${tag}"
+                  done
+                done
+                ;;
+              edu_master)
+                for service in session-keeper webinar-checker; do
+                  case "$service" in
+                    session-keeper)
+                      context="edu_master/phpsessid-bot"
+                      image="${REGISTRY}/forust/session-keeper"
+                      ;;
+                    webinar-checker)
+                      context="edu_master/webinar-checker"
+                      image="${REGISTRY}/forust/webinar-checker"
+                      ;;
+                  esac
+                  tags=("latest")
+                  case "${GITHUB_REF_NAME}" in
+                    main)
+                      tags+=("main" "prod")
+                      ;;
+                    dev)
+                      tags+=("dev")
+                      ;;
+                  esac
+                  build_args=()
+                  for tag in "${tags[@]}"; do
+                    build_args+=(-t "${image}:${tag}")
+                  done
+                  docker build "${build_args[@]}" "$context"
+                  for tag in "${tags[@]}"; do
+                    docker push "${image}:${tag}"
+                  done
+                done
+                ;;
+            esac
+          done
+
+  deploy-userbot-panel:
+    needs: build
+    if: github.ref_name == 'main' && contains(needs.build.outputs.services, 'userbot')
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Apply and roll out userbot panel
+        shell: bash
+        run: |
+          kubectl apply -f userbot/k8s/base/panel.yaml
+          kubectl get secret userbot-common-secrets -n default -o json \
+            | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
+            | kubectl apply -f -
+          # Keep legacy deployments (forust/anna) in sync with manifests; they have no replicas field, so apply leaves scaling to the user manager only.
+          kubectl apply -f userbot/k8s/base/userbots.yaml
+          kubectl rollout restart deployment/userbot-panel -n userbot
+          kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s

.github/workflows/deploy.yaml

@@ -0,0 +1,161 @@
+name: deploy
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: deploy-main
+  cancel-in-progress: false
+
+jobs:
+  redeploy:
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    steps:
+      - name: Redeploy workstation
+        shell: bash
+        env:
+          DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
+          DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
+          DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
+          DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
+          DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+          # Set APPLY_PRUNE=true to enable kubectl apply --prune. Requires every
+          # manifest to carry label app.kubernetes.io/managed-by=homelab-deploy,
+          # otherwise previously applied resources get deleted on the next run.
+          APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
+        run: |
+          set -euo pipefail
+
+          : "${DEPLOY_HOST:?missing DEPLOY_HOST}"
+          : "${DEPLOY_USER:?missing DEPLOY_USER}"
+          : "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
+
+          deploy_port="${DEPLOY_PORT:-22}"
+          deploy_path="${DEPLOY_PATH:-/srv/homelab}"
+
+          ssh_key="$RUNNER_TEMP/deploy_key"
+          mkdir -p "$RUNNER_TEMP"
+          printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
+          chmod 600 "$ssh_key"
+
+          ssh_opts=(
+            -i "$ssh_key"
+            -p "$deploy_port"
+            -o BatchMode=yes
+            -o StrictHostKeyChecking=accept-new
+          )
+
+          ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
+            "DEPLOY_PATH=$(printf '%q' \"$deploy_path\") APPLY_PRUNE=$(printf '%q' \"${APPLY_PRUNE:-false}\") bash -se" <<'EOF'
+          set -euo pipefail
+
+          repo="${DEPLOY_PATH:-/srv/homelab}"
+
+          if [ ! -d "$repo/.git" ]; then
+            echo "Repository not found at $repo"
+            exit 1
+          fi
+
+          git -C "$repo" fetch origin main
+          git -C "$repo" reset --hard origin/main
+
+          # Runtime selection: a service is k8s-managed when $SERVICE/k8s/active
+          # exists. Otherwise it is compose-managed, and only k8s/routing/*
+          # manifests (external Services / EndpointSlices / ServersTransport /
+          # Ingresses that route to docker backends) are applied.
+          #   migrate:   touch SERVICE/k8s/active   (+ move routing files up)
+          #   rollback:  rm SERVICE/k8s/active
+          collect_k8s() {
+            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
+              ! -path '*/routing/*' ! -path '*/overlays/*' \
+              ! -name 'kustomization.y*ml' ! -name '*.example.y*ml' \
+              ! -name '*values.y*ml' ! -name 'patch-*.y*ml' \
+              | sort
+          }
+
+          collect_k8s_inactive() {
+            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
+              \( -name 'namespace.y*ml' -o -path '*/routing/*' \) \
+              ! -path '*/overlays/*' ! -name '*.example.y*ml' \
+              | sort
+          }
+
+          mapfile -t compose_stacks < <(
+            find "$repo" -type f \( -name 'compose.yaml' -o -name 'compose.yml' \) | sort
+          )
+
+          mapfile -t k8s_manifests < <(
+            for kd in $(find "$repo" -type d -name k8s ! -path '*/.git/*' | sort); do
+              if [ -f "$kd/active" ]; then
+                collect_k8s "$kd"
+              else
+                collect_k8s_inactive "$kd"
+              fi
+            done
+          )
+
+          echo "== Validate compose stacks =="
+          for cf in "${compose_stacks[@]}"; do
+            dir=$(dirname "$cf")
+            if [ -f "$dir/k8s/active" ]; then
+              echo "  skip (k8s-managed): $dir"
+              continue
+            fi
+            echo "  config: $cf"
+            docker compose -f "$cf" config --quiet
+          done
+
+          echo "== Validate k8s manifests (kubectl dry-run) =="
+          for m in "${k8s_manifests[@]}"; do
+            echo "  apply --dry-run=client $m"
+            kubectl apply --dry-run=client -f "$m" >/dev/null
+          done
+
+          echo "== Applying Kubernetes manifests =="
+          ns_files=()
+          other_files=()
+          for m in "${k8s_manifests[@]}"; do
+            case "$m" in
+              */namespace.y?ml) ns_files+=("$m") ;;
+              *) other_files+=("$m") ;;
+            esac
+          done
+
+          prune_opts=()
+          if [ "${APPLY_PRUNE:-false}" = "true" ]; then
+            prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
+          fi
+
+          if [ "${#ns_files[@]}" -gt 0 ]; then
+            echo "  namespaces first: ${ns_files[*]}"
+            kubectl apply -f "${ns_files[@]}"
+          fi
+          if [ -f "$repo/prometheus-stack/k8s/active" ]; then
+            echo "== Upgrading kube-prometheus-stack =="
+            helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
+              --namespace prometheus \
+              --version 86.2.3 \
+              --values "$repo/prometheus-stack/k8s/grafana-values.yaml" \
+              --wait
+          fi
+
+          if [ "${#other_files[@]}" -gt 0 ]; then
+            echo "  resources: ${other_files[*]}"
+            kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
+          fi
+
+          echo "== Redeploying docker compose stacks =="
+          for cf in "${compose_stacks[@]}"; do
+            dir=$(dirname "$cf")
+            if [ -f "$dir/k8s/active" ]; then
+              echo "  skip (k8s-managed): $dir"
+              continue
+            fi
+            echo "  compose: $dir"
+            if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
+              docker compose -f "$cf" build
+              docker compose -f "$cf" push
+            fi
+            docker compose -f "$cf" up -d --pull always --remove-orphans
+          done
+          EOF

.gitignore

@@ -21,9 +21,6 @@ checkmk/checkmk/*
 downtify/Downtify_downloads
 headscale/config/*
 headscale/data/*
-# NetBird local hostnames and generated secrets
-netbird/.env
-netbird/secrets/
 searxng/core-config/*
 
 # Steaming services files
@@ -96,8 +93,6 @@ replacements.txt
 # Temp files
 edu_master/temp/
 temp/*
-# Local-only tooling scratch space (pinned CI tools, verification scripts)
-tmp/
 
 # Environment
 .env
@@ -109,10 +104,6 @@ tmp/
 # kubernetes
 */k8s/*secret*
 !*/k8s/*secret*.example
-**/k8s/*secret*
-!**/k8s/*secret*.example
-# Local-only tweaks, not for upstream
-prometheus-stack/k8s/grafana-values.yaml
 traefik/k8s/local-tls.yaml
 converters/k8s/config.yaml
 convertx/k8s/config.yaml

adguardhome/k8s/adguard.yaml

@@ -58,45 +58,12 @@ spec:
   selector:
     matchLabels:
       app: adguard
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
         app: adguard
-      annotations:
-        reloader.stakater.com/auto: "true"
     spec:
       containers:
-        - name: netbird
-          image: netbirdio/netbird:0.80.0
-          envFrom:
-            - configMapRef:
-                name: adguard-config
-          env:
-            - name: NB_SETUP_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: adguard-netbird-secrets
-                  key: NB_SETUP_KEY
-          securityContext:
-            capabilities:
-              add:
-                - NET_ADMIN
-                - SYS_ADMIN
-                - SYS_RESOURCE
-          resources:
-            requests:
-              memory: "64Mi"
-              cpu: "50m"
-            limits:
-              memory: "256Mi"
-              cpu: "200m"
-          volumeMounts:
-            - name: netbird-state
-              mountPath: /var/lib/netbird
-            - name: dev-tun
-              mountPath: /dev/net/tun
         - name: adguard
           image: adguard/adguardhome:v0.107.79
           resources:
@@ -104,7 +71,7 @@ spec:
               memory: "1.5Gi"
               cpu: "300m"
             requests:
-              memory: "512Mi"
+              memory: "500Mi"
               cpu: "50m"
           ports:
             - containerPort: 3000
@@ -113,13 +80,6 @@ spec:
               name: dns
             - containerPort: 853
               name: dot
-          readinessProbe:
-            tcpSocket:
-              port: dns
-            initialDelaySeconds: 5
-            periodSeconds: 5
-            successThreshold: 1
-            failureThreshold: 3
           volumeMounts:
             - name: adguard-data
               mountPath: /opt/adguardhome/work
@@ -131,12 +91,6 @@ spec:
               mountPath: /certs
               readOnly: true
       volumes:
-        - name: netbird-state
-          emptyDir: {}
-        - name: dev-tun
-          hostPath:
-            path: /dev/net/tun
-            type: CharDevice
         - name: adguard-data
           persistentVolumeClaim:
             claimName: adguard-pvc

adguardhome/k8s/certificate.yaml

@@ -1,12 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: adguard-certs
-  namespace: adguard
-spec:
-  secretName: adguard-certs
-  dnsNames:
-    - dns.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer

adguardhome/k8s/config.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: adguard-config
-  namespace: adguard
-data:
-  NB_MANAGEMENT_URL: "https://nb.forust.xyz"
-  NB_HOSTNAME: "adguard"
-  NB_LOG_LEVEL: "info"
-  NB_DISABLE_DNS: "true"

adguardhome/k8s/ingress.yaml

@@ -7,18 +7,21 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`dns.forust.xyz`)
+    - match: Host(`adguard.forust.xyz`) || Host(`dns.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: adguard-service
           port: 3000
-    - match: (Host(`dns.forust.xyz`)) && PathPrefix(`/dns-query`)
+    - match: (Host(`adguard.forust.xyz`) || Host(`dns.forust.xyz`)) && PathPrefix(`/dns-query`)
       kind: Rule
       services:
         - name: adguard-service
           port: 3000
   tls:
-    secretName: adguard-certs
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -39,5 +42,3 @@ spec:
       services:
         - name: adguard-service
           port: 3000
-  tls:
-    secretName: internal-wildcard-tls

adguardhome/k8s/internal-certificate.yaml

@@ -1,15 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: adguard
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

adguardhome/k8s/netbird-secrets.yaml.example

@@ -1,8 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: adguard-netbird-secrets
-  namespace: adguard
-type: Opaque
-stringData:
-  NB_SETUP_KEY: "REPLACE_ME"

authentik/k8s/authentik.yaml

@@ -34,8 +34,6 @@ spec:
   selector:
     matchLabels:
       app: authentik-server
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -54,8 +52,8 @@ spec:
             - containerPort: 9000
           resources:
             requests:
-              memory: "768Mi"
-              cpu: "100m"
+              memory: "700Mi"
+              cpu: "300m"
             limits:
               memory: "1.5Gi"
               cpu: "1000m"
@@ -70,8 +68,6 @@ spec:
   selector:
     matchLabels:
       app: authentik-worker
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -90,8 +86,8 @@ spec:
                 name: authentik-secrets
           resources:
             requests:
-              memory: "320Mi"
-              cpu: "100m"
+              memory: "512Mi"
+              cpu: "300m"
             limits:
-              memory: "768Mi"
+              memory: "1Gi"
               cpu: "700m"

authentik/k8s/certificates.yaml

@@ -1,28 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: authentik-prod-tls
-  namespace: authentik
-spec:
-  secretName: authentik-prod-tls
-  dnsNames:
-    - auth.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: authentik
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

authentik/k8s/ingress.yaml

@@ -9,11 +9,14 @@ spec:
   routes:
     - match: Host(`auth.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: authentik-server-service
           port: 9000
   tls:
-    secretName: authentik-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -29,5 +32,3 @@ spec:
       services:
         - name: authentik-server-service
           port: 9000
-  tls:
-    secretName: internal-wildcard-tls

cert-manager/k8s/cert-manager-values.yaml

@@ -1,9 +0,0 @@
-crds:
-  enabled: true
-prometheus:
-  servicemonitor:
-    enabled: true
-    interval: 60s
-    scrapeTimeout: 30s
-    labels:
-      release: prometheus-stack

cert-manager/k8s/clusterissuer.yaml

@@ -1,29 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-staging
-spec:
-  acme:
-    email: bobrovod@national.shitposting.agency
-    server: https://acme-staging-v02.api.letsencrypt.org/directory
-    privateKeySecretRef:
-      name: letsencrypt-staging-account-key
-    solvers:
-      - http01:
-          ingress:
-            class: traefik
----
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-prod
-spec:
-  acme:
-    email: bobrovod@national.shitposting.agency
-    server: https://acme-v02.api.letsencrypt.org/directory
-    privateKeySecretRef:
-      name: letsencrypt-prod-account-key
-    solvers:
-      - http01:
-          ingress:
-            class: traefik

cert-manager/k8s/homelab-internal-root-ca.crt

@@ -1,30 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFFjCCAv6gAwIBAgIUetKpTfEDOn2985FFMu6G26itT+wwDQYJKoZIhvcNAQEN
-BQAwIzEhMB8GA1UEAxMYaG9tZWxhYiBpbnRlcm5hbCByb290IENBMB4XDTI2MDky
-MzEyNDA0N1oXDTM2MDkyMDEyNDA0N1owIzEhMB8GA1UEAxMYaG9tZWxhYiBpbnRl
-cm5hbCByb290IENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvmNP
-ZCOoD8NtNuYJKVXBlTPjX7D7sJCSK5neH7ZbYV5+lmUlEErY8Mik7j37V5k5NfpF
-Ig85pOjP7RckTPz5V6ek3yaN40s4AL053sN5ZPauDVYjalaEHTgj5sEMqlLACQWI
-yZmJOZspZykae8dIpQnqCoFpRT4FurJ78v4a0ylnFVLMQn/lyCHedwTjkEdtYWYr
-ccJy8vQwqkzs/rWvEH1lDqZhennLOrmcCjfonG7D/pruMn4z+6E28p4+ejkRrI6x
-luak3KnpT1XMeHtgU21hiRGaMDBchHMFgAhnY1qosymKenXvfTZItwgjZbwa1hJI
-GAiDm+jQDKMjzRZ3rH6Xfc0auUcykNz73PpNu1NGm78nndXwCXcXn1LFKNQJ+r1U
-sJiyAmUZmXVn4aM4OMf2F38k7wTYIKg7nRGaUkNeKDlNkjA4HvgWw+jwO1KmdHQ/
-mOem1rosDWHRK01wg+Gga9mQCnhNhxglg3t/UeSic6uOaRsvaz4qkzHq8MbCujVz
-DpKQjqdikYOAXZOs4KlBLWrS7NaK4NzfSD02pBUErh54ruJfY/bWz9KyXzBD/lQZ
-VUTKyvUVB0bkVHEdf1jJmX3H4IZRQSF5JPqOBotW6bJI5fEGNBvj9Zxy4nm2WWGz
-yyP3uWsQz8U/Wdx9nXZLHInTZBsvgLYtKUAWA30CAwEAAaNCMEAwDgYDVR0PAQH/
-BAQDAgKkMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFEkKm2rxPaK6+O9WD80z
-BLC6F9QsMA0GCSqGSIb3DQEBDQUAA4ICAQAnFyHz97Umf5VIu+dKTJid7C73VugJ
-TIar/xJBs/4CxP+znBxhJjXygRoyIfzoVGWcB2ZSL//vL78Qlts79K/Imc9a4RFF
-wMvCxsRXAEQ4TpeWi3ophPNcs4rhsP+gQKQFtnyKP9519bqpfxp0bTqwOV2o18fn
-za7rlQViiEnNV58j7CVoM9+mJvVVfBEX1Km+GyJL9GadzbIQ7FxClVJZefCbft93
-zHVk9gDOw8ys1XGSR2OUCyCLinXO6mqS16CmBb2MAKXq/YyH7E0N8iotAPGtfA8V
-M/0ddy947rY0xCrtECfWwvGQpJS7NRv/Z9b2jCfXrI5LXmL2nfQRg0y9GE4Vjwr+
-WxtGU5jOeFt0jQ+xRzcgG0Op+qK3x55l5LSo2hOcOVYbiHxcHEJFgwNi1ADeBFwb
-q/HdysfURSOghqjIpMMAUabBp+DBUg2EUF7pIaUqbdqExFYcr9EYisEMiNsmKmN+
-8ZbcOeerFKDQj+t/R0bFXa7UBn2UWsjI8zlR74aa2kLDXwtyz/XlO/FlYm66eBFo
-2/eYUSeU+S4ej+wUAs/dvjF7f190/DUQGuwOTlLTahqWDztmhCk7qzbECu56CwKT
-E5Ect2P72UleYwdblkVOVd352AmiwEzdOaziIRrPh8uenEknH6JBYPO3mjk7cCg+
-GPGcNIBctjdXhg==
------END CERTIFICATE-----

cert-manager/k8s/internal-ca.yaml

@@ -1,33 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: selfsigned
-spec:
-  selfSigned: {}
----
-# Homelab internal root CA (10y). Install the .crt on clients (see below).
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-ca-root
-  namespace: cert-manager
-spec:
-  isCA: true
-  commonName: homelab internal root CA
-  duration: 87600h
-  renewBefore: 7200h
-  secretName: internal-ca-root
-  privateKey:
-    algorithm: RSA
-    size: 4096
-  issuerRef:
-    name: selfsigned
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: internal-ca
-spec:
-  ca:
-    secretName: internal-ca-root

cert-manager/k8s/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: cert-manager

cfddns/k8s/deployment.yaml

@@ -9,8 +9,6 @@ spec:
   selector:
     matchLabels:
       app: cfddns
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -24,10 +22,10 @@ spec:
           imagePullPolicy: Always
           resources:
             requests:
-              memory: "32Mi"
+              memory: "20Mi"
               cpu: "30m"
             limits:
-              memory: "128Mi"
+              memory: "64Mi"
               cpu: "50m"
           envFrom:
             - secretRef:

checkmk/k8s/certificates.yaml

@@ -1,28 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: checkmk-prod-tls
-  namespace: checkmk
-spec:
-  secretName: checkmk-prod-tls
-  dnsNames:
-    - cmk.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: checkmk
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

checkmk/k8s/checkmk.yaml

@@ -24,8 +24,6 @@ spec:
   selector:
     matchLabels:
       app: checkmk
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:

checkmk/k8s/ingress.yaml

@@ -9,11 +9,14 @@ spec:
   routes:
     - match: Host(`cmk.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: checkmk-service
           port: 5000
   tls:
-    secretName: checkmk-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRouteTCP
@@ -45,5 +48,3 @@ spec:
       services:
         - name: checkmk-service
           port: 5000
-  tls:
-    secretName: internal-wildcard-tls

cloudflared/k8s/.gitignore

@@ -1 +0,0 @@
-secret.yaml

cloudflared/k8s/deployment.yaml

@@ -1,39 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: cloudflared
-  labels:
-    app: cloudflared
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: cloudflared
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        app: cloudflared
-    spec:
-      containers:
-        - name: cloudflared
-          image: cloudflare/cloudflared:2026.9.3
-          imagePullPolicy: IfNotPresent
-          args:
-            - tunnel
-            - --no-autoupdate
-            - run
-          env:
-            - name: TUNNEL_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: cloudflared-secrets
-                  key: TUNNEL_TOKEN
-          resources:
-            requests:
-              memory: "128Mi"
-              cpu: "30m"
-            limits:
-              memory: "256Mi"
-              cpu: "200m"

cloudflared/k8s/secret.yaml.example

@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: cloudflared-secrets
-type: Opaque
-stringData:
-  TUNNEL_TOKEN: your_tunnel_token_here

converters/compose.yaml

@@ -1,7 +1,7 @@
 services:
   convertx:
     container_name: convertx
-    image: ghcr.io/c4illin/convertx:v0.19.0
+    image: ghcr.io/c4illin/convertx:v0.18.0
     restart: unless-stopped
     ports:
       - "9992:3000"
@@ -54,7 +54,7 @@ services:
       - "traefik.http.routers.bentopdf.tls.certresolver=letsencrypt"
       - "traefik.http.routers.bentopdf.tls=true"
       # Local router
-      - "traefik.http.routers.bentopdf-local.rule=Host(`pdf.workstation.internal`)"
+      - "traefik.http.routers.bentopdf-local.rule=Host(`pdf.wokstation.internal`)"
       - "traefik.http.routers.bentopdf-local.entrypoints=websecure"
       - "traefik.http.routers.bentopdf-local.tls=true"
       # Dev router

converters/k8s/bentopdf.yaml

@@ -31,13 +31,12 @@ spec:
           name: bentopdf
           ports:
             - containerPort: 8080
-          # p95 4M, max 11M over 7 days. Was 50Mi/700Mi.
           resources:
             requests:
-              memory: "32Mi"
+              memory: "50Mi"
               cpu: "50m"
               ephemeral-storage: "100Mi"
             limits:
-              memory: "128Mi"
+              memory: "700Mi"
               cpu: "700m"
               ephemeral-storage: "5Gi"

converters/k8s/certificates.yaml

@@ -1,42 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: convertx-prod-tls
-  namespace: converters
-spec:
-  secretName: convertx-prod-tls
-  dnsNames:
-    - forust.xyz
-    - www.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: bentopdf-prod-tls
-  namespace: converters
-spec:
-  secretName: bentopdf-prod-tls
-  dnsNames:
-    - pdf.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: converters
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

converters/k8s/convertx.yaml

@@ -20,15 +20,13 @@ spec:
   selector:
     matchLabels:
       app: convertx
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
         app: convertx
     spec:
       containers:
-        - image: ghcr.io/c4illin/convertx:v0.19.0
+        - image: ghcr.io/c4illin/convertx:v0.18.0
           name: convertx
           envFrom:
             - configMapRef:
@@ -40,14 +38,13 @@ spec:
           volumeMounts:
             - mountPath: /data
               name: data
-          # p95 85M, max 136M over 7 days, spikes while converting. Was 250Mi/1.5Gi.
           resources:
             requests:
-              memory: "128Mi"
+              memory: "250Mi"
               cpu: "100m"
             limits:
               cpu: "1500m"
-              memory: "512Mi"
+              memory: "1.5Gi"
       volumes:
         - name: data
           persistentVolumeClaim:

converters/k8s/ingress.yaml

@@ -14,7 +14,7 @@ spec:
         - name: convertx-service
           port: 3000
   tls:
-    secretName: convertx-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -31,9 +31,6 @@ spec:
       services:
         - name: convertx-service
           port: 3000
-  tls:
-    secretName: internal-wildcard-tls
-
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -50,7 +47,7 @@ spec:
         - name: bentopdf-service
           port: 8080
   tls:
-    secretName: bentopdf-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -66,5 +63,3 @@ spec:
       services:
         - name: bentopdf-service
           port: 8080
-  tls:
-    secretName: internal-wildcard-tls

crowdsec/k8s/crowdsec-middleware.yaml

@@ -0,0 +1,14 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: crowdsec-bouncer
+  namespace: crowdsec
+spec:
+  plugin:
+    crowdsec-bouncer:
+      enabled: true
+      LogLevel: INFO
+      CrowdsecMode: live
+      CrowdsecLapiScheme: http
+      CrowdsecLapiHost: crowdsec-service.crowdsec.svc.cluster.local:8080
+      CrowdsecLapiKeyFile: "/etc/traefik/secrets/traefik-api-key"

crowdsec/k8s/crowdsec-values.yaml

@@ -1,203 +1,57 @@
 container_runtime: containerd
-
 agent:
-  acquisition: []
-  additionalAcquisition:
-    - labels:
-        type: traefik
-      limit: 1000
-      query: |
-        {namespace="traefik"}
-      source: loki
-      url: http://loki.prometheus.svc.cluster.local:3100/
-      wait_for_ready: 30s
   env:
     - name: COLLECTIONS
-      value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
+      value: "crowdsecurity/traefik crowdsecurity/base-http-scenarios"
     - name: DISABLE_COLLECTIONS
-      value: crowdsecurity/sshd
-    # Bans on 401/403 bursts hurt more than they protect: with L3 enforcement
-    # a false positive cuts the IP off everything (SSH included), and past
-    # incidents show legit automation (deploy runner, mesh peers, registry
-    # pulls) tripping this probe. Probing/XSS/SQLi/CVE scenarios stay.
-    - name: DISABLE_SCENARIOS
-      value: crowdsecurity/http-generic-bf
-  metrics:
-    enabled: true
-    serviceMonitor:
-      additionalLabels:
-        release: prometheus-stack
-      enabled: true
-  # Static machine identity: agent pods mount pre-created LAPI credentials
-  # (Secret crowdsec-agent-credentials, key local_api_credentials.yaml)
-  # at the exact path the agent entrypoint expects. Together with the
-  # patched register-init (enforced by janitor-cronjob.yaml) the agent
-  # never calls `cscli lapi register` in steady state, so pod names,
-  # restarts and reboots can no longer break it.
-  extraVolumes:
-    - name: static-creds
-      secret:
-        secretName: crowdsec-agent-credentials
-        items:
-          - key: local_api_credentials.yaml
-            path: local_api_credentials.yaml
-  extraVolumeMounts:
-    - name: static-creds
-      mountPath: /tmp_config/local_api_credentials.yaml
-      subPath: local_api_credentials.yaml
-      readOnly: true
+      value: "crowdsecurity/linux crowdsecurity/sshd"
+
+  acquisition:
+    - namespace: traefik
+      podName: "*traefik*"
+      program: traefik
+      poll_without_inotify: true
+
   resources:
-    limits:
-      cpu: 200m
-      memory: 500Mi
     requests:
       cpu: 50m
       memory: 100Mi
-
-config:
-  parsers:
-    s02-enrich:
-      mobile-whitelist.yaml: |
-        name: forust/mobile-whitelist
-        description: "Whitelist SWAN/4ka mobile network"
-        whitelist:
-          reason: "Mobile IP whitelist"
-          cidr:
-            - "84.245.64.0/18"
-      # CrowdSec's own guidance: CIDR allowlisting belongs at the parser stage.
-      # A parser whitelist discards the event before it reaches a bucket, so
-      # these addresses never produce an overflow and never become a decision.
-      # A postoverflow whitelist is checked only *after* the ban exists, and
-      # the bouncer answers 403 for as long as it does - which is a window we
-      # do not want the deploy sitting in.
-      local-network.yaml: |
-        name: forust/local-network
-        description: "Whitelist loopback, private and VPN networks"
-        whitelist:
-          reason: "Local network"
-          cidr:
-            - "127.0.0.0/8"
-            - "10.0.0.0/8"
-            - "172.16.0.0/12"
-            - "192.168.0.0/16"
-            # CGNAT range (RFC 6598). The workstation and the k0s node live
-            # here on WireGuard, and 100.64.0.0/10 is not covered by the
-            # RFC 1918 blocks above.
-            - "100.64.0.0/10"
-            - "169.254.0.0/16"
-            - "fc00::/7"
-            - "fe80::/10"
-      vps-whitelist.yaml: |
-        name: forust/vps-whitelist
-        description: "Whitelist static VPS"
-        whitelist:
-          reason: "VPS"
-          ip:
-            - "193.181.211.79"
-
-  postoverflows:
-    s01-whitelist:
-      # The one whitelist that has to stay here: resolving a hostname is a
-      # network call, and the docs put expensive lookups in postoverflows on
-      # purpose - it runs only when a bucket actually overflows.
-      # ddns.forust.xyz is the public home address, not a private one, so
-      # forust/local-network does not cover it.
-      home-dynamic-ip.yaml: |
-        name: forust/home-dynamic-ip
-        description: "Whitelist home dynamic IP"
-        whitelist:
-          reason: "Home dynamic IP"
-          expression:
-            - evt.Overflow.Alert.Source.IP in LookupHost("ddns.forust.xyz")
-
-  # LAPI-only main config override, merged over config.yaml. NOTE: the
-  # chart's own default for this key is REPLACED, not merged, so its
-  # auto_registration block is repeated verbatim below - drop it and the
-  # agent can no longer register itself.
-  config.yaml.local: |
-    api:
-      server:
-        auto_registration: # Activate if not using TLS for authentication
-          enabled: true
-          token: "${REGISTRATION_TOKEN}" # /!\ Do not modify this variable (auto-generated and handled by the chart)
-          allowed_ranges: # /!\ Make sure to adapt to the pod IP ranges used by your cluster
-            - "127.0.0.1/32"
-            - "192.168.0.0/16"
-            - "10.0.0.0/8"
-            - "172.16.0.0/12"
-        # This homelab has no egress to console.crowdsec.cloud: DNS does
-        # not resolve. The LAPI kept trying anyway ("Signal push: N
-        # signals to push", "capi metrics: sending" every 10s) and each
-        # attempt sat on a resolver timeout WHILE HOLDING A WRITE
-        # TRANSACTION, which is what kept stalling per-request decision
-        # lookups even with WAL enabled. Nothing to share and nothing to
-        # pull - turn the Central API off instead of letting it block the
-        # only database writer we have.
-        online_client:
-          sharing: false
-          pull:
-            community: false
-            blocklists: false
-        disable_usage_metrics_export: true
-    db_config:
-      # SQLite without WAL serialises every reader behind the writer's
-      # rollback journal, and the LAPI writes constantly: the agent pushes
-      # Traefik alerts read from Loki, the metrics collector counts
-      # decisions, the bouncer touches "last pull" on every request.
-      # Symptom: decision lookups taking 10-30s (and a second connection
-      # that could not even open the database) while the LAPI sat at 28m
-      # CPU - the process was blocked in fsync, not computing. Every
-      # bouncer-protected request then blew through the plugin timeout and
-      # fail-closed with 403, on every site at once.
-      # The PVC is local-path-retain (hostPath), not a network share, so
-      # WAL is safe here; the crowdsec docs recommend it for exactly this
-      # ("allowing more concurrency in SQLite that will improve
-      # performances in most scenarios").
-      use_wal: true
-      # Keeps the alert table bounded. At the 5000/7d default the file
-      # reached 54MB in 15 days off the Traefik access log alone, and the
-      # metrics collector counts decisions on a timer; a smaller working
-      # set means fewer full scans. Crowdsec only prunes - SQLite never
-      # shrinks the file, so the size stays until a manual VACUUM.
-      flush:
-        max_items: 1000
-        max_age: 24h
+    limits:
+      cpu: 200m
+      memory: 500Mi
 
 lapi:
   env:
     - name: COLLECTIONS
-      value: crowdsecurity/traefik crowdsecurity/base-http-scenarios
+      value: "crowdsecurity/traefik crowdsecurity/base-http-scenarios"
     - name: DISABLE_COLLECTIONS
-      value: crowdsecurity/linux crowdsecurity/sshd
-  metrics:
+      value: "crowdsecurity/linux crowdsecurity/sshd"
+  service:
+    type: ClusterIP
+  persistentVolume:
+    data:
+      enabled: true
+      storageClassName: local-path-retain
+      size: 1Gi
+    config:
+      enabled: true
+      storageClassName: local-path-retain
+      size: 100Mi
+  storeLAPICscliCredentialsInSecret: true
+  resources:
+    requests:
+      cpu: 50m
+      memory: 150Mi
+    limits:
+      cpu: 400m
+      memory: 500Mi
+
+metrics:
   enabled: true
   serviceMonitor:
     additionalLabels:
       release: prometheus-stack
     enabled: true
-  persistentVolume:
-    config:
-      enabled: true
-      size: 100Mi
-      storageClassName: local-path-retain
-    data:
-      enabled: true
-      size: 1Gi
-      storageClassName: local-path-retain
-  # LAPI answers a blocking /v1/decisions lookup for EVERY bouncer-protected
-  # request (whole Traefik front door), so it is the hot path of the proxy.
-  # At 400m/500Mi it went CPU-throttled and idle lookups measured 1.3-7.4s,
-  # which pushed requests into the bouncer's fail-closed 403.
-  # Single replica on purpose: LAPI is stateful (BoltDB on the `data` PVC,
-  # credentials on the `config` PVC) - two replicas sharing those RWO
-  # volumes would corrupt the decision store. Scale up CPU, not replicas.
-  resources:
-    limits:
-      cpu: 1500m
-      memory: 1Gi
-    requests:
-      cpu: 250m
-      memory: 500Mi
-  service:
-    type: ClusterIP
-  storeLAPICscliCredentialsInSecret: true
+    interval: 30s
+    scrapeTimeout: 10s
+    namespace: prometheus

crowdsec/k8s/grafana-dashboards.yaml

@@ -1,32 +0,0 @@
-apiVersion: v1
-data:
-  crowdsec-overview.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph (old)\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"stat\",\n      \"name\": \"Stat\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"timeseries\",\n      \"name\": \"Time series\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 24,\n      \"panels\": [],\n      \"title\": \"Summary\",\n      \"type\": \"row\"\n    },\n    {\n      \"cacheTimeout\": null,\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [\n            {\n              \"options\": {\n                \"match\": \"null\",\n                \"result\": {\n                  \"text\": \"N/A\"\n                }\n              },\n              \"type\": \"special\"\n            }\n          ],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"#E02F44\",\n                \"value\": null\n              },\n              {\n                \"color\": \"#E02F44\",\n                \"value\": 10\n              },\n              {\n                \"color\": \"#299c46\",\n                \"value\": 10\n              }\n            ]\n          },\n          \"unit\": \"none\"\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 6,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2,\n      \"interval\": null,\n      \"links\": [],\n      \"maxDataPoints\": 100,\n      \"options\": {\n        \"colorMode\": \"background\",\n        \"graphMode\": \"none\",\n        \"justifyMode\": \"auto\",\n        \"orientation\": \"horizontal\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"text\": {},\n        \"textMode\": \"auto\"\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"targets\": [\n        {\n          \"exemplar\": true,\n          \"expr\": \"count(cs_info)\",\n          \"interval\": \"\",\n          \"legendFormat\": \"\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"timeFrom\": null,\n      \"timeShift\": null,\n      \"title\": \"Running Crowdsec\",\n      \"transparent\": true,\n      \"type\": \"stat\"\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"decimals\": 1,\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"links\": []\n        },\n        \"overrides\": []\n      },\n      \"fill\": 1,\n      \"fillGradient\": 0,\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 18,\n        \"x\": 6,\n        \"y\": 1\n      },\n      \"hiddenSeries\": false,\n      \"id\": 8,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": false,\n        \"max\": false,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sort\": \"total\",\n        \"sortDesc\": true,\n        \"total\": true,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"nullPointMode\": \"null\",\n      \"options\": {\n        \"alertThreshold\": true\n      },\n      \"percentage\": false,\n      \"pluginVersion\": \"8.1.2\",\n      \"pointradius\": 2,\n      \"points\": false,\n      \"renLine truncated
-kind: ConfigMap
-metadata:
-  labels:
-    app.kubernetes.io/managed-by: manual
-    grafana_dashboard: "1"
-  name: crowdsec-crowdsec-overview
-  namespace: prometheus
----
-apiVersion: v1
-data:
-  crowdsec-lapi-metrics.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"bargauge\",\n      \"name\": \"Bar gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1655915193937,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 10,\n      \"panels\": [],\n      \"title\": \"Agents\",\n      \"type\": \"row\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"green\",\n                \"value\": null\n              },\n              {\n                \"color\": \"red\",\n                \"value\": 80\n              }\n            ]\n          }\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2,\n      \"options\": {\n        \"displayMode\": \"gradient\",\n        \"orientation\": \"vertical\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"showUnfilled\": false,\n        \"text\": {}\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"repeat\": \"query0\",\n      \"repeatDirection\": \"h\",\n      \"targets\": [\n        {\n          \"exemplar\": false,\n          \"expr\": \"sum(rate(cs_lapi_request_duration_seconds_bucket{endpoint=\\\"/v1/watchers/login\\\", instance=\\\"$lapi\\\"}[$__rate_interval])) by (le)\",\n          \"format\": \"heatmap\",\n          \"interval\": \"\",\n          \"legendFormat\": \"{{le}}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"title\": \"Agents Login\",\n      \"type\": \"heatmap\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n      \"fieldConfig\": {\n        \"defaults\": {\n          \"color\": {\n            \"mode\": \"thresholds\"\n          },\n          \"mappings\": [],\n          \"thresholds\": {\n            \"mode\": \"absolute\",\n            \"steps\": [\n              {\n                \"color\": \"green\",\n                \"value\": null\n              }\n            ]\n          },\n          \"unit\": \"none\"\n        },\n        \"overrides\": []\n      },\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 1\n      },\n      \"id\": 6,\n      \"options\": {\n        \"displayMode\": \"gradient\",\n        \"orientation\": \"auto\",\n        \"reduceOptions\": {\n          \"calcs\": [\n            \"lastNotNull\"\n          ],\n          \"fields\": \"\",\n          \"values\": false\n        },\n        \"showUnfilled\": false,\n        \"text\": {}\n      },\n      \"pluginVersion\": \"8.1.2\",\n      \"targets\": [\n        {\n          \"exemplar\": true,\n          \"expr\": \"sum(rate(cs_lapi_request_duration_seconds_bucket{endpoint=\\\"/v1/watchers/login\\\"}[$__rate_interval])) by (le)\",\n          \"format\": \"heatmap\",\n          \"interval\": \"\",\n          \"legendFormat\": \"{{le}}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"title\": \"Heartbeat\",\n      \"type\": \"heatmap\"\n    },\n    {\n      \"collapsed\": false,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 12,\n      \"panels\": [],\n      \"title\": \"Decisions\",\n      \"type\": \"row\"\n    },\n    {\n      \"datasource\": \"${DS_PROMETHEUS}\",\n  Line truncated
-kind: ConfigMap
-metadata:
-  labels:
-    app.kubernetes.io/managed-by: manual
-    grafana_dashboard: "1"
-  name: crowdsec-crowdsec-lapi-metrics
-  namespace: prometheus
----
-apiVersion: v1
-data:
-  crowdsec-insight.json: "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_PROMETHEUS\",\n      \"label\": \"Prometheus\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"bargauge\",\n      \"name\": \"Bar gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"gauge\",\n      \"name\": \"Gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"8.1.2\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"stat\",\n      \"name\": \"Stat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"target\": {\n          \"limit\": 100,\n          \"matchAny\": false,\n          \"tags\": [],\n          \"type\": \"dashboard\"\n        },\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1655915159751,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"datasource\": null,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 22,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"datasource\": \"${DS_PROMETHEUS}\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"color\": {\n                \"mode\": \"thresholds\"\n              },\n              \"mappings\": [\n                {\n                  \"options\": {\n                    \"match\": \"null\",\n                    \"result\": {\n                      \"text\": \"N/A\"\n                    }\n                  },\n                  \"type\": \"special\"\n                }\n              ],\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  },\n                  {\n                    \"color\": \"red\",\n                    \"value\": 80\n                  }\n                ]\n              },\n              \"unit\": \"dateTimeAsIso\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 5,\n            \"x\": 2,\n            \"y\": 1\n          },\n          \"id\": 2,\n          \"interval\": null,\n          \"links\": [],\n          \"maxDataPoints\": 100,\n          \"options\": {\n            \"colorMode\": \"none\",\n            \"graphMode\": \"none\",\n            \"justifyMode\": \"auto\",\n            \"orientation\": \"horizontal\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"lastNotNull\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\n            \"text\": {},\n            \"textMode\": \"auto\"\n          },\n          \"pluginVersion\": \"8.1.2\",\n          \"targets\": [\n            {\n              \"exemplar\": true,\n              \"expr\": \"(process_start_time_seconds{instance=\\\"$instance\\\"})*1000\",\n              \"interval\": \"\",\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Up since\",\n          \"type\": \"stat\"\n        },\n        {\n          \"datasource\": \"${DS_PROMETHEUS}\",\n          \"fieldConfig\": {\n            \"defaults\": {\n              \"displayName\": \"\",\n              \"mappings\": [],\n              \"thresholds\": {\n                \"mode\": \"absolute\",\n                \"steps\": [\n                  {\n                    \"color\": \"green\",\n                    \"value\": null\n                  }\n                ]\n              },\n              \"unit\": \"decbytes\"\n            },\n            \"overrides\": []\n          },\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 5,\n            \"x\": 7,\n            \"y\": 1\n          },\n          \"id\": 4,\n          \"options\": {\n            \"orientation\": \"auto\",\n            \"reduceOptions\": {\n              \"calcs\": [\n                \"mean\"\n              ],\n              \"fields\": \"\",\n              \"values\": false\n            },\Line truncated
-kind: ConfigMap
-metadata:
-  labels:
-    app.kubernetes.io/managed-by: manual
-    grafana_dashboard: "1"
-  name: crowdsec-crowdsec-insight
-  namespace: prometheus

crowdsec/k8s/janitor-cronjob.yaml

@@ -1,202 +0,0 @@
-# CrowdSec self-healing: static machine identity + enforcement loops.
-#
-# Problem it fixes: the chart's agent init container runs
-#   `cscli lapi register --machine "$POD_NAME" ...`
-# unconditionally. Credentials live in an emptyDir, the machine row lives
-# in LAPI's persistent DB. Any init re-run for an already-known pod name
-# (kubelet restart, node reboot) dies with
-#   403 Forbidden: user '<pod>' already exist
-# and the DaemonSet pod sticks in Init forever. Every DS restart also
-# leaves an orphan machine row that is never cleaned.
-#
-# Design (name-independent):
-#   * Agent identity is a STATIC machine `crowdsec-agent-workstation`
-#     whose password lives in Secret `crowdsec-agent-credentials`
-#     (created once, manually - like all other secrets in this repo).
-#     The secret is mounted into agent pods at
-#     /tmp_config/local_api_credentials.yaml (see extraVolumeMounts in
-#     crowdsec-values.yaml), which is exactly the path the agent's main
-#     container copies into place at startup.
-#   * The DS init command is patched (strategic merge, by container name)
-#     to SKIP registration when that file exists, keeping the legacy
-#     register path only as fallback. Detection marker in the patched
-#     command: `[ -s /tmp_config`.
-#   * This CronJob enforces the desired state hourly, so recovery is
-#     automatic even after `helm upgrade` reverts the DS patch or the
-#     LAPI database is wiped:
-#       1. patch DS init if it still has the unconditional register
-#          (no-op otherwise - no restart churn);
-#       2. prune machines with no heartbeat for 2h (orphan hygiene);
-#       3. ensure the static machine exists, recreating it with the
-#          Secret password if missing (agent retry loops reconnect
-#          on their own - same name + same password);
-#       4. prune bouncer entries idle for 30d.
-#
-# It used to also delete LePresidente/http-generic-403-bf decisions hourly.
-# That was a workaround for the bouncer failing closed on a slow LAPI and
-# 403-ing the deploy runner into a 4h ban. The bouncer now polls decisions
-# into a cache and never blocks on an unreachable LAPI, so it cannot
-# manufacture those 403s any more, and the scenario only fires against real
-# scanners - deleting their decisions hourly was undoing a working ban.
-#
-# Manual apply (crowdsec/k8s is NOT managed by deploy.yaml):
-#   kubectl apply -f crowdsec/k8s/janitor-cronjob.yaml
-# Force a run:
-#   kubectl create job -n crowdsec --from=cronjob/crowdsec-janitor janitor-now
-#
-# Helm upgrades: the janitor's strategic patch puts the DS field under
-# the `kubectl-patch` field manager, so a plain `helm upgrade` FAILS
-# with an SSA conflict on initContainers[].command. Procedure:
-#   1. revert init to chart state (kills the conflict):
-#        helm template crowdsec crowdsec/crowdsec --version <ver> \
-#          -n crowdsec -f crowdsec/k8s/crowdsec-values.yaml > /tmp/r.yaml
-#        python3 -c "import yaml,json; ..." # build revert patch from
-#        the rendered DaemonSet init command, then
-#        kubectl patch ds crowdsec-agent -n crowdsec \
-#          --type strategic -p "\$(cat /tmp/revert_patch.json)"
-#   2. helm upgrade --install crowdsec ... (no --force needed)
-#   3. janitor-now right away (upgrade reverts init; new pods would
-#      sit in Init until the next hourly run otherwise).
-#
-# One-time bootstrap (order matters):
-#   1. Create Secret + static machine (see commands in chat).
-#   2. Apply this file, trigger janitor-now, wait for agent 1/1.
-#   3. One-time orphan cleanup:
-#        kubectl exec -n crowdsec deploy/crowdsec-lapi -- \
-#          cscli machines prune --duration 1h --force
-#   4. Only then `helm upgrade` crowdsec with the extraVolumes values.
-#      Upgrade reverts the DS patch; trigger janitor-now right after it
-#      (otherwise new pods sit in Init until the next hourly run, then
-#      self-heal anyway).
-#
-# Password rotation: update the Secret, delete the machine
-# (`cscli machines delete crowdsec-agent-workstation`), trigger
-# janitor-now (recreates it), then `kubectl rollout restart
-# ds/crowdsec-agent -n crowdsec` (agent reads the file at startup only).
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: crowdsec-janitor
-  namespace: crowdsec
-  labels:
-    app.kubernetes.io/part-of: crowdsec
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: crowdsec-janitor
-  namespace: crowdsec
-  labels:
-    app.kubernetes.io/part-of: crowdsec
-rules:
-  - apiGroups: [""]
-    resources: ["pods"]
-    verbs: ["get", "list"]
-  - apiGroups: [""]
-    resources: ["pods/exec"]
-    verbs: ["create"]
-  - apiGroups: ["apps"]
-    resources: ["daemonsets"]
-    verbs: ["get", "patch"]
-  # `kubectl exec deploy/<name>` resolves deploy -> replicaset -> pod,
-  # which needs read access to these (exec itself is pods/exec above).
-  - apiGroups: ["apps"]
-    resources: ["deployments", "replicasets"]
-    verbs: ["get", "list"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: crowdsec-janitor
-  namespace: crowdsec
-  labels:
-    app.kubernetes.io/part-of: crowdsec
-subjects:
-  - kind: ServiceAccount
-    name: crowdsec-janitor
-    namespace: crowdsec
-roleRef:
-  kind: Role
-  name: crowdsec-janitor
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: crowdsec-janitor
-  namespace: crowdsec
-  labels:
-    app.kubernetes.io/part-of: crowdsec
-spec:
-  schedule: "17 * * * *"
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 3
-  failedJobsHistoryLimit: 3
-  jobTemplate:
-    spec:
-      activeDeadlineSeconds: 300
-      template:
-        metadata:
-          labels:
-            app.kubernetes.io/part-of: crowdsec
-        spec:
-          serviceAccountName: crowdsec-janitor
-          restartPolicy: OnFailure
-          containers:
-            - name: janitor
-              # Same image the chart itself uses for registration jobs;
-              # IfNotPresent so it works while the node is offline
-              # (layer cached from the chart install).
-              image: alpine/kubectl:latest
-              imagePullPolicy: IfNotPresent
-              env:
-                - name: AGENT_PASSWORD
-                  valueFrom:
-                    secretKeyRef:
-                      name: crowdsec-agent-credentials
-                      key: password
-              command:
-                - /bin/sh
-                - -c
-                - |
-                  set -eu
-                  LAPI_EXEC="kubectl exec -n crowdsec deploy/crowdsec-lapi --"
-                  echo "== 1. enforce patched agent init =="
-                  CUR=$(kubectl get ds crowdsec-agent -n crowdsec \
-                    -o jsonpath='{.spec.template.spec.initContainers[0].command[2]}')
-                  case "$CUR" in
-                    *'-s /tmp_config'*)
-                      echo "init already patched"
-                      ;;
-                    *)
-                      echo "patching init"
-                      WAIT='until nc "$LAPI_HOST" "$LAPI_PORT" -z'
-                      WAIT="$WAIT; do echo waiting for lapi to start; sleep 5; done"
-                      LINK='ln -s /staging/etc/crowdsec /etc/crowdsec'
-                      REG='cscli lapi register --machine "$USERNAME"'
-                      REG="$REG -u \"\$LAPI_URL\" --token \"\$REGISTRATION_TOKEN\""
-                      CREDS=/tmp_config/local_api_credentials.yaml
-                      CMD="$WAIT; $LINK; [ -s $CREDS ] || {"
-                      CMD="$CMD $REG && cp"
-                      CMD="$CMD /etc/crowdsec/local_api_credentials.yaml $CREDS; }"
-                      ESC=$(printf '%s' "$CMD" | sed 's/"/\\"/g')
-                      PATCH='{"spec":{"template":{"spec":{"initContainers":'
-                      PATCH=$PATCH'[{"name":"wait-for-lapi-and-register",'
-                      PATCH=$PATCH'"command":["sh","-c","'$ESC'"]}]}}}}'
-                      kubectl patch ds crowdsec-agent -n crowdsec \
-                        --type strategic -p "$PATCH"
-                      ;;
-                  esac
-                  echo "== 2. prune orphan machines (no heartbeat for 2h) =="
-                  $LAPI_EXEC cscli machines prune --duration 2h --force
-                  echo "== 3. ensure static machine exists =="
-                  if $LAPI_EXEC cscli machines inspect \
-                    crowdsec-agent-workstation >/dev/null 2>&1; then
-                    echo "static machine present"
-                  else
-                    echo "recreating static machine"
-                    $LAPI_EXEC cscli machines add crowdsec-agent-workstation \
-                      --password "$AGENT_PASSWORD" --force
-                  fi
-                  echo "== 4. prune stale bouncers (no pull for 30d) =="
-                  $LAPI_EXEC cscli bouncers prune -d 720h --force

crowdsec/k8s/network-policy.yaml

@@ -25,10 +25,3 @@ spec:
       ports:
         - protocol: TCP
           port: 8080
-    - from:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: prometheus
-      ports:
-        - protocol: TCP
-          port: 6060

dockmon/compose.yaml

@@ -1,6 +1,6 @@
 services:
   dockmon:
-    image: darthnorse/dockmon:2.5.0
+    image: darthnorse/dockmon:2.4.5
     container_name: dockmon
     restart: unless-stopped
     # ports:

dockmon/k8s/certificates.yaml

@@ -1,28 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: dockmon-prod-tls
-  namespace: dockmon
-spec:
-  secretName: dockmon-prod-tls
-  dnsNames:
-    - dockmon.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: dockmon
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

dockmon/k8s/dockmon.yaml

@@ -29,7 +29,7 @@ spec:
     spec:
       containers:
         - name: dockmon
-          image: darthnorse/dockmon:2.5.0
+          image: darthnorse/dockmon:2.4.5
           ports:
             - containerPort: 443
           volumeMounts:

dockmon/k8s/ingress.yaml

@@ -18,13 +18,15 @@ spec:
     - match: Host(`dockmon.forust.xyz`)
       kind: Rule
       middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
         - name: security-headers@file
       services:
         - name: dockmon-service
           port: 443
           serversTransport: dockmon-transport
   tls:
-    secretName: dockmon-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -41,5 +43,3 @@ spec:
         - name: dockmon-service
           port: 443
           serversTransport: dockmon-transport
-  tls:
-    secretName: internal-wildcard-tls

downtify/compose.yaml

@@ -1,7 +1,7 @@
 services:
   downtify:
     container_name: downtify
-    image: ghcr.io/henriquesebastiao/downtify:3.4.0
+    image: ghcr.io/henriquesebastiao/downtify:2.13.0
     restart: unless-stopped
     # ports:
     #   - '7077:8000'

downtify/k8s/downtify.yaml

@@ -20,8 +20,6 @@ spec:
   selector:
     matchLabels:
       app: downtify
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -29,7 +27,7 @@ spec:
     spec:
       containers:
         - name: downtify
-          image: ghcr.io/henriquesebastiao/downtify:3.4.0
+          image: ghcr.io/henriquesebastiao/downtify:2.13.0
           ports:
             - containerPort: 8000
           volumeMounts:

downtify/k8s/ingress.yaml

@@ -10,12 +10,14 @@ spec:
     - match: Host(`downtify.forust.xyz`)
       kind: Rule
       middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
         - name: security-chain@file
       services:
         - name: downtify-service
           port: 8000
   tls:
-    secretName: downtify-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -31,5 +33,3 @@ spec:
       services:
         - name: downtify-service
           port: 8000
-  tls:
-    secretName: internal-wildcard-tls

dtek_notif/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.9-alpine
+
+WORKDIR /app
+
+# Установка зависимостей
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Копирование кода
+COPY main.py .
+COPY .env .
+# Запуск бота
+CMD ["python", "-u", "main.py"]

dtek_notif/LICENSE

@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in 
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

dtek_notif/docker-compose.yaml

@@ -0,0 +1,15 @@
+services:
+  dtek_notif:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: gcr.forust.xyz/forust/dtek-notif:latest
+    pull_policy: build
+    restart: unless-stopped
+    environment:
+      - TZ=Europe/Kyiv
+    dns:
+      - 1.1.1.1
+      - 8.8.8.8
+    networks:
+      - default

dtek_notif/main.py

@@ -0,0 +1,748 @@
+import asyncio
+import contextlib
+import logging
+import os
+from datetime import datetime, timedelta
+
+import requests
+from aiogram import Bot, Dispatcher
+from aiogram.filters import Command
+from aiogram.types import KeyboardButton, Message
+from aiogram.utils.keyboard import ReplyKeyboardBuilder
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+
+# Загрузка переменных окружения
+load_dotenv()
+
+# Настройки
+TELEGRAM_TOKEN = os.getenv('TELEGRAM_TOKEN', 'YOUR_TOKEN_HERE')
+ALLOWED_CHAT_IDS = list(map(int, os.getenv('ALLOWED_CHAT_IDS', '').split(','))) if os.getenv('ALLOWED_CHAT_IDS') else []
+CHECK_INTERVAL = int(os.getenv('CHECK_INTERVAL', '120'))
+
+# Параметры для запроса
+VOE_CITY_ID = int(os.getenv('VOE_CITY_ID', 'VOE_CITY_ID'))
+VOE_STREET_ID = int(os.getenv('VOE_STREET_ID', 'VOE_STREET_ID'))
+VOE_HOUSE_ID = int(os.getenv('VOE_HOUSE_ID', 'VOE_HOUSE_ID'))
+
+# Настройка логирования
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Глобальные переменные
+bot = Bot(token=TELEGRAM_TOKEN)
+dp = Dispatcher()
+last_schedule: list[dict] | None = None
+last_notification_time: dict[str, datetime] = {}
+
+
+# ============================================================================
+# УТИЛИТЫ
+# ============================================================================
+
+
+def format_time_duration(minutes: int) -> str:
+    """Форматирует время из минут в часы и минуты"""
+    hours = minutes // 60
+    mins = minutes % 60
+
+    if hours == 0:
+        return f'{mins}м'
+    elif mins == 0:
+        return f'{hours}ч'
+    return f'{hours}ч {mins}м'
+
+
+def get_day_statistics(day_blocks: list[dict]) -> dict[str, int]:
+    """Получает статистику по дню"""
+    total_minutes = 0
+    confirmed_minutes = 0
+    possible_minutes = 0
+
+    for block in day_blocks:
+        for half in [block['first_half'], block['second_half']]:
+            if half['status'] == 'off':
+                total_minutes += 30
+                if half['confirmed']:
+                    confirmed_minutes += 30
+                else:
+                    possible_minutes += 30
+
+    return {'total': total_minutes, 'confirmed': confirmed_minutes, 'possible': possible_minutes}
+
+
+# ============================================================================
+# ПАРСИНГ ДАННЫХ
+# ============================================================================
+
+
+def parse_html(html: str) -> list[dict]:
+    """Парсит HTML с графиком отключений (логика от 15.11.2024)"""
+    soup = BeautifulSoup(html, 'html.parser')
+    cells = soup.select('.disconnection-detailed-table-cell.cell')
+
+    schedule = []
+    current_hour = 0
+    current_day = 0
+
+    for cell in cells:
+        if 'legend' in cell.get('class', []) or 'head' in cell.get('class', []):
+            continue
+
+        cell_classes = cell.get('class', [])
+
+        # ПРоверка статуса отключения на весь час
+        full_hour_off = 'has_disconnection' in cell_classes and 'full_hour' in cell_classes
+
+        hour_block = cell.select_one('.hour_block')
+        if not hour_block:
+            continue
+
+        # Проверка подтверждённости отключения для всего часа
+        cell_confirmed = None
+        if 'confirm_1' in cell_classes:
+            cell_confirmed = True
+        elif 'confirm_0' in cell_classes:
+            cell_confirmed = False
+
+        # Проверка половин часа
+        left = hour_block.select_one('.half.left')
+        right = hour_block.select_one('.half.right')
+
+        def parse_half(half, is_full_hour_off: bool, cell_confirmed: bool | None = None) -> dict:
+            """Парсит половину часа"""
+            if not half:
+                return {'status': 'on', 'queue': None, 'confirmed': None}
+
+            half_classes = half.get('class', [])
+
+            # Если вся ячейка full_hour - используем статус ячейки
+            if is_full_hour_off:
+                return {'status': 'off', 'queue': None, 'confirmed': cell_confirmed}
+
+            # Определяем статус половины
+            if 'has_disconnection' in half_classes:
+                status = 'off'
+            elif 'no_disconnection' in half_classes:
+                status = 'on'
+            else:
+                status = 'on'  # По умолчанию считаем включенным
+
+            # Если выключено - ищем подробности
+            queue = None
+            confirmed = None
+
+            if status == 'off':
+                disconnection_div = half.select_one('.disconnection')
+                if disconnection_div:
+                    # Ищем номер черги в title
+                    if disconnection_div.has_attr('title'):
+                        title = disconnection_div['title']
+                        if 'Номер черги' in title or 'Номер черги:' in title:
+                            with contextlib.suppress(BaseException):
+                                queue = title.split(':')[-1].strip()
+
+                    # Определяем подтверждение
+                    disc_classes = disconnection_div.get('class', [])
+                    if 'disconnection_confirm_1' in disc_classes:
+                        confirmed = True
+                    elif 'disconnection_confirm_0' in disc_classes:
+                        confirmed = False
+
+            return {'status': status, 'queue': queue, 'confirmed': confirmed}
+
+        first_half_data = parse_half(left, full_hour_off, cell_confirmed)
+        second_half_data = parse_half(right, full_hour_off, cell_confirmed)
+
+        schedule.append(
+            {
+                'hour': current_hour,
+                'day': current_day,
+                'first_half': first_half_data,
+                'second_half': second_half_data,
+            }
+        )
+
+        current_hour += 1
+        if current_hour >= 24:
+            current_hour = 0
+            current_day += 1
+
+    return schedule
+
+
+def get_voe_html(city_id: int, street_id: int, house_id: int) -> str:
+    """Получает HTML с сайта VOE"""
+    url = 'https://www.voe.com.ua/disconnection/detailed?ajax_form=1&_wrapper_format=drupal_ajax'
+    headers = {
+        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+        'X-Requested-With': 'XMLHttpRequest',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+    }
+    data = {
+        'search_type': 0,
+        'city_id': city_id,
+        'street_id': street_id,
+        'house_id': house_id,
+        'form_build_id': 'form-Irv5aHw1R2FT_Ik2apyHOZ47hTH5xPNH_LQnBrmpSTc',
+        'form_id': 'disconnection_detailed_search_form',
+        '_triggering_element_name': 'search',
+        '_triggering_element_value': 'Показати',
+        '_drupal_ajax': 1,
+    }
+
+    try:
+        response = requests.post(url, headers=headers, data=data, timeout=10)
+        response.raise_for_status()
+        resp_json = response.json()
+
+        insert_html = next((item['data'] for item in resp_json if item.get('command') == 'insert'), None)
+
+        if not insert_html:
+            raise ValueError('HTML не найден в ответе')
+
+        return insert_html
+    except requests.exceptions.RequestException as e:
+        logger.error(f'Ошибка запроса VOE: {e}')
+        raise
+
+
+# ============================================================================
+# ФОРМАТИРОВАНИЕ СООБЩЕНИЙ
+# ============================================================================
+
+
+def get_main_keyboard():
+    """Создает главную клавиатуру"""
+    builder = ReplyKeyboardBuilder()
+    builder.row(KeyboardButton(text='📊 Графік'), KeyboardButton(text='🔄 Оновити'))
+    builder.row(KeyboardButton(text='📅 Сьогодні'), KeyboardButton(text='📅 Завтра'))
+    builder.row(KeyboardButton(text='ℹ️ Про бота'))
+    return builder.as_markup(resize_keyboard=True)
+
+
+def format_schedule_message(schedule: list[dict], days_to_show: int = 2) -> str:
+    """Форматирует полный график на несколько дней"""
+    lines = [
+        '⚡️ <b>Графік відключень світла</b>',
+        f'🕐 Оновлено: {datetime.now().strftime("%d.%m.%Y %H:%M:%S")}',
+        '─' * 30,
+        '',
+    ]
+
+    start_date = datetime.now()
+
+    for day in range(min(days_to_show, 2)):
+        day_blocks = [b for b in schedule if b['day'] == day]
+        if not day_blocks:
+            continue
+
+        date_str = (start_date + timedelta(days=day)).strftime('%d.%m.%Y')
+        day_name = '🌅 <b>Сьогодні</b>' if day == 0 else '🌄 <b>Завтра</b>'
+
+        lines.append(f'{day_name} ({date_str})')
+
+        # Статистика
+        stats = get_day_statistics(day_blocks)
+        if stats['total'] > 0:
+            lines.append(f'⏱ Всього: <code>{format_time_duration(stats["total"])}</code>')
+            if stats['confirmed'] > 0:
+                lines.append(f'🔴 Підтверджено: <code>{format_time_duration(stats["confirmed"])}</code>')
+            if stats['possible'] > 0:
+                lines.append(f'🟠 Можливо: <code>{format_time_duration(stats["possible"])}</code>')
+        else:
+            lines.append('🟢 <b>Відключень немає!</b>')
+
+        lines.append('')
+
+        # Детальный список отключений
+        disconnections = []
+        current_status = None
+        start_time = None
+        current_confirmed = None
+        current_queue = None
+
+        for block in day_blocks:
+            hour = block['hour']
+
+            for half_idx, half in enumerate([block['first_half'], block['second_half']]):
+                time_str = f'{hour:02d}:00' if half_idx == 0 else f'{hour:02d}:30'
+
+                if half['status'] == 'off':
+                    if current_status != 'off':
+                        start_time = time_str
+                        current_confirmed = half['confirmed']
+                        current_queue = half['queue']
+                    current_status = 'off'
+                else:
+                    if current_status == 'off':
+                        icon = '🔴' if current_confirmed else '🟠'
+                        queue_text = f' (Ч{current_queue})' if current_queue else ''
+                        disconnections.append(f'{icon} <code>{start_time} - {time_str}</code>{queue_text}')
+                    current_status = half['status']
+
+        # Если день закончился на отключении
+        if current_status == 'off':
+            icon = '🔴' if current_confirmed else '🟠'
+            queue_text = f' (Ч{current_queue})' if current_queue else ''
+            next_hour = (day_blocks[-1]['hour'] + 1) % 24
+            end_time = f'{next_hour:02d}:00'
+            disconnections.append(f'{icon} <code>{start_time} - {end_time}</code>{queue_text}')
+
+        if disconnections:
+            for idx, disc in enumerate(disconnections, 1):
+                lines.append(f'{idx}. {disc}')
+
+        lines.append('')
+
+    lines.append('<i>🔴 = підтверджено • 🟠 = можливо • 🟢 = світло</i>')
+
+    return '\n'.join(lines)
+
+
+def format_single_day_schedule(schedule: list[dict], day: int) -> str:
+    """Форматирует график на один день"""
+    day_blocks = [b for b in schedule if b['day'] == day]
+    if not day_blocks:
+        return '❌ Немає даних для цього дня'
+
+    start_date = datetime.now()
+    date_str = (start_date + timedelta(days=day)).strftime('%d.%m.%Y')
+    day_name = '🟠 <b>Сьогодні</b>' if day == 0 else '🔶 <b>Завтра</b>'
+
+    lines = [f'{day_name}  •  {date_str}', '']
+
+    # Статистика
+    lines.append('<b>📊 Статистика</b>')
+    stats = get_day_statistics(day_blocks)
+
+    if stats['total'] == 0:
+        lines.append('└ 🟢 <b>Відключень немає!</b>')
+    else:
+        total_time = format_time_duration(stats['total'])
+        lines.append(f'├ ⏱ Всього: <code>{total_time}</code>')
+
+        if stats['confirmed'] > 0:
+            confirmed_time = format_time_duration(stats['confirmed'])
+            lines.append(f'├ 🔴 Підтверджено: <code>{confirmed_time}</code>')
+
+        if stats['possible'] > 0:
+            possible_time = format_time_duration(stats['possible'])
+            lines.append(f'└ 🟠 Можливо: <code>{possible_time}</code>')
+        else:
+            lines.append('└ 🟢 Решта часу світло')
+
+    lines.append('')
+
+    # Детальный список отключений
+    disconnections = []
+    current_status = None
+    start_time = None
+    current_confirmed = None
+    current_queue = None
+
+    for block in day_blocks:
+        hour = block['hour']
+
+        for half_idx, half in enumerate([block['first_half'], block['second_half']]):
+            time_str = f'{hour:02d}:00' if half_idx == 0 else f'{hour:02d}:30'
+
+            if half['status'] == 'off':
+                if current_status != 'off':
+                    start_time = time_str
+                    current_confirmed = half['confirmed']
+                    current_queue = half['queue']
+                current_status = 'off'
+            else:
+                if current_status == 'off':
+                    icon = '🔴' if current_confirmed else '🟠'
+                    queue_text = f' (Ч.{current_queue})' if current_queue else ''
+                    disconnections.append(f'{icon} <code>{start_time} - {time_str}</code>{queue_text}')
+                current_status = half['status']
+
+    # Если день закончился на отключении
+    if current_status == 'off':
+        icon = '🔴' if current_confirmed else '🟠'
+        queue_text = f' (Ч.{current_queue})' if current_queue else ''
+        next_hour = (day_blocks[-1]['hour'] + 1) % 24
+        end_time = f'{next_hour:02d}:00'
+        disconnections.append(f'{icon} <code>{start_time} - {end_time}</code>{queue_text}')
+
+    if disconnections:
+        lines.append('<b>⚡️ Розклад відключень</b>')
+        for idx, disc in enumerate(disconnections, 1):
+            lines.append(f'{idx}. {disc}')
+
+    lines.append('')
+    lines.append('<i>🔴 підтверджено  •  🟠 можливо  •  🟢 світло</i>')
+
+    return '\n'.join(lines)
+
+
+def schedules_differ(old_schedule: list[dict] | None, new_schedule: list[dict] | None) -> bool:
+    """Проверяет отличия между графиками"""
+    if old_schedule is None or new_schedule is None:
+        return True
+
+    if len(old_schedule) != len(new_schedule):
+        return True
+
+    for old, new in zip(old_schedule, new_schedule, strict=False):
+        if old['day'] >= 2:
+            break
+
+        if old['first_half'] != new['first_half'] or old['second_half'] != new['second_half']:
+            return True
+
+    return False
+
+
+# ============================================================================
+# УВЕДОМЛЕНИЯ
+# ============================================================================
+
+
+async def send_to_all_users(message_text: str, parse_mode: str = 'HTML'):
+    """Отправляет сообщение всем пользователям"""
+    if not ALLOWED_CHAT_IDS:
+        logger.warning('Нет допущенных ID чатов для отправки уведомлений')
+        return
+
+    for chat_id in ALLOWED_CHAT_IDS:
+        try:
+            await bot.send_message(chat_id, message_text, parse_mode=parse_mode)
+            logger.info(f'✅ Сообщение отправлено пользователю {chat_id}')
+        except Exception as e:
+            logger.error(f'❌ Ошибка отправки пользователю {chat_id}: {e}')
+        await asyncio.sleep(0.5)
+
+
+async def check_schedule():
+    """Проверяет график и отправляет уведомления"""
+    global last_schedule
+
+    try:
+        logger.info('🔍 Проверка графика...')
+        html = get_voe_html(VOE_CITY_ID, VOE_STREET_ID, VOE_HOUSE_ID)
+        new_schedule = parse_html(html)
+
+        if schedules_differ(last_schedule, new_schedule):
+            logger.info('✨ Обнаружены изменения!')
+            message = format_schedule_message(new_schedule, days_to_show=2)
+
+            if last_schedule is not None:
+                await send_to_all_users(f'🔄 <b>Графік оновлено!</b>\n\n{message}')
+
+            last_schedule = new_schedule
+        else:
+            logger.info('✓ Графік без змін')
+
+    except Exception as e:
+        logger.error(f'❌ Ошибка при проверке графика: {e}')
+
+
+async def check_upcoming_disconnections():
+    """Проверяет предстоящие события и отправляет предупреждения за 5 минут"""
+    global last_notification_time
+
+    if last_schedule is None:
+        return
+
+    now = datetime.now()
+    today_blocks = [b for b in last_schedule if b['day'] == 0]
+
+    # Создаем список всех переходов (off -> on или on -> off)
+    transitions = []
+    prev_status = None
+
+    for block in today_blocks:
+        hour = block['hour']
+
+        for half_idx, half in enumerate([block['first_half'], block['second_half']]):
+            minute = 0 if half_idx == 0 else 30
+            time_str = f'{hour:02d}:{minute:02d}'
+
+            current_status = half['status']
+
+            # Если статус изменился - это переход
+            if prev_status is not None and prev_status != current_status:
+                transitions.append(
+                    {
+                        'hour': hour,
+                        'minute': minute,
+                        'time_str': time_str,
+                        'from_status': prev_status,
+                        'to_status': current_status,
+                        'confirmed': half.get('confirmed'),
+                        'queue': half.get('queue'),
+                    }
+                )
+
+            prev_status = current_status
+
+    # Проверяем переходы
+    for transition in transitions:
+        event_time = now.replace(hour=transition['hour'], minute=transition['minute'], second=0, microsecond=0)
+
+        time_until = (event_time - now).total_seconds() / 60
+        notification_key = f'{transition["hour"]}:{transition["minute"]}_{transition["to_status"]}'
+
+        # Если за 5 минут до события (±1 минута) и еще не отправляли
+        if 4 <= time_until <= 6:
+            # Проверяем, не отправляли ли уже уведомление сегодня
+            if notification_key in last_notification_time:
+                last_notif_time = last_notification_time[notification_key]
+                if last_notif_time.date() == now.date():
+                    continue  # Уже отправляли сегодня
+
+            # Переход на ОТКЛЮЧЕНИЕ (on -> off)
+            if transition['from_status'] == 'on' and transition['to_status'] == 'off':
+                icon = '🔴' if transition['confirmed'] else '🟠'
+                status = 'підтверджено' if transition['confirmed'] else 'можливе'
+                queue_info = f' (Черга {transition["queue"]})' if transition['queue'] else ''
+
+                warning = (
+                    f'⚠️ <b>УВАГА! ВІДКЛЮЧЕННЯ</b>\n\n'
+                    f'Через ~5 хвилин\n'
+                    f'Час: <code>{transition["time_str"]}</code>\n'
+                    f'Статус: {icon} {status}{queue_info}'
+                )
+
+                await send_to_all_users(warning)
+                last_notification_time[notification_key] = now
+                logger.info(f'📢 Відправлено попередження про ВІДКЛЮЧЕННЯ в {transition["time_str"]}')
+
+            # Переход на ВКЛЮЧЕНИЕ (off -> on)
+            elif transition['from_status'] == 'off' and transition['to_status'] == 'on':
+                warning = (
+                    f'✅ <b>УВАГА! ВКЛЮЧЕННЯ</b>\n\n'
+                    f'Через ~5 хвилин буде світло\n'
+                    f'Час: <code>{transition["time_str"]}</code>'
+                )
+
+                await send_to_all_users(warning)
+                last_notification_time[notification_key] = now
+                logger.info(f'📢 Відправлено попередження про ВКЛЮЧЕННЯ в {transition["time_str"]}')
+
+
+async def monitoring_loop():
+    """Основной цикл мониторинга"""
+    await check_schedule()
+
+    while True:
+        try:
+            await asyncio.sleep(CHECK_INTERVAL)
+            await check_schedule()
+            await check_upcoming_disconnections()
+        except Exception as e:
+            logger.error(f'Ошибка в цикле мониторинга: {e}')
+            await asyncio.sleep(5)
+
+
+# ============================================================================
+# ОБРАБОТЧИКИ КОМАНД
+# ============================================================================
+
+
+@dp.message(Command('start'))
+async def cmd_start(message: Message):
+    """Обработчик /start"""
+    if message.chat.id not in ALLOWED_CHAT_IDS:
+        await message.answer('❌ У вас немає доступу до цього бота.')
+        return
+
+    await message.answer(
+        '👋 <b>Ласкаво просимо!</b>\n\n'
+        '🤖 <b>Бот для моніторингу графіку відключень світла</b>\n\n'
+        '✨ <b>Можливості:</b>\n'
+        '• 📊 Перегляд графіку на сьогодні і завтра\n'
+        '• 🔔 Автоматичні сповіщення за 5 хвилин до подій\n'
+        '• 🔄 Моніторинг змін графіку\n\n'
+        'Використовуйте кнопки нижче 👇',
+        parse_mode='HTML',
+        reply_markup=get_main_keyboard(),
+    )
+
+
+@dp.message(lambda msg: msg.text == 'ℹ️ Про бота')
+async def cmd_info(message: Message):
+    """Показывает информацию о боте"""
+    if message.chat.id not in ALLOWED_CHAT_IDS:
+        return
+
+    await message.answer(
+        '<b>ℹ️ Про бота</b>\n\n'
+        '🚀 <b>Версія:</b> 2.2 (Стабільна)\n\n'
+        '📝 <b>Реліз-ноути:</b>\n'
+        '├ 15.11.2024: Адаптація під оновлену логіку сайту VOE\n'
+        '├ Виправлено парсинг half.left та half.right\n'
+        '├ Покращено визначення підтвердження відключень\n'
+        '└ Оптимізовано обробку статусу для всієї години\n\n'
+        '⚡ <b>Функціональність:</b>\n'
+        '├ Моніторинг графіку 24/7\n'
+        '├ Сповіщення за 5 хвилин\n'
+        '├ Детальна статистика дня\n'
+        '└ Красива візуалізація\n\n'
+        '🔐 <b>Безпека:</b> Використовуються .env файли\n'
+        '💾 <b>Джерело:</b> voe.com.ua',
+        parse_mode='HTML',
+        reply_markup=get_main_keyboard(),
+    )
+
+
+@dp.message(Command('schedule'))
+async def cmd_schedule(message: Message):
+    """Показывает полный график"""
+    if message.chat.id not in ALLOWED_CHAT_IDS:
+        await message.answer('❌ У вас немає доступу.')
+        return
+
+    try:
+        await message.answer('⏳ Завантаження графіку...')
+        html = get_voe_html(VOE_CITY_ID, VOE_STREET_ID, VOE_HOUSE_ID)
+        schedule = parse_html(html)
+        text = format_schedule_message(schedule, days_to_show=2)
+        await message.answer(text, parse_mode='HTML', reply_markup=get_main_keyboard())
+    except Exception as e:
+        await message.answer(f'❌ <b>Помилка:</b> {str(e)}', parse_mode='HTML', reply_markup=get_main_keyboard())
+
+
+@dp.message(Command('today'))
+async def cmd_today(message: Message):
+    """Показывает график на сегодня"""
+    if message.chat.id not in ALLOWED_CHAT_IDS:
+        await message.answer('❌ У вас немає доступу.')
+        return
+
+    try:
+        await message.answer('⏳ Завантаження графіку сьогодні...')
+        html = get_voe_html(VOE_CITY_ID, VOE_STREET_ID, VOE_HOUSE_ID)
+        schedule = parse_html(html)
+        text = format_single_day_schedule(schedule, 0)
+        await message.answer(text, parse_mode='HTML', reply_markup=get_main_keyboard())
+    except Exception as e:
+        await message.answer(f'❌ <b>Помилка:</b> {str(e)}', parse_mode='HTML', reply_markup=get_main_keyboard())
+
+
+@dp.message(Command('tomorrow'))
+async def cmd_tomorrow(message: Message):
+    """Показывает график на завтра"""
+    if message.chat.id not in ALLOWED_CHAT_IDS:
+        await message.answer('❌ У вас немає доступу.')
+        return
+
+    try:
+        await message.answer('⏳ Завантаження графіку завтра...')
+        html = get_voe_html(VOE_CITY_ID, VOE_STREET_ID, VOE_HOUSE_ID)
+        schedule = parse_html(html)
+        text = format_single_day_schedule(schedule, 1)
+        await message.answer(text, parse_mode='HTML', reply_markup=get_main_keyboard())
+        # await message.answer("❌ Функція тимчасово недоступна. Чекаємо на оновлення сайту", parse_mode="HTML", reply_markup=get_main_keyboard())
+    except Exception as e:
+        await message.answer(f'❌ <b>Помилка:</b> {str(e)}', parse_mode='HTML', reply_markup=get_main_keyboard())
+
+
+@dp.message(Command('check'))
+async def cmd_check(message: Message):
+    """Принудительная проверка графика"""
+    if message.chat.id not in ALLOWED_CHAT_IDS:
+        await message.answer('❌ У вас немає доступу.')
+        return
+
+    try:
+        await message.answer('🔄 <b>Перевіряю графік...</b>', parse_mode='HTML')
+        html = get_voe_html(VOE_CITY_ID, VOE_STREET_ID, VOE_HOUSE_ID)
+        new_schedule = parse_html(html)
+
+        prefix = (
+            '✅ <b>Знайдено зміни!</b>\n\n'
+            if schedules_differ(last_schedule, new_schedule)
+            else '✓ <b>Графік без змін</b>\n\n'
+        )
+        result = prefix + format_schedule_message(new_schedule, days_to_show=2)
+
+        await message.answer(result, parse_mode='HTML', reply_markup=get_main_keyboard())
+    except Exception as e:
+        await message.answer(f'❌ <b>Помилка:</b> {str(e)}', parse_mode='HTML', reply_markup=get_main_keyboard())
+
+
+@dp.message()
+async def handle_text(message: Message):
+    """Обработчик текстовых сообщений и кнопок"""
+    if message.chat.id not in ALLOWED_CHAT_IDS:
+        return
+
+    text = message.text
+
+    # Кнопка "Графік"
+    if text == '📊 Графік':
+        await cmd_schedule(message)
+
+    # Кнопка "Сьогодні"
+    elif text == '📅 Сьогодні':
+        await cmd_today(message)
+
+    # Кнопка "Завтра"
+    elif text == '📅 Завтра':
+        await cmd_tomorrow(message)
+
+    # Кнопка "Оновити"
+    elif text == '🔄 Оновити':
+        await cmd_check(message)
+
+    # Кнопка "Про бота"
+    elif text == 'ℹ️ Про бота':
+        await cmd_info(message)
+
+    # Неизвестная команда
+    else:
+        await message.answer(
+            '❓ <b>Команда не розпізнана</b>\n\n'
+            'Використовуйте кнопки на клавіатурі або команди:\n'
+            '/start • /today • /tomorrow • /schedule • /check',
+            parse_mode='HTML',
+            reply_markup=get_main_keyboard(),
+        )
+
+
+# ============================================================================
+# ГЛАВНАЯ ФУНКЦИЯ
+# ============================================================================
+
+
+async def main():
+    """Главная функция"""
+    logger.info('=' * 50)
+    logger.info('ЗАПУСК БОТА V2.2 (stable 2.2, 15.11.2025)')
+    logger.info('=' * 50)
+
+    if not TELEGRAM_TOKEN or os.getenv('TELEGRAM_TOKEN', 'YOUR_TOKEN_HERE') == TELEGRAM_TOKEN:
+        logger.error('❌ TELEGRAM_TOKEN не конфігурований! Напишіть токен в .env файл')
+        return
+
+    if not ALLOWED_CHAT_IDS:
+        logger.error('❌ ALLOWED_CHAT_IDS не конфігуровані! Напишіть ID в .env файл')
+        return
+
+    logger.info(f'📌 Allowed chat ids: {ALLOWED_CHAT_IDS}')
+    logger.info(f'⏱ Інтервал перевірки: {CHECK_INTERVAL} сек')
+    logger.info('=' * 50)
+
+    # Запускаем мониторинг
+    monitoring_task = asyncio.create_task(monitoring_loop())
+
+    try:
+        await dp.start_polling(bot)
+    except KeyboardInterrupt:
+        logger.info('⏹ Бот зупинений користувачем')
+    finally:
+        monitoring_task.cancel()
+        await bot.session.close()
+        logger.info('✓ Підключення закрито')
+
+
+if __name__ == '__main__':
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        logger.info('⏹ Завершено')

dtek_notif/pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "dtek-notif"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = []

dtek_notif/requirements.txt

@@ -0,0 +1,5 @@
+requests>=2.31.0
+beautifulsoup4>=4.12.0
+aiogram>=3.3.0
+python-dotenv>=1.0.0
+aiohttp>=3.9.0

edu_master/PLAYWRIGHT_VERSION

@@ -1 +0,0 @@
-1.56.0

edu_master/compose.yaml

@@ -1,6 +1,6 @@
 services:
   redis:
-    image: redis:8.10.2-alpine
+    image: redis:8.10.1-alpine
     restart: unless-stopped
     volumes:
       - redis-data:/data
@@ -11,13 +11,13 @@ services:
       retries: 5
 
   playwright-service:
-    image: mcr.microsoft.com/playwright:v1.56.0-jammy
+    image: mcr.microsoft.com/playwright:v1.63.0-jammy
     restart: unless-stopped
     command: npx -y playwright@1.56.0 run-server --port 3000 --path /ws
 
   session-keeper:
     build: ./phpsessid-bot
-    image: gcr.forust.xyz/forust/session-keeper:prod
+    image: gcr.forust.xyz/forust/session-keeper:latest
     pull_policy: build
     env_file: .env
     restart: unless-stopped
@@ -33,7 +33,7 @@ services:
 
   webinar-checker:
     build: ./webinar-checker
-    image: gcr.forust.xyz/forust/webinar-checker:prod
+    image: gcr.forust.xyz/forust/webinar-checker:latest
     pull_policy: build
     env_file: .env
     restart: unless-stopped

edu_master/k8s/alerts.yaml

@@ -1,96 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: edu-master-webinar
-  namespace: edu-master
-  labels:
-    release: prometheus-stack
-spec:
-  groups:
-    - name: edu_master.webinar
-      rules:
-        # No successful webinar check for 5m (~2-3 missed 2-min checks).
-        # Catches: playwright hangs/timeouts, version skew, site changes, hung job.
-        # The last_success > 0 guard is mandatory: checker.py initialises
-        # last_success to 0, so without it `time() - 0` equals the current epoch
-        # and humanizeDuration renders ~20722d on every pod restart. Keep the
-        # duration expression on the left so $value stays the real gap.
-        - alert: WebinarCheckerNoSuccessfulCheck
-          expr: |
-            ((time() - webinar_check_last_success_timestamp_seconds) > 300)
-              and (webinar_check_last_success_timestamp_seconds > 0)
-              and (webinar_check_last_run_timestamp_seconds > 0)
-          for: 2m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Webinar checker has no successful check for 5m"
-            description: "edu-master/webinar-checker: last successful webinar check was {{ $value | humanizeDuration }} ago. Checks are failing or hanging (see consecutive failures alert). Notifications about new webinars are NOT being sent."
-
-        # Checks are running but none has ever succeeded since pod start.
-        # Split out from the rule above so a zeroed gauge never feeds
-        # humanizeDuration.
-        - alert: WebinarCheckerNeverSucceeded
-          expr: |
-            (webinar_check_last_success_timestamp_seconds == 0)
-              and (webinar_check_last_run_timestamp_seconds > 0)
-          for: 10m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Webinar checker has never completed a successful check"
-            description: 'edu-master/webinar-checker: checks have been running for 10m but not one has ever succeeded since the pod started, so every check is failing. Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).'
-
-        # Fast path: 3 consecutive failures (~6+ min at 2-min interval).
-        - alert: WebinarCheckerConsecutiveFailures
-          expr: |
-            webinar_check_consecutive_failures >= 3
-          for: 5m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Webinar checker failing consecutively"
-            description: 'edu-master/webinar-checker: {{ $value }} consecutive webinar check failures (timeout / playwright error / page error). Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).'
-
-        # Metrics endpoint not scraped for 10m: pod down, metrics server dead, or ServiceMonitor broken.
-        - alert: WebinarCheckerScrapeDown
-          expr: |
-            absent(webinar_check_last_run_timestamp_seconds) == 1
-          for: 10m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Webinar checker metrics missing"
-            description: "edu-master/webinar-checker: no metrics series for 10m. Pod may be down, metrics server dead, or ServiceMonitor/Service broken. Webinar checks are unobserved."
-
-        # EDU session lost: session-keeper down or credentials expired. Without PHPSESSID every check is skipped.
-        - alert: EduPhpsessidMissing
-          expr: |
-            edu_phpsessid_present == 0
-          for: 10m
-          labels:
-            severity: critical
-          annotations:
-            summary: "EDU_PHPSESSID missing"
-            description: "edu-master: EDU_PHPSESSID absent from redis for 10m. Webinar/diari/schedule checks are all skipped. Check session-keeper logs and EDU credentials."
-
-        # Hard deps: checker and playwright deployments unavailable.
-        - alert: WebinarCheckerDeploymentDown
-          expr: |
-            kube_deployment_status_replicas_unavailable{deployment="webinar-checker", namespace="edu-master"} > 0
-          for: 10m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Webinar checker deployment unavailable"
-            description: "edu-master/webinar-checker deployment has {{ $value }} unavailable replica(s) for 10m."
-
-        - alert: PlaywrightServiceDown
-          expr: |
-            kube_deployment_status_replicas_unavailable{deployment="playwright-service", namespace="edu-master"} > 0
-          for: 10m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Playwright service unavailable"
-            description: "edu-master/playwright-service deployment has {{ $value }} unavailable replica(s) for 10m. All webinar/diari/schedule checks fail without it."

edu_master/k8s/playwright.yaml

@@ -10,8 +10,6 @@ spec:
   selector:
     matchLabels:
       app: edu-master-playwright
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -19,18 +17,8 @@ spec:
     spec:
       containers:
         - name: playwright
-          # renovate: datasource=docker depName=mcr.microsoft.com/playwright versioning=docker
-          image: mcr.microsoft.com/playwright:v1.56.0-jammy
+          image: mcr.microsoft.com/playwright:v1.63.0-jammy
           imagePullPolicy: IfNotPresent
-          # p95 412M, max 478M over 7 days, no limit before. Request is set at p95
-          # so the pod is not an eviction candidate; the limit stays above 2x the
-          # request because browser page lifetimes are unpredictable.
-          resources:
-            requests:
-              cpu: "200m"
-              memory: "416Mi"
-            limits:
-              memory: "1Gi"
           command:
             - npx
             - -y

edu_master/k8s/redis.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: redis
-          image: redis:8.10.2-alpine
+          image: redis:8.10.1-alpine
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 6379
@@ -28,10 +28,10 @@ spec:
           resources:
             requests:
               cpu: 25m
-              memory: 32Mi
+              memory: 64Mi
             limits:
               cpu: 250m
-              memory: 128Mi
+              memory: 256Mi
           readinessProbe:
             exec:
               command: ["redis-cli", "ping"]

edu_master/k8s/secrets.yaml.example

@@ -21,8 +21,6 @@ stringData:
   WEBINAR_TELEGRAM_TOKEN: ""
   WEBINAR_ADMIN_ID: ""
   WEBINAR_CHECK_INTERVAL: "60"
-  # Prometheus metrics endpoint (scraped via ServiceMonitor, alerts in k8s/alerts.yaml)
-  METRICS_PORT: "8000"
   # Database
   REDIS_HOST: "redis"
   REDIS_PORT: "6379"

edu_master/k8s/service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: webinar-checker
-  namespace: edu-master
-  labels:
-    app: edu-master-webinar-checker
-spec:
-  selector:
-    app: edu-master-webinar-checker
-  ports:
-    - name: metrics
-      port: 8000
-      targetPort: metrics
-      protocol: TCP

edu_master/k8s/servicemonitor.yaml

@@ -1,16 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: webinar-checker
-  namespace: edu-master
-  labels:
-    release: prometheus-stack
-spec:
-  selector:
-    matchLabels:
-      app: edu-master-webinar-checker
-  endpoints:
-    - port: metrics
-      path: /metrics
-      interval: 30s
-      scrapeTimeout: 10s

edu_master/k8s/session-keeper.yaml

@@ -10,8 +10,6 @@ spec:
   selector:
     matchLabels:
       app: edu-master-session-keeper
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -19,7 +17,7 @@ spec:
     spec:
       initContainers:
         - name: wait-redis
-          image: redis:8.10.2-alpine
+          image: redis:8.10.1-alpine
           command:
             - /bin/sh
             - -ec
@@ -33,17 +31,18 @@ spec:
               echo "redis is ready"
       containers:
         - name: session-keeper
-          image: gcr.forust.xyz/forust/session-keeper:prod
+          image: gcr.forust.xyz/forust/session-keeper:latest
+          imagePullPolicy: Always
           envFrom:
             - secretRef:
                 name: edu-master-secrets
           resources:
             requests:
               cpu: 25m
-              memory: 32Mi
+              memory: 96Mi
             limits:
               cpu: 250m
-              memory: 128Mi
+              memory: 256Mi
           readinessProbe:
             exec:
               command: ["/bin/sh", "-ec", "redis-cli -h redis EXISTS EDU_PHPSESSID | grep -q 1"]

edu_master/k8s/webinar-checker.yaml

@@ -10,8 +10,6 @@ spec:
   selector:
     matchLabels:
       app: edu-master-webinar-checker
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -21,7 +19,7 @@ spec:
       # redis healthy -> session-keeper healthy (EXISTS EDU_PHPSESSID) -> playwright started
       initContainers:
         - name: wait-deps
-          image: redis:8.10.2-alpine
+          image: redis:8.10.1-alpine
           command:
             - /bin/sh
             - -ec
@@ -47,19 +45,8 @@ spec:
               echo "playwright ok"
       containers:
         - name: webinar-checker
-          image: gcr.forust.xyz/forust/webinar-checker:prod
-          ports:
-            - name: metrics
-              containerPort: 8000
-              protocol: TCP
-          readinessProbe:
-            httpGet:
-              path: /health
-              port: metrics
-            periodSeconds: 10
-            timeoutSeconds: 3
-            failureThreshold: 12
-            initialDelaySeconds: 10
+          image: gcr.forust.xyz/forust/webinar-checker:latest
+          imagePullPolicy: Always
           envFrom:
             - secretRef:
                 name: edu-master-secrets
@@ -69,7 +56,7 @@ spec:
           resources:
             requests:
               cpu: "50m"
-              memory: "192Mi"
+              memory: "128Mi"
             limits:
               cpu: "600m"
-              memory: "384Mi"
+              memory: "512Mi"

edu_master/webinar-checker/Dockerfile

@@ -2,11 +2,8 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# renovate: datasource=pypi depName=playwright versioning=pep440
-ARG PLAYWRIGHT_VERSION=1.56.0
-
-# Install dependencies - PLAYWRIGHT_VERSION is single-source, renovate updates ARG above and all other places via regexManagers
-RUN pip install --no-cache-dir pip==25.0.1 && pip install --no-cache-dir playwright==${PLAYWRIGHT_VERSION} redis==5.2.1 requests==2.32.3 "python-telegram-bot[job-queue]==21.10"
+# Install dependencies
+RUN pip install --no-cache-dir pip==25.0.1 && pip install --no-cache-dir playwright==1.56.0 redis==5.2.1 requests==2.32.3 "python-telegram-bot[job-queue]==21.10"
 
 COPY checker.py .
 

edu_master/webinar-checker/checker.py

@@ -1,15 +1,12 @@
-import asyncio
 import contextlib
 import json
 import logging
 import os
 import re
 import tempfile
-import threading
 import time
 from datetime import datetime, timedelta
 from html import escape
-from http.server import BaseHTTPRequestHandler, HTTPServer
 
 import redis
 from playwright.async_api import async_playwright
@@ -51,106 +48,6 @@ USER_AGENT = _env(
 )
 WEBINAR_TELEGRAM_TOKEN = _env('WEBINAR_TELEGRAM_TOKEN')
 ADMIN_ID = int(_env('WEBINAR_ADMIN_ID', '0'))
-METRICS_PORT = int(_env('METRICS_PORT', '8000'))
-
-# --- Prometheus metrics (stdlib only, no extra deps) ---
-# Scraped by prometheus-stack via ServiceMonitor (edu_master/k8s/servicemonitor.yaml).
-# Critical alerts in edu_master/k8s/alerts.yaml fire to Telegram via Alertmanager.
-_METRICS_LOCK = threading.Lock()
-_METRICS = {
-    'last_run': 0.0,  # Unix ts of last check start
-    'last_success': 0.0,  # Unix ts of last successful check
-    'last_duration': 0.0,  # Duration of last check in seconds
-    'success_total': 0,
-    'failure_total': 0,
-    'consecutive_failures': 0,
-    'phpsessid_present': 1,  # 1 if EDU_PHPSESSID found in redis, else 0
-}
-
-
-def _metric_check_start():
-    with _METRICS_LOCK:
-        _METRICS['last_run'] = time.time()
-
-
-def _metric_check_ok(duration: float):
-    now = time.time()
-    with _METRICS_LOCK:
-        _METRICS['last_success'] = now
-        _METRICS['last_duration'] = duration
-        _METRICS['success_total'] += 1
-        _METRICS['consecutive_failures'] = 0
-        _METRICS['phpsessid_present'] = 1
-
-
-def _metric_check_fail(duration: float, phpsessid_missing: bool = False):
-    with _METRICS_LOCK:
-        _METRICS['last_duration'] = duration
-        _METRICS['failure_total'] += 1
-        _METRICS['consecutive_failures'] += 1
-        _METRICS['phpsessid_present'] = 0 if phpsessid_missing else 1
-
-
-def _metrics_render() -> bytes:
-    with _METRICS_LOCK:
-        m = dict(_METRICS)
-    lines = [
-        '# HELP webinar_check_last_run_timestamp_seconds Unix timestamp of last webinar check start.',
-        '# TYPE webinar_check_last_run_timestamp_seconds gauge',
-        f'webinar_check_last_run_timestamp_seconds {m["last_run"]}',
-        '# HELP webinar_check_last_success_timestamp_seconds Unix timestamp of last successful webinar check.',
-        '# TYPE webinar_check_last_success_timestamp_seconds gauge',
-        f'webinar_check_last_success_timestamp_seconds {m["last_success"]}',
-        '# HELP webinar_check_last_duration_seconds Duration of last webinar check in seconds.',
-        '# TYPE webinar_check_last_duration_seconds gauge',
-        f'webinar_check_last_duration_seconds {m["last_duration"]}',
-        '# HELP webinar_check_success_total Total successful webinar checks.',
-        '# TYPE webinar_check_success_total counter',
-        f'webinar_check_success_total {m["success_total"]}',
-        '# HELP webinar_check_failure_total Total failed webinar checks (timeout, playwright error, page error).',
-        '# TYPE webinar_check_failure_total counter',
-        f'webinar_check_failure_total {m["failure_total"]}',
-        '# HELP webinar_check_consecutive_failures Consecutive failed webinar checks (reset on success).',
-        '# TYPE webinar_check_consecutive_failures gauge',
-        f'webinar_check_consecutive_failures {m["consecutive_failures"]}',
-        '# HELP edu_phpsessid_present 1 if EDU_PHPSESSID exists in redis, 0 otherwise.',
-        '# TYPE edu_phpsessid_present gauge',
-        f'edu_phpsessid_present {m["phpsessid_present"]}',
-    ]
-    return ('\n'.join(lines) + '\n').encode()
-
-
-class _MetricsHandler(BaseHTTPRequestHandler):
-    def do_GET(self):
-        if self.path == '/metrics':
-            body = _metrics_render()
-            self.send_response(200)
-            self.send_header('Content-Type', 'text/plain; version=0.0.4')
-            self.send_header('Content-Length', str(len(body)))
-            self.end_headers()
-            self.wfile.write(body)
-        elif self.path in ('/healthz', '/health'):
-            body = b'ok\n'
-            self.send_response(200)
-            self.send_header('Content-Type', 'text/plain')
-            self.send_header('Content-Length', str(len(body)))
-            self.end_headers()
-            self.wfile.write(body)
-        else:
-            self.send_response(404)
-            self.end_headers()
-
-    def log_message(self, *args):
-        pass  # keep bot logs clean
-
-
-def start_metrics_server(port: int = METRICS_PORT):
-    server = HTTPServer(('0.0.0.0', port), _MetricsHandler)  # noqa: S104 - k8s ServiceMonitor scrapes pod IP
-    thread = threading.Thread(target=server.serve_forever, name='metrics-server', daemon=True)
-    thread.start()
-    logger.info(f'Metrics server listening on :{port}/metrics')
-    return server
-
 
 # Redis Keys
 KEY_WHITELIST = 'bot:whitelist'
@@ -700,9 +597,8 @@ async def _collect_event_times(page) -> dict:
 async def fetch_diary_data(phpsessid: str) -> dict | None:
     logger.info('Fetching diary data via Playwright...')
     try:
-        async with asyncio.timeout(60):
         async with async_playwright() as p:
-                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
+            browser = await p.chromium.connect(PLAYWRIGHT_WS)
             try:
                 context_browser = await browser.new_context(user_agent=USER_AGENT)
                 await context_browser.add_cookies(
@@ -711,7 +607,7 @@ async def fetch_diary_data(phpsessid: str) -> dict | None:
                 page = await context_browser.new_page()
 
                 try:
-                        await asyncio.wait_for(page.goto(DIARY_URL, wait_until='domcontentloaded'), timeout=30)
+                    await page.goto(DIARY_URL, wait_until='domcontentloaded')
                     await page.wait_for_selector('table.calendar', timeout=10000)
                     await page.wait_for_timeout(1500)
 
@@ -750,16 +646,10 @@ async def fetch_diary_data(phpsessid: str) -> dict | None:
                     logger.error(f'Error parsing diary: {e}')
                     return None
                 finally:
-                        with contextlib.suppress(Exception):
-                            await asyncio.wait_for(page.close(), timeout=5)
-                        with contextlib.suppress(Exception):
-                            await asyncio.wait_for(context_browser.close(), timeout=5)
+                    await page.close()
+                    await context_browser.close()
             finally:
-                    with contextlib.suppress(Exception):
-                        await asyncio.wait_for(browser.close(), timeout=5)
-    except TimeoutError:
-        logger.error('Diary fetch timed out (60s)')
-        return None
+                await browser.close()
     except Exception as e:
         logger.error(f'Playwright error in diary fetch: {e}')
         return None
@@ -1041,9 +931,8 @@ def _parse_schedule_html(table_html: str) -> dict:
 async def fetch_schedule_data(phpsessid: str) -> dict | None:
     logger.info('Fetching schedule data via Playwright...')
     try:
-        async with asyncio.timeout(60):
         async with async_playwright() as p:
-                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
+            browser = await p.chromium.connect(PLAYWRIGHT_WS)
             try:
                 context_browser = await browser.new_context(user_agent=USER_AGENT)
                 await context_browser.add_cookies(
@@ -1052,7 +941,7 @@ async def fetch_schedule_data(phpsessid: str) -> dict | None:
                 page = await context_browser.new_page()
 
                 try:
-                        await asyncio.wait_for(page.goto(SCHEDULE_URL, wait_until='domcontentloaded'), timeout=30)
+                    await page.goto(SCHEDULE_URL, wait_until='domcontentloaded')
                     await page.wait_for_selector('table.schedule-table', timeout=10000)
                     await page.wait_for_timeout(1500)
 
@@ -1080,16 +969,10 @@ async def fetch_schedule_data(phpsessid: str) -> dict | None:
                     logger.error(f'Error parsing schedule: {e}')
                     return None
                 finally:
-                        with contextlib.suppress(Exception):
-                            await asyncio.wait_for(page.close(), timeout=5)
-                        with contextlib.suppress(Exception):
-                            await asyncio.wait_for(context_browser.close(), timeout=5)
+                    await page.close()
+                    await context_browser.close()
             finally:
-                    with contextlib.suppress(Exception):
-                        await asyncio.wait_for(browser.close(), timeout=5)
-    except TimeoutError:
-        logger.error('Schedule fetch timed out (60s)')
-        return None
+                await browser.close()
     except Exception as e:
         logger.error(f'Playwright error in schedule fetch: {e}')
         return None
@@ -1602,13 +1485,10 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
         int: Number of webinars found, or None if check failed
     """
     logger.info('Running webinar check...')
-    _t0 = time.time()
-    _metric_check_start()
 
     phpsessid = redis_client.get(KEY_PHPSESSID)
     if not phpsessid:
         logger.warning('PHPSESSID missing. Skipping check.')
-        _metric_check_fail(time.time() - _t0, phpsessid_missing=True)
         # --- DEBUG LOGGING ---
         try:
             with open('phpsessid_missing.log', 'a') as f:
@@ -1622,10 +1502,9 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
     content = ''
 
     try:
-        async with asyncio.timeout(90):
         async with async_playwright() as p:
             # Connect to remote Playwright service
-                browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
+            browser = await p.chromium.connect(PLAYWRIGHT_WS)
 
             try:
                 # Create browser context with user agent
@@ -1641,7 +1520,7 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
 
                 try:
                     # Navigate to webinar page
-                        await asyncio.wait_for(page.goto(WEBINAR_URL, wait_until='domcontentloaded'), timeout=30)
+                    await page.goto(WEBINAR_URL, wait_until='domcontentloaded')
 
                     # Wait for the table to load
                     await page.wait_for_selector('#meetings table', timeout=10000)
@@ -1685,25 +1564,16 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
                         if page and not content:
                             content = await page.content()
 
-                        _metric_check_fail(time.time() - _t0)
                     return None
                 finally:
-                        with contextlib.suppress(Exception):
-                            await asyncio.wait_for(page.close(), timeout=5)
-                        with contextlib.suppress(Exception):
-                            await asyncio.wait_for(context_browser.close(), timeout=5)
+                    await page.close()
+                    await context_browser.close()
 
             finally:
-                    with contextlib.suppress(Exception):
-                        await asyncio.wait_for(browser.close(), timeout=5)
+                await browser.close()
 
-    except TimeoutError:
-        logger.error('Webinar check timed out after 90s (playwright hang)')
-        _metric_check_fail(time.time() - _t0)
-        return None
     except Exception as e:
         logger.error(f'Playwright error: {e}')
-        _metric_check_fail(time.time() - _t0)
         return None
 
     # --- DEBUG LOGGING (Saving last response content) ---
@@ -1767,7 +1637,6 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
         else:
             logger.info(f'Found {len(current_webinars)} webinar(s), but all are already known')
 
-    _metric_check_ok(time.time() - _t0)
     return len(current_webinars)
 
 
@@ -1811,8 +1680,6 @@ def main():
     job_queue = app.job_queue
     job_queue.run_repeating(check_webinars_job, interval=WEBINAR_CHECK_INTERVAL, first=10)
 
-    start_metrics_server()
-
     logger.info('Bot started polling...')
     app.run_polling()
 

errorpages/compose.yaml

@@ -1,7 +1,7 @@
 services:
   errorpage:
     build: .
-    image: gcr.forust.xyz/forust/error-pages:prod
+    image: gcr.forust.xyz/forust/error-pages:latest
     pull_policy: build
     container_name: error-pages
     restart: unless-stopped

errorpages/k8s/error-pages.yaml

@@ -20,8 +20,6 @@ spec:
   selector:
     matchLabels:
       app: error-pages
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -29,21 +27,7 @@ spec:
     spec:
       containers:
         - name: error-pages
-          image: gcr.forust.xyz/forust/error-pages:prod
-          # p95 6M, max 10M, no limit before.
-          resources:
-            requests:
-              cpu: "10m"
-              memory: "32Mi"
-            limits:
-              memory: "128Mi"
+          image: gcr.forust.xyz/forust/error-pages:latest
           ports:
             - containerPort: 80
-          readinessProbe:
-            httpGet:
-              path: /404.html
-              port: 80
-            periodSeconds: 10
-            timeoutSeconds: 2
-            failureThreshold: 3
 ---

gitea/compose.yaml

@@ -1,6 +1,6 @@
 services:
   server:
-    image: docker.gitea.com/gitea:28.0.0
+    image: docker.gitea.com/gitea:1.27.3
     container_name: gitea
     restart: always
     environment:
@@ -13,12 +13,9 @@ services:
       - GITEA__database__PASSWD=gitea
       - GITEA__database__NAME=gitea
       # Server
-      - GITEA__server__ROOT_URL=https://git.forust.xyz
+      - GITEA__server__ROOT_URL=https://gitea.forust.xyz
       - GITEA__server__SSH_DOMAIN=gitssh.forust.xyz
       - GITEA__server__SSH_PORT=2221
-      # Pin 28.0 defaults explicitly (see k8s/config.yaml for rationale)
-      - GITEA__service__DISABLE_REGISTRATION=true
-      - GITEA__actions__RUN_RETENTION_DAYS=90
       # Mailer
       - GITEA__mailer__ENABLED=true
       - GITEA__mailer__FROM=${SERVICE_EMAIL}
@@ -37,7 +34,7 @@ services:
       - "traefik.http.services.gitea.loadbalancer.server.port=3000"
 
       # Prod Router
-      - "traefik.http.routers.gitea.rule=Host(`git.forust.xyz`) || Host(`gitea.forust.xyz`)"
+      - "traefik.http.routers.gitea.rule=Host(`gitea.forust.xyz`)"
       - "traefik.http.routers.gitea.entrypoints=websecure"
       - "traefik.http.routers.gitea.tls.certresolver"
       # Local Router

gitea/k8s/certificates.yaml

@@ -1,30 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: gitea-prod-tls
-  namespace: gitea
-spec:
-  secretName: gitea-prod-tls
-  dnsNames:
-    - gcr.forust.xyz
-    - gitea.forust.xyz
-    - git.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: gitea
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

gitea/k8s/config.yaml

@@ -4,14 +4,11 @@ metadata:
   name: gitea-config
   namespace: gitea
 data:
-  GITEA__server__ROOT_URL: "https://git.forust.xyz"
+  GITEA__server__DOMAIN: "gitea.forust.xyz"
+  GITEA__server__ROOT_URL: "https://gitea.forust.xyz"
   GITEA__server__SSH_DOMAIN: "gitssh.forust.xyz"
   GITEA__server__SSH_PORT: "2221"
 
-  GITEA__service__DISABLE_REGISTRATION: "true"
-
-  GITEA__actions__RUN_RETENTION_DAYS: "90"
-
   GITEA__database__DB_TYPE: "postgres"
   GITEA__database__HOST: "postgres.database.svc.cluster.local:5432"
   GITEA__database__NAME: "gitea"
@@ -20,12 +17,6 @@ data:
 
   GITEA__mailer__ENABLED: "false"
 
-  # No code/issue search needed: bleve reindexes the whole issue index on
-  # every pod restart (cron.rebuild_issue_indexer RUN_AT_START) and hammers
-  # the rotational disk for an hour. "db" serves issue search from postgres.
-  GITEA__indexer__ISSUE_INDEXER_TYPE: "db"
-  GITEA__indexer__REPO_INDEXER_ENABLED: "false"
-
   GITEA__log__logger.access.MODE: "console, file"
   USER_UID: "1000"
   USER_GID: "1000"

gitea/k8s/gitea.yaml

@@ -24,8 +24,6 @@ spec:
   selector:
     matchLabels:
       app: gitea
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -33,7 +31,7 @@ spec:
     spec:
       containers:
         - name: gitea
-          image: gitea/gitea:28.0.0
+          image: docker.gitea.com/gitea:1.27.3
           envFrom:
             - configMapRef:
                 name: gitea-config
@@ -49,10 +47,10 @@ spec:
               mountPath: /data
           resources:
             requests:
-              memory: "320Mi"
-              cpu: "100m"
+              memory: "512Mi"
+              cpu: "300m"
             limits:
-              memory: "1Gi"
+              memory: "1.5Gi"
               cpu: "1300m"
       volumes:
         - name: gitea-data

gitea/k8s/ingress.yaml

@@ -7,18 +7,24 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: Host(`gitea.forust.xyz`) || Host(`git.forust.xyz`)
+    - match: Host(`gitea.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: gitea-service
           port: 3000
     - match: Host(`gcr.forust.xyz`) && PathPrefix(`/v2`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: gitea-service
           port: 3000
   tls:
-    secretName: gitea-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -29,7 +35,7 @@ spec:
   entryPoints:
     - websecure
   routes:
-    - match: (Host(`gitea.workstation.internal`) || Host(`gitea.gigaforust.internal`)) || (Host(`git.workstation.internal`) || Host(`git.gigaforust.internal`))
+    - match: Host(`gitea.workstation.internal`) || Host(`gitea.gigaforust.internal`)
       kind: Rule
       services:
         - name: gitea-service
@@ -39,9 +45,6 @@ spec:
       services:
         - name: gitea-service
           port: 3000
-  tls:
-    secretName: internal-wildcard-tls
-
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRouteTCP

glance/k8s/certificates.yaml

@@ -1,29 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: glance-prod-tls
-  namespace: glance
-spec:
-  secretName: glance-prod-tls
-  dnsNames:
-    - forust.xyz
-    - www.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: glance
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

glance/k8s/glance-config.yaml

@@ -177,8 +177,8 @@ data:
                   # url: https://gitssh.forust.xyz
                 # - title: gcr.forust.xyz
                   # url: https://gcr.forust.xyz/v2/
-                - title: git.forust.xyz
-                  url: https://git.forust.xyz
+                - title: gitea.forust.xyz
+                  url: https://gitea.forust.xyz
                 - title: nextcloud.forust.xyz
                   url: https://nextcloud.forust.xyz
                 - title: mc.forust.xyz

glance/k8s/glance.yaml

@@ -20,8 +20,6 @@ spec:
   selector:
     matchLabels:
       app: glance
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -59,10 +57,10 @@ spec:
           resources:
             requests:
               cpu: "50m"
-              memory: "32Mi"
+              memory: "64Mi"
             limits:
               cpu: "200m"
-              memory: "128Mi"
+              memory: "256Mi"
       volumes:
         - name: glance-config
           configMap:

glance/k8s/ingress.yaml

@@ -16,7 +16,7 @@ spec:
         - name: glance-service
           port: 8080
   tls:
-    secretName: glance-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -35,9 +35,6 @@ spec:
       services:
         - name: glance-service
           port: 8080
-  tls:
-    secretName: internal-wildcard-tls
-
 ---
 apiVersion: traefik.io/v1alpha1
 kind: Middleware

headscale/compose.yaml

@@ -1,6 +1,6 @@
 services:
   headscale:
-    image: headscale/headscale:v0.29.4
+    image: headscale/headscale:0.29.3
     restart: unless-stopped
     container_name: headscale-server
     command: serve

headscale/k8s/routing/certificates.yaml

@@ -1,41 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: headscale-prod-tls
-  namespace: headscale
-spec:
-  secretName: headscale-prod-tls
-  dnsNames:
-    - hs.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: headplane-prod-tls
-  namespace: headscale
-spec:
-  secretName: headplane-prod-tls
-  dnsNames:
-    - hp.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: headscale
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

headscale/k8s/routing/ingress.yaml

@@ -23,16 +23,22 @@ spec:
           port: 8080
     - match: Host(`hs.forust.xyz`) && PathPrefix(`/admin`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: headscale-ui-external
           port: 80
     - match: Host(`hs.forust.xyz`) && PathPrefix(`/metrics`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: headscale-server-external
           port: 9090
   tls:
-    secretName: headscale-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -47,6 +53,8 @@ spec:
       kind: Rule
       middlewares:
         - name: headplane-prefix
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: headplane-external
           port: 3000
@@ -56,7 +64,7 @@ spec:
         - name: headplane-external
           port: 3000
   tls:
-    secretName: headplane-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -82,9 +90,6 @@ spec:
       services:
         - name: headscale-server-external
           port: 9090
-  tls:
-    secretName: internal-wildcard-tls
-
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -105,5 +110,3 @@ spec:
       services:
         - name: headplane-external
           port: 3000
-  tls:
-    secretName: internal-wildcard-tls

homepages/compose.yaml

@@ -3,7 +3,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.forust
-    image: gcr.forust.xyz/forust/forust-homepage:prod
+    image: gcr.forust.xyz/forust/forust-homepage:latest
     pull_policy: build
     # ports:
     #   - "8085:80"
@@ -35,7 +35,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.xdfnx
-    image: gcr.forust.xyz/forust/xdfnx-homepage:prod
+    image: gcr.forust.xyz/forust/xdfnx-homepage:latest
     pull_policy: build
     restart: unless-stopped
     # ports:

homepages/forust_files/index.html

@@ -174,7 +174,7 @@
         <h2>./projects</h2>
         <ul class="repo-list">
           <li>
-            <a href="https://git.forust.xyz/forust/gosleep" target="_blank">forust/gosleep</a>
+            <a href="https://gitea.forust.xyz/forust/gosleep" target="_blank">forust/gosleep</a>
             <span class="comment">// linux sleep timer written in rust (originally in go)</span>
           </li>
         </ul>

homepages/k8s/certificates.yaml

@@ -1,42 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: forust-homepage-prod-tls
-  namespace: homepages
-spec:
-  secretName: forust-homepage-prod-tls
-  dnsNames:
-    - forust.xyz
-    - www.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: xdfnx-homepage-prod-tls
-  namespace: homepages
-spec:
-  secretName: xdfnx-homepage-prod-tls
-  dnsNames:
-    - xdfnx.cfd
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: homepages
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

homepages/k8s/gateway.yaml

@@ -1,31 +0,0 @@
-# Gateway API PoC for homepages. Lives next to the TLS secrets so
-# certificateRefs stay same-namespace and no ReferenceGrant is needed.
-# Listener ports must match the Traefik entryPoints (80/443),
-# otherwise Traefik marks the listener Invalid.
-# Local .internal hosts are deliberately left on IngressRoute,
-# only prod is migrated here.
-apiVersion: gateway.networking.k8s.io/v1
-kind: Gateway
-metadata:
-  name: homepages
-  namespace: homepages
-spec:
-  gatewayClassName: traefik
-  listeners:
-    - name: http
-      protocol: HTTP
-      port: 80
-      allowedRoutes:
-        namespaces:
-          from: Same
-    - name: https
-      protocol: HTTPS
-      port: 443
-      tls:
-        mode: Terminate
-        certificateRefs:
-          - name: forust-homepage-prod-tls
-          - name: xdfnx-homepage-prod-tls
-      allowedRoutes:
-        namespaces:
-          from: Same

homepages/k8s/homepages.yaml

@@ -20,8 +20,6 @@ spec:
   selector:
     matchLabels:
       app: forust-homepage
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -29,22 +27,16 @@ spec:
     spec:
       containers:
         - name: forust-homepage
-          image: gcr.forust.xyz/forust/forust-homepage:prod
+          image: gcr.forust.xyz/forust/forust-homepage:latest
+          imagePullPolicy: Always
           ports:
             - containerPort: 80
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 80
-            periodSeconds: 10
-            timeoutSeconds: 2
-            failureThreshold: 3
           resources:
             requests:
-              memory: "32Mi"
+              memory: "10Mi"
               cpu: "20m"
             limits:
-              memory: "128Mi"
+              memory: "100Mi"
               cpu: "50m"
 ---
 apiVersion: v1
@@ -69,8 +61,6 @@ spec:
   selector:
     matchLabels:
       app: xdfnx-homepage
-  strategy:
-    type: Recreate
   template:
     metadata:
       labels:
@@ -78,20 +68,14 @@ spec:
     spec:
       containers:
         - name: xdfnx-homepage
-          image: gcr.forust.xyz/forust/xdfnx-homepage:prod
+          image: gcr.forust.xyz/forust/xdfnx-homepage:latest
+          imagePullPolicy: Always
           ports:
             - containerPort: 80
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 80
-            periodSeconds: 10
-            timeoutSeconds: 2
-            failureThreshold: 3
           resources:
             requests:
-              memory: "32Mi"
+              memory: "10Mi"
               cpu: "20m"
             limits:
-              memory: "128Mi"
+              memory: "100Mi"
               cpu: "50m"

homepages/k8s/httproute.yaml

@@ -1,69 +0,0 @@
-# PoC: homepages prod hosts via Gateway API.
-# Runs alongside k8s/ingress.yaml - delete the prod IngressRoutes only after verification.
-# There are no local .internal hosts here, they stay on the local IngressRoute.
-# No per-route security middlewares: L3 enforcement moved to the host
-# firewall bouncer, so HTTPRoutes stay clean.
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: homepages-http-redirect
-  namespace: homepages
-spec:
-  parentRefs:
-    - name: homepages
-      kind: Gateway
-      sectionName: http
-  hostnames:
-    - forust.xyz
-    - www.forust.xyz
-    - xdfnx.cfd
-  rules:
-    - filters:
-        - type: RequestRedirect
-          requestRedirect:
-            scheme: https
-            statusCode: 301
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: forust-homepage-https
-  namespace: homepages
-spec:
-  parentRefs:
-    - name: homepages
-      kind: Gateway
-      sectionName: https
-  hostnames:
-    - forust.xyz
-    - www.forust.xyz
-  rules:
-    - matches:
-        - path:
-            type: PathPrefix
-            value: /
-      backendRefs:
-        - name: forust-homepage-service
-          port: 80
----
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: xdfnx-homepage-https
-  namespace: homepages
-spec:
-  parentRefs:
-    - name: homepages
-      kind: Gateway
-      sectionName: https
-  hostnames:
-    - xdfnx.cfd
-  rules:
-    - matches:
-        - path:
-            type: PathPrefix
-            value: /
-      backendRefs:
-        - name: xdfnx-homepage-service
-          port: 80

homepages/k8s/ingress.yaml

@@ -9,12 +9,15 @@ spec:
   routes:
     - match: Host(`forust.xyz`) || Host(`www.forust.xyz`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       priority: 10
       services:
         - name: forust-homepage-service
           port: 80
   tls:
-    secretName: forust-homepage-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -31,9 +34,6 @@ spec:
       services:
         - name: forust-homepage-service
           port: 80
-  tls:
-    secretName: internal-wildcard-tls
-
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -46,11 +46,14 @@ spec:
   routes:
     - match: Host(`xdfnx.cfd`)
       kind: Rule
+      middlewares:
+        - name: crowdsec-bouncer
+          namespace: crowdsec
       services:
         - name: xdfnx-homepage-service
           port: 80
   tls:
-    secretName: xdfnx-homepage-prod-tls
+    certResolver: letsencrypt
 ---
 apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
@@ -66,5 +69,3 @@ spec:
       services:
         - name: xdfnx-homepage-service
           port: 80
-  tls:
-    secretName: internal-wildcard-tls

immich/.env.example

@@ -1,24 +0,0 @@
-# You can find documentation for all the supported env variables at https://docs.immich.app/install/environment-variables
-
-# The location where your uploaded files are stored. The k8s manifests bind
-# mount /mnt/immich/library, which is the sdc9 partition - the same place, so
-# the two deployment paths look at one library.
-UPLOAD_LOCATION=/mnt/immich/library
-
-# The location where your database files are stored. Network shares are not supported for the database
-DB_DATA_LOCATION=./postgres
-
-# To set a timezone, uncomment the next line and change Etc/UTC to a TZ identifier from this list: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List
-# TZ=Etc/UTC
-
-# The Immich version to use. You can pin this to a specific version like "v2.1.0"
-IMMICH_VERSION=v3
-
-# Connection secret for postgres. You should change it to a random password
-# Please use only the characters `A-Za-z0-9`, without special characters or spaces
-DB_PASSWORD=postgres
-
-# The values below this line do not need to be changed
-###################################################################################
-DB_USERNAME=postgres
-DB_DATABASE_NAME=immich

immich/compose.yaml

@@ -1,63 +0,0 @@
-name: immich
-
-services:
-  immich-server:
-    container_name: immich_server
-    image: ghcr.io/immich-app/immich-server:v3
-    volumes:
-      - ${UPLOAD_LOCATION}:/data
-      - /etc/localtime:/etc/localtime:ro
-    env_file:
-      - .env
-    ports:
-      - "2283:2283"
-    depends_on:
-      - redis
-      - database
-    restart: always
-    healthcheck:
-      disable: false
-
-  immich-machine-learning:
-    container_name: immich_machine_learning
-    # For hardware acceleration, add one of -[armnn, cuda, rocm, openvino, rknn] to the image tag.
-    # Example tag: ${IMMICH_VERSION:-release}-cuda
-    image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release}
-    # extends: # uncomment this section for hardware acceleration - see https://docs.immich.app/features/ml-hardware-acceleration
-    #   file: hwaccel.ml.yml
-    #   service: cpu # set to one of [armnn, cuda, rocm, openvino, openvino-wsl, rknn] for accelerated inference - use the `-wsl` version for WSL2 where applicable
-    volumes:
-      - model-cache:/cache
-    env_file:
-      - .env
-    restart: always
-    healthcheck:
-      disable: false
-
-  redis:
-    container_name: immich_redis
-    image: docker.io/valkey/valkey:9@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd
-    healthcheck:
-      test: redis-cli ping | grep -q PONG || exit 1
-    restart: always
-
-  database:
-    container_name: immich_postgres
-    image: ghcr.io/immich-app/postgres:16-vectorchord0.4.3-pgvectors0.2.0@sha256:1a078b237c1d9b420b0ee59147386b4aa60d3a07a8e6a402fc84a57e41b043a4
-    environment:
-      POSTGRES_PASSWORD: ${DB_PASSWORD}
-      POSTGRES_USER: ${DB_USERNAME}
-      POSTGRES_DB: ${DB_DATABASE_NAME}
-      POSTGRES_INITDB_ARGS: "--data-checksums"
-    # Uncomment the DB_STORAGE_TYPE: 'HDD' var if your database isn't stored on SSDs
-    # DB_STORAGE_TYPE: 'HDD'
-    volumes:
-      # Do not edit the next line. If you want to change the database storage location on your system, edit the value of DB_DATA_LOCATION in the .env file
-      - ${DB_DATA_LOCATION}:/var/lib/postgresql/data
-    shm_size: 128mb
-    restart: always
-    healthcheck:
-      disable: false
-
-volumes:
-  model-cache:

immich/k8s/certificates.yaml

@@ -1,28 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: immich-prod-tls
-  namespace: immich
-spec:
-  secretName: immich-prod-tls
-  dnsNames:
-    - immich.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: immich
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

immich/k8s/config.yaml

@@ -1,24 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: immich-config
-  namespace: immich
-data:
-  TZ: "Europe/Bratislava"
-
-  # The database in this namespace, not the shared one in the database
-  # namespace: v3 needs VectorChord, and only the dedicated image carries it.
-  DB_HOSTNAME: "immich-postgres"
-  DB_PORT: "5432"
-  DB_USERNAME: "immich"
-  DB_DATABASE_NAME: "immich"
-  DB_SSL_MODE: "disable"
-  DB_VECTOR_EXTENSION: "vectorchord"
-
-  REDIS_HOSTNAME: "immich-valkey"
-  REDIS_PORT: "6379"
-
-  # Traefik is the only client of the server, and it is a pod: the address immich
-  # sees is inside the node's pod CIDR. Without this the server does not trust
-  # X-Forwarded-For and every request looks like it came from Traefik itself.
-  IMMICH_TRUSTED_PROXIES: "10.244.0.0/24"

immich/k8s/immich.yaml

@@ -1,95 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: immich-service
-  namespace: immich
-spec:
-  selector:
-    app: immich
-  ports:
-    - name: http
-      port: 2283
-      targetPort: 2283
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: immich-deployment
-  namespace: immich
-  labels:
-    app: immich
-spec:
-  replicas: 2
-  selector:
-    matchLabels:
-      app: immich
-  template:
-    metadata:
-      labels:
-        app: immich
-    spec:
-      containers:
-        - name: immich
-          image: ghcr.io/immich-app/immich-server:v3
-          envFrom:
-            - configMapRef:
-                name: immich-config
-            - secretRef:
-                name: immich-secrets
-          ports:
-            - name: http
-              containerPort: 2283
-          volumeMounts:
-            - name: immich-data
-              mountPath: /data
-          # The first boot runs migrations and warms the transcoder, which can
-          # take minutes, so liveness has to wait on the startup probe.
-          startupProbe:
-            httpGet:
-              path: /api/server/ping
-              port: http
-            failureThreshold: 60
-            periodSeconds: 10
-            timeoutSeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /api/server/ping
-              port: http
-            periodSeconds: 10
-            timeoutSeconds: 5
-          livenessProbe:
-            httpGet:
-              path: /api/server/ping
-              port: http
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            timeoutSeconds: 5
-          # Only the request is scheduled against, and the node is already
-          # oversubscribed (5.58 of 6 cores requested) while actually running
-          # at about 1.5. So the request states what this sits at while idle -
-          # tens of millicores - and the limit leaves room for the burst that
-          # matters: thumbnails, transcodes and metadata extraction.
-          #
-          # The limit used to be 2Gi, but the server OOMKilled on boot while
-          # chewing through a backlog of unprocessed assets (API +Workers in
-          # one container spike well past idle).
-          resources:
-            requests:
-              cpu: "100m"
-              memory: "512Mi"
-            limits:
-              cpu: "1500m"
-              memory: "4Gi"
-      volumes:
-        - name: immich-data
-          # The library lives on the node's own disk, not in a PVC. A PVC here
-          # meant declaring a size up front for data that does not exist yet,
-          # on a provisioner that cannot grow it, and the only copy of the
-          # photos was one `kubectl delete namespace` away.
-          #
-          # Directory, not DirectoryOrCreate, on purpose: if sdc9 is not
-          # mounted, this must fail loudly instead of quietly writing the
-          # library onto the root filesystem.
-          hostPath:
-            path: /mnt/immich/library
-            type: Directory

immich/k8s/ingress.yaml

@@ -1,33 +0,0 @@
-apiVersion: traefik.io/v1alpha1
-kind: IngressRoute
-metadata:
-  name: immich-prod
-  namespace: immich
-spec:
-  entryPoints:
-    - websecure
-  routes:
-    - match: Host(`immich.forust.xyz`)
-      kind: Rule
-      services:
-        - name: immich-service
-          port: 2283
-  tls:
-    secretName: immich-prod-tls
----
-apiVersion: traefik.io/v1alpha1
-kind: IngressRoute
-metadata:
-  name: immich-local
-  namespace: immich
-spec:
-  entryPoints:
-    - websecure
-  routes:
-    - match: Host(`immich.workstation.internal`) || Host(`immich.gigaforust.internal`)
-      kind: Rule
-      services:
-        - name: immich-service
-          port: 2283
-  tls:
-    secretName: internal-wildcard-tls