chore(deploy): rework k8s pipeline, monitoring and postgres 17

Deploy workflow uses git-tracked manifests, DISABLED flag and kustomize overlays; add webinar-checker metrics with ServiceMonitor and alerts; upgrade shared postgres to 17 with statuspage DB and probes/resources.

.gitea/workflows/ci.yaml

@@ -347,23 +347,3 @@ jobs:
                 ;;
             esac
           done
-
-  deploy-userbot-panel:
-    needs: build
-    if: github.ref_name == 'main' && contains(needs.build.outputs.services, 'userbot')
-    runs-on: [self-hosted, linux, arch, homelab, prod]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-
-      - name: Apply and roll out userbot panel
-        shell: bash
-        run: |
-          kubectl apply -f userbot/k8s/base/panel.yaml
-          kubectl get secret userbot-common-secrets -n default -o json \
-            | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
-            | kubectl apply -f -
-          # Keep legacy deployments (forust/anna) in sync with manifests; they have no replicas field, so apply leaves scaling to the user manager only.
-          kubectl apply -f userbot/k8s/base/userbots.yaml
-          kubectl rollout restart deployment/userbot-panel -n userbot
-          kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s

.gitea/workflows/deploy.yaml

@@ -19,9 +19,6 @@ jobs:
           DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
           DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
           DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-          # Set APPLY_PRUNE=true to enable kubectl apply --prune. Requires every
-          # manifest to carry label app.kubernetes.io/managed-by=homelab-deploy,
-          # otherwise previously applied resources get deleted on the next run.
           APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
         run: |
           set -euo pipefail
@@ -57,59 +54,170 @@ jobs:
           fi
 
           git -C "$repo" fetch origin main
-          git -C "$repo" reset --hard origin/main
 
-          # Runtime selection: a service is k8s-managed when $SERVICE/k8s/active
-          # exists. Otherwise it is compose-managed, and only k8s/routing/*
-          # manifests (external Services / EndpointSlices / ServersTransport /
-          # Ingresses that route to docker backends) are applied.
-          #   migrate:   touch SERVICE/k8s/active   (+ move routing files up)
-          #   rollback:  rm SERVICE/k8s/active
+          echo "== Workstation state =="
+          echo "  local:  $(git -C "$repo" rev-parse --short HEAD)"
+          echo "  remote: $(git -C "$repo" rev-parse --short origin/main)"
+
+          if [ -n "$(git -C "$repo" status --porcelain --untracked-files=no)" ]; then
+            echo "ERROR: workstation has local tracked modifications, refusing reset:"
+            git -C "$repo" status --porcelain --untracked-files=no
+            git -C "$repo" diff --stat
+            exit 1
+          fi
+
+          git -C "$repo" reset --hard origin/main
+          cd "$repo"
+
+          is_disabled() {
+            local target="$1"
+            if [ -f "$target" ]; then
+              target="$(dirname "$target")"
+            fi
+            while true; do
+              if [ -f "$target/DISABLED" ]; then
+                return 0
+              fi
+              if [ "$target" = "$repo" ]; then
+                break
+              fi
+              target="$(dirname "$target")"
+              case "$target" in
+                "$repo"/*) ;;
+                *) break ;;
+              esac
+            done
+            return 1
+          }
+
           collect_k8s() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              ! -path '*/routing/*' ! -path '*/overlays/*' \
-              ! -name 'kustomization.y*ml' ! -name '*.example.y*ml' \
-              ! -name '*values.y*ml' ! -name 'patch-*.y*ml' \
+            git ls-files -- "$1" \
+              | grep -E '\.ya?ml$' \
+              | grep -Ev '/routing/|/overlays/' \
+              | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$' \
+              | grep -Ev '(^|/)[^/]*secret[^/]*\.ya?ml$' \
               | sort
           }
 
           collect_k8s_inactive() {
-            find "$1" -type f \( -name '*.yaml' -o -name '*.yml' \) \
-              \( -name 'namespace.y*ml' -o -path '*/routing/*' \) \
-              ! -path '*/overlays/*' ! -name '*.example.y*ml' \
-              | sort
+            collect_k8s "$1" \
+              | grep -E '(^|/)namespace\.ya?ml$|/routing/'
           }
 
-          mapfile -t compose_stacks < <(
-            find "$repo" -type f \( -name 'compose.yaml' -o -name 'compose.yml' \) | sort
+          kustomize_overlay() {
+            if [ -f "$1/overlays/prod/kustomization.yaml" ]; then
+              echo "$1/overlays/prod"
+            elif [ -f "$1/base/kustomization.yaml" ]; then
+              echo "$1/base"
+            fi
+          }
+
+          mapfile -t k8s_dirs < <(
+            git ls-files '*.yaml' '*.yml' \
+              | grep -E '(^|/)k8s/' \
+              | sed -E 's#((^|.*/)k8s)/.*#\1#' \
+              | sort -u
           )
 
-          mapfile -t k8s_manifests < <(
-            for kd in $(find "$repo" -type d -name k8s ! -path '*/.git/*' | sort); do
-              if [ -f "$kd/active" ]; then
-                collect_k8s "$kd"
+          k8s_manifests=()
+          kustomize_apps=()
+          for kd_rel in "${k8s_dirs[@]}"; do
+            kd="$repo/$kd_rel"
+            if is_disabled "$kd"; then
+              echo "skip (DISABLED): $kd_rel"
+              continue
+            fi
+            if [ -f "$kd/active" ]; then
+              overlay="$(kustomize_overlay "$kd" || true)"
+              if [ -n "${overlay:-}" ]; then
+                echo "kustomize app: ${overlay#$repo/}"
+                kustomize_apps+=("$overlay")
               else
-                collect_k8s_inactive "$kd"
+                while IFS= read -r f; do
+                  [ -n "$f" ] && k8s_manifests+=("$repo/$f")
+                done < <(collect_k8s "$kd_rel" || true)
               fi
-            done
+            else
+              while IFS= read -r f; do
+                [ -n "$f" ] && k8s_manifests+=("$repo/$f")
+              done < <(collect_k8s_inactive "$kd_rel" || true)
+            fi
+          done
+
+          mapfile -t compose_rel < <(
+            git ls-files '*/compose.yaml' '*/compose.yml' compose.yaml compose.yml | sort
           )
 
+          compose_stacks=()
+          for cf_rel in "${compose_rel[@]}"; do
+            cf="$repo/$cf_rel"
+            if is_disabled "$cf"; then
+              echo "skip (DISABLED): $cf_rel"
+              continue
+            fi
+            if [ -f "$(dirname "$cf")/k8s/active" ]; then
+              echo "skip (k8s-managed): $cf_rel"
+              continue
+            fi
+            compose_stacks+=("$cf")
+          done
+
           echo "== Validate compose stacks =="
           for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
             echo "  config: $cf"
             docker compose -f "$cf" config --quiet
           done
 
-          echo "== Validate k8s manifests (kubectl dry-run) =="
+          echo "== Validate k8s manifests (kubectl dry-run=client) =="
           for m in "${k8s_manifests[@]}"; do
             echo "  apply --dry-run=client $m"
             kubectl apply --dry-run=client -f "$m" >/dev/null
           done
+          for k in "${kustomize_apps[@]}"; do
+            echo "  apply -k --dry-run=client $k"
+            kubectl apply -k "$k" --dry-run=client >/dev/null
+          done
+
+          echo "== Validate k8s manifests (kubectl dry-run=server) =="
+          for m in "${k8s_manifests[@]}"; do
+            echo "  apply --dry-run=server $m"
+            kubectl apply --dry-run=server -f "$m" >/dev/null
+          done
+          for k in "${kustomize_apps[@]}"; do
+            echo "  apply -k --dry-run=server $k"
+            kubectl apply -k "$k" --dry-run=server >/dev/null
+          done
+
+          echo "== Checking referenced Secrets exist =="
+          echo "  (deploy never applies *secret*.yaml; create missing ones from the laptop)"
+          ref_secrets=()
+          if [ "${#k8s_manifests[@]}" -gt 0 ]; then
+            while IFS= read -r s; do
+              [ -n "$s" ] && ref_secrets+=("$s")
+            done < <(
+              {
+                grep -h -A1 -E 'secretRef:|secretKeyRef:' "${k8s_manifests[@]}" 2>/dev/null || true
+                grep -h -E 'secretName:' "${k8s_manifests[@]}" 2>/dev/null || true
+              } | grep -E 'name:' | sed -E 's/.*name:[[:space:]]*//' | tr -d '"'"'"' "'"'" | sed -E 's/[[:space:]]*#.*//' | awk 'NF' | sort -u || true
+            )
+          fi
+          missing_secrets=()
+          all_secrets="$(kubectl get secrets -A --no-headers -o custom-columns=:metadata.name 2>/dev/null || true)"
+          for s in "${ref_secrets[@]}"; do
+            if printf '%s\n' "$all_secrets" | grep -qx "$s"; then
+              echo "  ok: $s"
+            else
+              echo "  MISSING: $s"
+              missing_secrets+=("$s")
+            fi
+          done
+          if [ "${#missing_secrets[@]}" -gt 0 ]; then
+            echo "ERROR: ${#missing_secrets[@]} referenced Secret(s) not found in the cluster:"
+            printf '  - %s\n' "${missing_secrets[@]}"
+            echo "Create them manually from the laptop, e.g.:"
+            echo "  kubectl apply -f SERVICE/k8s/secrets.yaml  # see SERVICE/k8s/secrets.yaml.example"
+            exit 1
+          fi
 
           echo "== Applying Kubernetes manifests =="
           ns_files=()
@@ -130,15 +238,19 @@ jobs:
             echo "  namespaces first: ${ns_files[*]}"
             kubectl apply -f "${ns_files[@]}"
           fi
-          if [ -f "$repo/prometheus-stack/k8s/active" ]; then
+          if [ -f "$repo/prometheus-stack/k8s/active" ] && ! is_disabled "$repo/prometheus-stack/k8s"; then
+            if [ ! -f "$repo/prometheus-stack/k8s/grafana-values.yaml" ]; then
+              echo "ERROR: prometheus-stack/k8s/grafana-values.yaml (gitignored) missing on workstation, restore it first."
+              exit 1
+            fi
             echo "== Upgrading kube-prometheus-stack =="
             helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
               --namespace prometheus \
               --version 86.2.3 \
               --values "$repo/prometheus-stack/k8s/grafana-values.yaml" \
-              --wait
+              --wait --timeout 10m
           fi
-          if [ -f "$repo/loki/k8s/active" ]; then
+          if [ -f "$repo/loki/k8s/active" ] && ! is_disabled "$repo/loki/k8s"; then
             echo "== Upgrading loki/alloy =="
             helm repo add grafana https://grafana.github.io/helm-charts >/dev/null 2>&1 || true
             helm repo update grafana >/dev/null 2>&1 || true
@@ -146,12 +258,12 @@ jobs:
               --version 7.3.0 \
               --namespace prometheus \
               --values "$repo/loki/k8s/loki-values.yaml" \
-              --wait
+              --wait --timeout 10m
             helm upgrade --install alloy grafana/alloy \
               --version 1.12.1 \
               --namespace prometheus \
               --values "$repo/loki/k8s/alloy-values.yaml" \
-              --wait
+              --wait --timeout 10m
           fi
 
           if [ "${#other_files[@]}" -gt 0 ]; then
@@ -159,14 +271,30 @@ jobs:
             kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
           fi
 
+          for k in "${kustomize_apps[@]}"; do
+            echo "== Applying kustomize app: ${k#$repo/} =="
+            kubectl apply -k "$k"
+          done
+
+          if [ -f "$repo/userbot/k8s/active" ] && ! is_disabled "$repo/userbot"; then
+            echo "== userbot panel hook =="
+            if kubectl get secret userbot-common-secrets -n userbot >/dev/null 2>&1; then
+              echo "  userbot-common-secrets already present in userbot ns, not touching"
+            elif kubectl get secret userbot-common-secrets -n default >/dev/null 2>&1; then
+              echo "  bootstrapping userbot-common-secrets into userbot ns"
+              kubectl get secret userbot-common-secrets -n default -o json \
+                | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
+                | kubectl apply -f -
+            else
+              echo "  WARNING: userbot-common-secrets missing in both default and userbot ns; create it manually from the laptop"
+            fi
+            kubectl rollout restart deployment/userbot-panel -n userbot
+            kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s
+          fi
+
           echo "== Redeploying docker compose stacks =="
           for cf in "${compose_stacks[@]}"; do
-            dir=$(dirname "$cf")
-            if [ -f "$dir/k8s/active" ]; then
-              echo "  skip (k8s-managed): $dir"
-              continue
-            fi
-            echo "  compose: $dir"
+            echo "  compose: $cf"
             if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
               docker compose -f "$cf" build
               docker compose -f "$cf" push