refactor(deploy): marker-driven selection (k8s/active, root active); enable headscale/nextcloud hybrid, disable dockmon/kener/downtify/n8n

.gitea/workflows/deploy-lib.sh

@@ -0,0 +1,241 @@
+#!/usr/bin/env bash
+# Shared stages for the deploy workflow. Runs on the workstation, invoked as:
+#   REPO=/srv/homelab APPLY_PRUNE=false bash -se <<'EOF'
+#   source "$REPO/.gitea/workflows/deploy-lib.sh"
+#   run_stage "$STAGE"
+# EOF
+set -euo pipefail
+
+: "${REPO:?REPO must be set}"
+APPLY_PRUNE="${APPLY_PRUNE:-false}"
+
+log() {
+  echo "== $* =="
+}
+
+collect_k8s() {
+  git -C "$REPO" ls-files -- "$1" \
+    | grep -E '\.ya?ml$' \
+    | grep -Ev '/overlays/' \
+    | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$' \
+    | grep -Ev '(^|/)[^/]*secret[^/]*\.ya?ml$' \
+    | sort
+}
+
+kustomize_overlay() {
+  if [ -f "$1/overlays/prod/kustomization.yaml" ]; then
+    echo "$1/overlays/prod"
+  elif [ -f "$1/base/kustomization.yaml" ]; then
+    echo "$1/base"
+  fi
+}
+
+select_manifests() {
+  K8S_MANIFESTS=()
+  KUSTOMIZE_APPS=()
+  COMPOSE_STACKS=()
+  local kd_rel kd overlay cf_rel cf f
+  while IFS= read -r kd_rel; do
+    kd="$REPO/$kd_rel"
+    if [ ! -f "$kd/active" ]; then
+      echo "skip (no k8s/active): $kd_rel"
+      continue
+    fi
+    overlay="$(kustomize_overlay "$kd" || true)"
+    if [ -n "${overlay:-}" ]; then
+      echo "kustomize app: ${overlay#$REPO/}"
+      KUSTOMIZE_APPS+=("$overlay")
+    else
+      while IFS= read -r f; do
+        [ -n "$f" ] && K8S_MANIFESTS+=("$REPO/$f")
+      done < <(collect_k8s "$kd_rel" || true)
+    fi
+  done < <(
+    git -C "$REPO" ls-files '*.yaml' '*.yml' \
+      | grep -E '(^|/)k8s/' \
+      | sed -E 's#((^|.*/)k8s)/.*#\1#' \
+      | sort -u
+  )
+  while IFS= read -r cf_rel; do
+    cf="$REPO/$cf_rel"
+    if [ -f "$(dirname "$cf")/active" ]; then
+      echo "compose: $cf_rel"
+      COMPOSE_STACKS+=("$cf")
+    else
+      echo "skip (no root active): $cf_rel"
+    fi
+  done < <(git -C "$REPO" ls-files '*/compose.yaml' '*/compose.yml' compose.yaml compose.yml | sort)
+}
+
+stage_preflight() {
+  if [ ! -d "$REPO/.git" ]; then
+    echo "Repository not found at $REPO"
+    exit 1
+  fi
+  git -C "$REPO" fetch origin main
+  log "Workstation state"
+  echo "  local:  $(git -C "$REPO" rev-parse --short HEAD)"
+  echo "  remote: $(git -C "$REPO" rev-parse --short origin/main)"
+  if [ -n "$(git -C "$REPO" status --porcelain --untracked-files=no)" ]; then
+    echo "ERROR: workstation has local tracked modifications, refusing reset:"
+    git -C "$REPO" status --porcelain --untracked-files=no
+    git -C "$REPO" diff --stat
+    exit 1
+  fi
+  git -C "$REPO" reset --hard origin/main
+}
+
+stage_validate() {
+  cd "$REPO"
+  select_manifests
+  local m k cf
+  log "Validate compose stacks"
+  for cf in ${COMPOSE_STACKS[@]+"${COMPOSE_STACKS[@]}"}; do
+    echo "  config: $cf"
+    docker compose -f "$cf" config --quiet
+  done
+  log "Validate k8s manifests (kubectl dry-run=client)"
+  for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do
+    kubectl apply --dry-run=client -f "$m" >/dev/null
+  done
+  for k in ${KUSTOMIZE_APPS[@]+"${KUSTOMIZE_APPS[@]}"}; do
+    kubectl apply -k "$k" --dry-run=client >/dev/null
+  done
+  log "Validate k8s manifests (kubectl dry-run=server)"
+  for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do
+    kubectl apply --dry-run=server -f "$m" >/dev/null
+  done
+  for k in ${KUSTOMIZE_APPS[@]+"${KUSTOMIZE_APPS[@]}"}; do
+    kubectl apply -k "$k" --dry-run=server >/dev/null
+  done
+  log "Checking referenced Secrets exist"
+  echo "  (deploy never applies *secret*.yaml; create missing ones from the laptop)"
+  local ref_secrets=() missing_secrets=() all_secrets s
+  if [ "${#K8S_MANIFESTS[@]}" -gt 0 ]; then
+    while IFS= read -r s; do
+      [ -n "$s" ] && ref_secrets+=("$s")
+    done < <(
+      {
+        grep -h -A1 -E 'secretRef:|secretKeyRef:' "${K8S_MANIFESTS[@]}" 2>/dev/null || true
+        grep -h -E 'secretName:' "${K8S_MANIFESTS[@]}" 2>/dev/null || true
+      } | grep -E 'name:' | sed -E 's/.*name:[[:space:]]*//' | tr -d '"'"'"' "'"'" | sed -E 's/[[:space:]]*#.*//' | awk 'NF' | sort -u || true
+    )
+  fi
+  all_secrets="$(kubectl get secrets -A --no-headers -o custom-columns=:metadata.name 2>/dev/null || true)"
+  for s in ${ref_secrets[@]+"${ref_secrets[@]}"}; do
+    if printf '%s\n' "$all_secrets" | grep -qx "$s"; then
+      echo "  ok: $s"
+    else
+      echo "  MISSING: $s"
+      missing_secrets+=("$s")
+    fi
+  done
+  if [ "${#missing_secrets[@]}" -gt 0 ]; then
+    echo "ERROR: ${#missing_secrets[@]} referenced Secret(s) not found in the cluster:"
+    printf '  - %s\n' "${missing_secrets[@]}"
+    echo "Create them manually from the laptop, e.g.:"
+    echo "  kubectl apply -f SERVICE/k8s/secrets.yaml  # see SERVICE/k8s/secrets.yaml.example"
+    exit 1
+  fi
+}
+
+stage_apply_k8s() {
+  cd "$REPO"
+  select_manifests >/dev/null
+  local ns_files=() other_files=() m k prune_opts=()
+  for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do
+    case "$m" in
+      */namespace.y?ml) ns_files+=("$m") ;;
+      *) other_files+=("$m") ;;
+    esac
+  done
+  if [ "$APPLY_PRUNE" = "true" ]; then
+    prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
+  fi
+  if [ "${#ns_files[@]}" -gt 0 ]; then
+    log "Applying namespaces (${#ns_files[@]} files)"
+    for m in "${ns_files[@]}"; do
+      kubectl apply -f "$m"
+    done
+  fi
+  if [ -f "$REPO/prometheus-stack/k8s/active" ]; then
+    if [ ! -f "$REPO/prometheus-stack/k8s/grafana-values.yaml" ]; then
+      echo "ERROR: prometheus-stack/k8s/grafana-values.yaml (gitignored) missing on workstation, restore it first."
+      exit 1
+    fi
+    log "Upgrading kube-prometheus-stack"
+    helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
+      --namespace prometheus \
+      --version 86.2.3 \
+      --values "$REPO/prometheus-stack/k8s/grafana-values.yaml" \
+      --wait --timeout 10m
+  fi
+  if [ -f "$REPO/loki/k8s/active" ]; then
+    log "Upgrading loki/alloy"
+    helm repo add grafana https://grafana.github.io/helm-charts >/dev/null 2>&1 || true
+    helm repo update grafana >/dev/null 2>&1 || true
+    helm upgrade --install loki grafana/loki \
+      --version 7.3.0 \
+      --namespace prometheus \
+      --values "$REPO/loki/k8s/loki-values.yaml" \
+      --wait --timeout 10m
+    helm upgrade --install alloy grafana/alloy \
+      --version 1.12.1 \
+      --namespace prometheus \
+      --values "$REPO/loki/k8s/alloy-values.yaml" \
+      --wait --timeout 10m
+  fi
+  if [ "${#other_files[@]}" -gt 0 ]; then
+    log "Applying resources (${#other_files[@]} files)"
+    for m in "${other_files[@]}"; do
+      kubectl apply "${prune_opts[@]}" -f "$m"
+    done
+  fi
+  for k in ${KUSTOMIZE_APPS[@]+"${KUSTOMIZE_APPS[@]}"}; do
+    log "Applying kustomize app: ${k#$REPO/}"
+    kubectl apply -k "$k"
+  done
+  if [ -f "$REPO/userbot/k8s/active" ]; then
+    log "userbot panel hook"
+    if kubectl get secret userbot-common-secrets -n userbot >/dev/null 2>&1; then
+      echo "  userbot-common-secrets already present in userbot ns, not touching"
+    elif kubectl get secret userbot-common-secrets -n default >/dev/null 2>&1; then
+      echo "  bootstrapping userbot-common-secrets into userbot ns"
+      kubectl get secret userbot-common-secrets -n default -o json \
+        | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
+        | kubectl apply -f -
+    else
+      echo "  WARNING: userbot-common-secrets missing in both default and userbot ns; create it manually from the laptop"
+    fi
+    kubectl rollout restart deployment/userbot-panel -n userbot
+    kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s
+  fi
+}
+
+stage_apply_compose() {
+  cd "$REPO"
+  select_manifests >/dev/null
+  local cf
+  log "Redeploying docker compose stacks (${#COMPOSE_STACKS[@]} stacks)"
+  for cf in ${COMPOSE_STACKS[@]+"${COMPOSE_STACKS[@]}"}; do
+    echo "  compose: $cf"
+    if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
+      docker compose -f "$cf" build
+      docker compose -f "$cf" push
+    fi
+    docker compose -f "$cf" up -d --pull always --remove-orphans
+  done
+}
+
+run_stage() {
+  case "${1:?stage required}" in
+    preflight) stage_preflight ;;
+    validate) stage_validate ;;
+    apply-k8s) stage_apply_k8s ;;
+    apply-compose) stage_apply_compose ;;
+    *)
+      echo "ERROR: unknown stage: $1"
+      exit 1
+      ;;
+  esac
+}

.gitea/workflows/deploy.yaml

@@ -10,299 +10,62 @@ concurrency:
   group: deploy-main
   cancel-in-progress: false
 
+env:
+  DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
+  DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
+  DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
+  DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
+  DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+  APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
+
 jobs:
-  redeploy:
+  preflight:
     runs-on: [self-hosted, linux, arch, homelab, prod]
     steps:
-      - name: Redeploy workstation
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Fetch and reset workstation
         shell: bash
-        env:
-          DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
-          DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
-          DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
-          DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
-          DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-          APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
         run: |
           set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh preflight
 
-          : "${DEPLOY_HOST:?missing DEPLOY_HOST}"
+  validate:
-          : "${DEPLOY_USER:?missing DEPLOY_USER}"
+    needs: [preflight]
-          : "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-          deploy_port="${DEPLOY_PORT:-22}"
+      - name: Dry-run manifests and check Secrets
-          deploy_path="${DEPLOY_PATH:-/srv/homelab}"
+        shell: bash
-          deploy_path="$(printf '%s' "$deploy_path" | tr -d '\"' | tr -d '\r' | xargs)"
+        run: |
-
-          ssh_key="$RUNNER_TEMP/deploy_key"
-          mkdir -p "$RUNNER_TEMP"
-          printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
-          chmod 600 "$ssh_key"
-
-          ssh_opts=(
-            -i "$ssh_key"
-            -p "$deploy_port"
-            -o BatchMode=yes
-            -o StrictHostKeyChecking=accept-new
-          )
-
-          ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
-            "DEPLOY_PATH=$deploy_path APPLY_PRUNE=${APPLY_PRUNE:-false} bash -se" <<'EOF'
           set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh validate
 
-          repo="${DEPLOY_PATH:-/srv/homelab}"
+  apply-k8s:
+    needs: [validate]
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-          if [ ! -d "$repo/.git" ]; then
+      - name: Apply Kubernetes manifests
-            echo "Repository not found at $repo"
+        shell: bash
-            exit 1
+        run: |
-          fi
+          set -euo pipefail
+          ./.gitea/workflows/ssh-run.sh apply-k8s
 
-          git -C "$repo" fetch origin main
+  apply-compose:
+    needs: [validate]
+    runs-on: [self-hosted, linux, arch, homelab, prod]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-          echo "== Workstation state =="
+      - name: Redeploy docker compose stacks
-          echo "  local:  $(git -C "$repo" rev-parse --short HEAD)"
+        shell: bash
-          echo "  remote: $(git -C "$repo" rev-parse --short origin/main)"
+        run: |
-
+          set -euo pipefail
-          if [ -n "$(git -C "$repo" status --porcelain --untracked-files=no)" ]; then
+          ./.gitea/workflows/ssh-run.sh apply-compose
-            echo "ERROR: workstation has local tracked modifications, refusing reset:"
-            git -C "$repo" status --porcelain --untracked-files=no
-            git -C "$repo" diff --stat
-            exit 1
-          fi
-
-          git -C "$repo" reset --hard origin/main
-          cd "$repo"
-
-          is_disabled() {
-            local target="$1"
-            if [ -f "$target" ]; then
-              target="$(dirname "$target")"
-            fi
-            while true; do
-              if [ -f "$target/DISABLED" ]; then
-                return 0
-              fi
-              if [ "$target" = "$repo" ]; then
-                break
-              fi
-              target="$(dirname "$target")"
-              case "$target" in
-                "$repo"/*) ;;
-                *) break ;;
-              esac
-            done
-            return 1
-          }
-
-          collect_k8s() {
-            git ls-files -- "$1" \
-              | grep -E '\.ya?ml$' \
-              | grep -Ev '/routing/|/overlays/' \
-              | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$' \
-              | grep -Ev '(^|/)[^/]*secret[^/]*\.ya?ml$' \
-              | sort
-          }
-
-          collect_k8s_inactive() {
-            collect_k8s "$1" \
-              | grep -E '(^|/)namespace\.ya?ml$|/routing/'
-          }
-
-          kustomize_overlay() {
-            if [ -f "$1/overlays/prod/kustomization.yaml" ]; then
-              echo "$1/overlays/prod"
-            elif [ -f "$1/base/kustomization.yaml" ]; then
-              echo "$1/base"
-            fi
-          }
-
-          mapfile -t k8s_dirs < <(
-            git ls-files '*.yaml' '*.yml' \
-              | grep -E '(^|/)k8s/' \
-              | sed -E 's#((^|.*/)k8s)/.*#\1#' \
-              | sort -u
-          )
-
-          k8s_manifests=()
-          kustomize_apps=()
-          for kd_rel in "${k8s_dirs[@]}"; do
-            kd="$repo/$kd_rel"
-            if is_disabled "$kd"; then
-              echo "skip (DISABLED): $kd_rel"
-              continue
-            fi
-            if [ -f "$kd/active" ]; then
-              overlay="$(kustomize_overlay "$kd" || true)"
-              if [ -n "${overlay:-}" ]; then
-                echo "kustomize app: ${overlay#$repo/}"
-                kustomize_apps+=("$overlay")
-              else
-                while IFS= read -r f; do
-                  [ -n "$f" ] && k8s_manifests+=("$repo/$f")
-                done < <(collect_k8s "$kd_rel" || true)
-              fi
-            else
-              while IFS= read -r f; do
-                [ -n "$f" ] && k8s_manifests+=("$repo/$f")
-              done < <(collect_k8s_inactive "$kd_rel" || true)
-            fi
-          done
-
-          mapfile -t compose_rel < <(
-            git ls-files '*/compose.yaml' '*/compose.yml' compose.yaml compose.yml | sort
-          )
-
-          compose_stacks=()
-          for cf_rel in "${compose_rel[@]}"; do
-            cf="$repo/$cf_rel"
-            if is_disabled "$cf"; then
-              echo "skip (DISABLED): $cf_rel"
-              continue
-            fi
-            if [ -f "$(dirname "$cf")/k8s/active" ]; then
-              echo "skip (k8s-managed): $cf_rel"
-              continue
-            fi
-            compose_stacks+=("$cf")
-          done
-
-          echo "== Validate compose stacks =="
-          for cf in "${compose_stacks[@]}"; do
-            echo "  config: $cf"
-            docker compose -f "$cf" config --quiet
-          done
-
-          echo "== Validate k8s manifests (kubectl dry-run=client) =="
-          for m in "${k8s_manifests[@]}"; do
-            echo "  apply --dry-run=client $m"
-            kubectl apply --dry-run=client -f "$m" >/dev/null
-          done
-          for k in "${kustomize_apps[@]}"; do
-            echo "  apply -k --dry-run=client $k"
-            kubectl apply -k "$k" --dry-run=client >/dev/null
-          done
-
-          echo "== Validate k8s manifests (kubectl dry-run=server) =="
-          for m in "${k8s_manifests[@]}"; do
-            echo "  apply --dry-run=server $m"
-            kubectl apply --dry-run=server -f "$m" >/dev/null
-          done
-          for k in "${kustomize_apps[@]}"; do
-            echo "  apply -k --dry-run=server $k"
-            kubectl apply -k "$k" --dry-run=server >/dev/null
-          done
-
-          echo "== Checking referenced Secrets exist =="
-          echo "  (deploy never applies *secret*.yaml; create missing ones from the laptop)"
-          ref_secrets=()
-          if [ "${#k8s_manifests[@]}" -gt 0 ]; then
-            while IFS= read -r s; do
-              [ -n "$s" ] && ref_secrets+=("$s")
-            done < <(
-              {
-                grep -h -A1 -E 'secretRef:|secretKeyRef:' "${k8s_manifests[@]}" 2>/dev/null || true
-                grep -h -E 'secretName:' "${k8s_manifests[@]}" 2>/dev/null || true
-              } | grep -E 'name:' | sed -E 's/.*name:[[:space:]]*//' | tr -d '"'"'"' "'"'" | sed -E 's/[[:space:]]*#.*//' | awk 'NF' | sort -u || true
-            )
-          fi
-          missing_secrets=()
-          all_secrets="$(kubectl get secrets -A --no-headers -o custom-columns=:metadata.name 2>/dev/null || true)"
-          for s in "${ref_secrets[@]}"; do
-            if printf '%s\n' "$all_secrets" | grep -qx "$s"; then
-              echo "  ok: $s"
-            else
-              echo "  MISSING: $s"
-              missing_secrets+=("$s")
-            fi
-          done
-          if [ "${#missing_secrets[@]}" -gt 0 ]; then
-            echo "ERROR: ${#missing_secrets[@]} referenced Secret(s) not found in the cluster:"
-            printf '  - %s\n' "${missing_secrets[@]}"
-            echo "Create them manually from the laptop, e.g.:"
-            echo "  kubectl apply -f SERVICE/k8s/secrets.yaml  # see SERVICE/k8s/secrets.yaml.example"
-            exit 1
-          fi
-
-          echo "== Applying Kubernetes manifests =="
-          ns_files=()
-          other_files=()
-          for m in "${k8s_manifests[@]}"; do
-            case "$m" in
-              */namespace.y?ml) ns_files+=("$m") ;;
-              *) other_files+=("$m") ;;
-            esac
-          done
-
-          prune_opts=()
-          if [ "${APPLY_PRUNE:-false}" = "true" ]; then
-            prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
-          fi
-
-          if [ "${#ns_files[@]}" -gt 0 ]; then
-            echo "  namespaces first: ${ns_files[*]}"
-            kubectl apply -f "${ns_files[@]}"
-          fi
-          if [ -f "$repo/prometheus-stack/k8s/active" ] && ! is_disabled "$repo/prometheus-stack/k8s"; then
-            if [ ! -f "$repo/prometheus-stack/k8s/grafana-values.yaml" ]; then
-              echo "ERROR: prometheus-stack/k8s/grafana-values.yaml (gitignored) missing on workstation, restore it first."
-              exit 1
-            fi
-            echo "== Upgrading kube-prometheus-stack =="
-            helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
-              --namespace prometheus \
-              --version 86.2.3 \
-              --values "$repo/prometheus-stack/k8s/grafana-values.yaml" \
-              --wait --timeout 10m
-          fi
-          if [ -f "$repo/loki/k8s/active" ] && ! is_disabled "$repo/loki/k8s"; then
-            echo "== Upgrading loki/alloy =="
-            helm repo add grafana https://grafana.github.io/helm-charts >/dev/null 2>&1 || true
-            helm repo update grafana >/dev/null 2>&1 || true
-            helm upgrade --install loki grafana/loki \
-              --version 7.3.0 \
-              --namespace prometheus \
-              --values "$repo/loki/k8s/loki-values.yaml" \
-              --wait --timeout 10m
-            helm upgrade --install alloy grafana/alloy \
-              --version 1.12.1 \
-              --namespace prometheus \
-              --values "$repo/loki/k8s/alloy-values.yaml" \
-              --wait --timeout 10m
-          fi
-
-          if [ "${#other_files[@]}" -gt 0 ]; then
-            echo "  resources: ${other_files[*]}"
-            kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
-          fi
-
-          for k in "${kustomize_apps[@]}"; do
-            echo "== Applying kustomize app: ${k#$repo/} =="
-            kubectl apply -k "$k"
-          done
-
-          if [ -f "$repo/userbot/k8s/active" ] && ! is_disabled "$repo/userbot"; then
-            echo "== userbot panel hook =="
-            if kubectl get secret userbot-common-secrets -n userbot >/dev/null 2>&1; then
-              echo "  userbot-common-secrets already present in userbot ns, not touching"
-            elif kubectl get secret userbot-common-secrets -n default >/dev/null 2>&1; then
-              echo "  bootstrapping userbot-common-secrets into userbot ns"
-              kubectl get secret userbot-common-secrets -n default -o json \
-                | jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
-                | kubectl apply -f -
-            else
-              echo "  WARNING: userbot-common-secrets missing in both default and userbot ns; create it manually from the laptop"
-            fi
-            kubectl rollout restart deployment/userbot-panel -n userbot
-            kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s
-          fi
-
-          echo "== Redeploying docker compose stacks =="
-          for cf in "${compose_stacks[@]}"; do
-            echo "  compose: $cf"
-            if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
-              docker compose -f "$cf" build
-              docker compose -f "$cf" push
-            fi
-            docker compose -f "$cf" up -d --pull always --remove-orphans
-          done
-          EOF

.gitea/workflows/ssh-run.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# usage: ssh-run.sh <stage>
+# Runs one deploy-lib.sh stage on the workstation over SSH.
+set -euo pipefail
+
+: "${DEPLOY_HOST:?missing DEPLOY_HOST}"
+: "${DEPLOY_USER:?missing DEPLOY_USER}"
+: "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
+
+deploy_port="${DEPLOY_PORT:-22}"
+deploy_path="${DEPLOY_PATH:-/srv/homelab}"
+deploy_path="$(printf '%s' "$deploy_path" | tr -d '\"' | tr -d '\r' | xargs)"
+
+ssh_key="$RUNNER_TEMP/deploy_key"
+mkdir -p "$RUNNER_TEMP"
+printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
+chmod 600 "$ssh_key"
+
+ssh -i "$ssh_key" -p "$deploy_port" \
+  -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
+  "${DEPLOY_USER}@${DEPLOY_HOST}" \
+  "REPO=$deploy_path APPLY_PRUNE=${APPLY_PRUNE:-false} STAGE=$1 bash -se" <<'EOF'
+source "$REPO/.gitea/workflows/deploy-lib.sh"
+run_stage "$STAGE"
+EOF