ci: lint workflows and shell scripts, validate k8s against the API server

Adds three lint jobs (actionlint, shellcheck, compose) and a server-side
dry-run of the active manifests. Previously the only k8s check was
kubeconform, which has no schemas for CRDs, so every IngressRoute,
Certificate, PrometheusRule and Middleware was silently skipped.

The server-side pass needs the live API server because that is the only
place the real CRD schemas and the cert-manager / Traefik admission
webhooks exist. It is scoped to services carrying a k8s/active marker,
since dry-run needs the target namespace to exist. userbot/ is excluded
from shellcheck: it is a git subtree, and linting upstream's scripts would
let a routine subtree pull turn the deploy gate red on code we do not own.

kubeconform, shellcheck and actionlint are now installed from pinned
versions in tool-versions.env rather than picked up from the runner's
PATH. The Compose helper is shared with the deploy workflow so both
check the same file set the same way.

.gitea/workflows/ci.yaml

@@ -15,8 +15,90 @@ env:
   REGISTRY: gcr.forust.xyz
 
 jobs:
+  lint-compose:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      # Structure check for every committed Compose file, active or not.
+      # Interpolation, env-file and bind-mount resolution are all switched off,
+      # because inactive stacks have no .env here and would only fail on their
+      # ${VAR:?} guards. Active stacks get the full check with interpolation in
+      # the deploy workflow, where the real .env files live.
+      - name: Validate Compose files
+        shell: bash
+        run: |
+          set -euo pipefail
+          source .gitea/workflows/compose-lint.sh
+
+          mapfile -t safe_flags < <(compose_safe_flags)
+          echo "docker compose config ${safe_flags[*]-}"
+
+          mapfile -t files < <(compose_files)
+          if [ "${#files[@]}" -eq 0 ]; then
+            echo "No Compose files found."
+            exit 0
+          fi
+
+          failed=0
+          for f in "${files[@]}"; do
+            if ! out="$(validate_compose_file "$f" ${safe_flags[@]+"${safe_flags[@]}"} 2>&1)"; then
+              failed=1
+              echo "::error file=${f}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+
+          if [ "$failed" -ne 0 ]; then
+            echo "Compose validation failed."
+            exit 1
+          fi
+          echo "checked ${#files[@]} Compose file(s)"
+
+  lint-actionlint:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Lint Gitea Actions workflows with actionlint
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh actionlint)"
+          export PATH="$tools_dir:$PATH"
+          actionlint -config-file .gitea/actionlint.yaml -color .gitea/workflows/*.yaml
+
+  lint-shellcheck:
+    runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Lint shell scripts with ShellCheck
+        shell: bash
+        run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh shellcheck)"
+          export PATH="$tools_dir:$PATH"
+          # userbot/ is a git subtree synced from forust/userbot, so its shell
+          # scripts are upstream's to maintain, not ours. Linting them would let a
+          # routine subtree pull turn the deploy gate red on code we do not own.
+          mapfile -t scripts < <(
+            git ls-files '*.sh' ':(glob)**/*.bash' ':!userbot/**'
+          )
+          if [ "${#scripts[@]}" -eq 0 ]; then
+            echo "No shell scripts found."
+            exit 0
+          fi
+          shellcheck --external-sources --source-path=SCRIPTDIR --severity=style "${scripts[@]}"
+
   lint-prettier:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -39,6 +121,7 @@ jobs:
 
   lint-ruff:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -50,6 +133,7 @@ jobs:
 
   lint-yaml:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -72,6 +156,7 @@ jobs:
 
   lint-dockerfiles:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 10
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -92,13 +177,18 @@ jobs:
 
   validate:
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Validate Kubernetes manifests
+      - name: Validate Kubernetes manifests against JSON schemas
         shell: bash
         run: |
+          set -euo pipefail
+          tools_dir="$(bash .gitea/workflows/install-ci-tools.sh kubeconform)"
+          export PATH="$tools_dir:$PATH"
+
           mapfile -t manifests < <(
             git ls-files ':(glob)**/k8s/**/*.yaml' ':(glob)**/k8s/**/*.yml' \
               | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
@@ -115,10 +205,80 @@ jobs:
             -summary \
             "${manifests[@]}"
 
+      # kubeconform has no schemas for CRDs, so every IngressRoute, Certificate,
+      # PrometheusRule, Middleware, ServersTransport and ServiceMonitor is silently
+      # skipped above. The live API server knows the real CRD schemas (and runs the
+      # cert-manager / Traefik admission webhooks), so validate there too.
+      #
+      # Only services marked with a k8s/active marker are checked: server-side
+      # dry-run needs the target namespace to exist, and inactive services are not
+      # deployed. Services being enabled for the first time are still covered by
+      # the JSON-schema pass above.
+      - name: Validate active manifests against the live API server
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if ! kubectl get --raw='/readyz' --request-timeout=10s >/dev/null 2>&1; then
+            echo "::warning::Cluster unreachable — skipped server-side validation of CRDs (IngressRoute, Certificate, PrometheusRule). Review manifest changes manually."
+            exit 0
+          fi
+
+          mapfile -t k8s_dirs < <(
+            git ls-files '*.yaml' '*.yml' \
+              | grep -E '(^|/)k8s/' \
+              | sed -E 's#((^|.*/)k8s)/.*#\1#' \
+              | sort -u
+          )
+
+          manifests=()
+          kustomize_apps=()
+          for dir in "${k8s_dirs[@]}"; do
+            if [ ! -f "${dir}/active" ]; then
+              echo "skip (no k8s/active): ${dir}"
+              continue
+            fi
+            if [ -f "${dir}/overlays/prod/kustomization.yaml" ]; then
+              kustomize_apps+=("${dir}/overlays/prod")
+            elif [ -f "${dir}/base/kustomization.yaml" ]; then
+              kustomize_apps+=("${dir}/base")
+            else
+              while IFS= read -r f; do
+                [ -n "$f" ] && manifests+=("$f")
+              done < <(
+                git ls-files "${dir}/*.yaml" "${dir}/*.yml" \
+                  | grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$'
+              )
+            fi
+          done
+
+          echo "server-side dry-run: ${#manifests[@]} manifests, ${#kustomize_apps[@]} kustomize apps"
+          failed=0
+          for m in ${manifests[@]+"${manifests[@]}"}; do
+            if ! out="$(kubectl apply --dry-run=server -f "$m" 2>&1)"; then
+              failed=1
+              echo "::error file=${m}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+          for k in ${kustomize_apps[@]+"${kustomize_apps[@]}"}; do
+            if ! out="$(kubectl apply -k "$k" --dry-run=server 2>&1)"; then
+              failed=1
+              echo "::error file=${k}::$(printf '%s' "$out" | head -1)"
+            fi
+          done
+
+          if [ "$failed" -ne 0 ]; then
+            echo "Server-side validation failed. The API server (or an admission webhook) rejected these manifests."
+            exit 1
+          fi
+          echo "server-side dry-run: all active manifests accepted by the API server"
+
   build:
-    needs: [lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
+    needs:
+      [lint-actionlint, lint-shellcheck, lint-compose, lint-prettier, lint-ruff, lint-yaml, lint-dockerfiles, validate]
     if: github.event_name != 'pull_request' && (github.ref_name == 'main' || github.ref_name == 'dev')
     runs-on: [self-hosted, linux, arch, homelab]
+    timeout-minutes: 60
     outputs:
       services: ${{ steps.services.outputs.services }}
     steps:
@@ -280,8 +440,8 @@ jobs:
                 done
                 ;;
               homepages)
-                for service in forust xdfnx; do
-                  case "$service" in
+                for variant in forust xdfnx; do
+                  case "$variant" in
                     forust)
                       image="${REGISTRY}/forust/forust-homepage"
                       ;;
@@ -305,15 +465,15 @@ jobs:
                   docker build \
                     --cache-from "type=registry,ref=${image}:buildcache" \
                     --cache-to "type=registry,ref=${image}:buildcache,mode=max" \
-                    "${build_args[@]}" -f "homepages/Dockerfile.${service}" homepages
+                    "${build_args[@]}" -f "homepages/Dockerfile.${variant}" homepages
                   for tag in "${tags[@]}"; do
                     docker push "${image}:${tag}"
                   done
                 done
                 ;;
               edu_master)
-                for service in session-keeper webinar-checker; do
-                  case "$service" in
+                for variant in session-keeper webinar-checker; do
+                  case "$variant" in
                     session-keeper)
                       context="edu_master/phpsessid-bot"
                       image="${REGISTRY}/forust/session-keeper"

.gitea/workflows/compose-lint.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Shared helpers for validating Compose files. Sourced both by steps in
+# .gitea/workflows/ci.yaml and by deploy-lib.sh on the workstation.
+#
+# Two levels of checking, matching how the repo is structured:
+#
+#   general  every committed Compose file, active or not. Pure structure check:
+#            no ${VAR} interpolation, no .env lookup, no bind-mount path
+#            resolution. Disabled stacks deliberately have no .env in the repo
+#            and no values on the CI runner, so a full `config` run would fail on
+#            their `${VAR:?}` guards for reasons that have nothing to do with the
+#            change under review.
+#
+#   full     active stacks only, with interpolation and env-file resolution, so
+#            required variables and referenced files are actually resolved. Needs
+#            the gitignored .env files, so this only runs in the deploy workflow
+#            on the workstation.
+#
+# This file is meant to be sourced, not executed.
+
+# All committed Compose files, including the ones deploy never starts.
+compose_files() {
+  git ls-files \
+    '*/compose.yaml' '*/compose.yml' 'compose.yaml' 'compose.yml' \
+    '*/docker-compose.yaml' '*/docker-compose.yml'
+}
+
+# Prints the flags that turn `docker compose config` into the general check.
+# Probed rather than hardcoded so an older Compose without --no-env-resolution
+# still gets the flags it does support.
+compose_safe_flags() {
+  local help flag
+  help="$(docker compose config --help 2>/dev/null || true)"
+  for flag in --no-interpolate --no-env-resolution --no-path-resolution; do
+    if printf '%s' "$help" | grep -q -- "$flag"; then
+      printf '%s\n' "$flag"
+    fi
+  done
+}
+
+# validate_compose_file <file> [extra docker compose config flags...]
+validate_compose_file() {
+  local file="$1"
+  shift
+  docker compose -f "$file" config --quiet "$@"
+}

.gitea/workflows/install-ci-tools.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+# Installs the pinned CI linters into "$TOOLS_DIR/bin" and echoes that directory
+# on stdout, so callers can do:
+#
+#   export PATH="$(bash .gitea/workflows/install-ci-tools.sh kubeconform shellcheck):$PATH"
+#
+# Versions come from tool-versions.env next to this script and are kept fresh by
+# Renovate. Re-running is cheap: an already-installed tool at the pinned version
+# is left alone.
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=tool-versions.env
+. "$here/tool-versions.env"
+
+TOOLS_DIR="${TOOLS_DIR:-${RUNNER_TEMP:-/tmp}/homelab-tools}"
+BIN_DIR="$TOOLS_DIR/bin"
+mkdir -p "$BIN_DIR"
+
+arch="$(uname -m)"
+# Upstream projects disagree on arch spelling: kubeconform and actionlint use
+# Go names (amd64/arm64), shellcheck uses uname names (x86_64/aarch64).
+case "$arch" in
+  x86_64 | amd64)
+    goarch=amd64
+    sharch=x86_64
+    ;;
+  aarch64 | arm64)
+    goarch=arm64
+    sharch=aarch64
+    ;;
+  *)
+    echo "install-ci-tools: unsupported architecture: $arch" >&2
+    exit 1
+    ;;
+esac
+
+fetch() {
+  # fetch <url> <dest>
+  if command -v curl >/dev/null 2>&1; then
+    curl -sSLf --retry 3 -o "$2" "$1"
+  elif command -v wget >/dev/null 2>&1; then
+    wget -q -O "$2" "$1"
+  else
+    echo "install-ci-tools: neither curl nor wget is available" >&2
+    exit 1
+  fi
+}
+
+# installed_version <command>
+# Prints the version of an already-installed tool, or nothing. Each tool spells
+# its version flag differently, hence the case.
+installed_version() {
+  local out
+  case "$1" in
+    kubeconform) out="$("$1" -v 2>/dev/null | head -1 || true)" ;;
+    *) out="$("$1" --version 2>/dev/null | head -1 || true)" ;;
+  esac
+  printf '%s' "$out"
+}
+
+# at_version <command> <expected>
+at_version() {
+  case "$(installed_version "$1")" in
+    *"$2"*) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+install_kubeconform() {
+  if at_version kubeconform "v${KUBECONFORM_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-linux-${goarch}.tar.gz" \
+    "$tmp/kubeconform.tar.gz"
+  tar -xzf "$tmp/kubeconform.tar.gz" -C "$tmp" kubeconform
+  install -m 0755 "$tmp/kubeconform" "$BIN_DIR/kubeconform"
+  rm -rf "$tmp"
+}
+
+install_shellcheck() {
+  if at_version shellcheck "${SHELLCHECK_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.${sharch}.tar.xz" \
+    "$tmp/shellcheck.tar.xz"
+  tar -xJf "$tmp/shellcheck.tar.xz" -C "$tmp" --strip-components=1 "shellcheck-v${SHELLCHECK_VERSION}/shellcheck"
+  install -m 0755 "$tmp/shellcheck" "$BIN_DIR/shellcheck"
+  rm -rf "$tmp"
+}
+
+install_actionlint() {
+  if at_version actionlint "${ACTIONLINT_VERSION}"; then
+    return 0
+  fi
+  local tmp
+  tmp="$(mktemp -d)"
+  fetch "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/actionlint_${ACTIONLINT_VERSION}_linux_${goarch}.tar.gz" \
+    "$tmp/actionlint.tar.gz"
+  tar -xzf "$tmp/actionlint.tar.gz" -C "$tmp" actionlint
+  install -m 0755 "$tmp/actionlint" "$BIN_DIR/actionlint"
+  rm -rf "$tmp"
+}
+
+wanted=("$@")
+if [ "${#wanted[@]}" -eq 0 ]; then
+  wanted=(kubeconform shellcheck actionlint)
+fi
+
+for tool in "${wanted[@]}"; do
+  case "$tool" in
+    kubeconform) install_kubeconform ;;
+    shellcheck) install_shellcheck ;;
+    actionlint) install_actionlint ;;
+    *)
+      echo "install-ci-tools: unknown tool: $tool" >&2
+      exit 1
+      ;;
+  esac
+done
+
+printf '%s\n' "$BIN_DIR"

.gitea/workflows/tool-versions.env

@@ -0,0 +1,9 @@
+# Pinned versions of the CI linters installed by install-ci-tools.sh.
+# Renovate keeps these up to date (see customManagers in renovate/renovate.json).
+#
+# The renovate image version is NOT pinned here: renovate/k8s/cronjob.yaml is the
+# single source of truth and the workflows read the tag from it, so there is
+# nothing to drift.
+ACTIONLINT_VERSION="1.7.7"
+SHELLCHECK_VERSION="0.11.0"
+KUBECONFORM_VERSION="0.8.0"