Merge branch 'main' into fix/compose-router-rules

Reviewed-on: https://gitea.forust.xyz/forust/homelab/pulls/86

.gitea/workflows/ci.yaml

@@ -3,7 +3,7 @@ name: ci
 on:
   push:
     branches:
-      - "**"
+      - main
   pull_request:
   workflow_dispatch:
 
@@ -142,6 +142,7 @@ jobs:
           export PATH="$tools_dir:$PATH"
           ruff check .
           ruff format --check .
+          python3 -m unittest discover -s tests -v
 
   lint-yaml:
     runs-on: [self-hosted, linux, arch, homelab]

.gitea/workflows/deploy-lib.sh

@@ -31,6 +31,16 @@ warn() {
   echo "WARNING: $*" >&2
 }
 
+# Prune needs the complete desired set in one invocation. Per-file pruning
+# treats resources from the other files as absent and can delete them.
+check_prune_mode() {
+  if [ "$APPLY_PRUNE" = "true" ]; then
+    echo "ERROR: APPLY_PRUNE=true is unsupported by the per-file deploy loop." >&2
+    echo "Disable it; remove obsolete resources explicitly after review." >&2
+    return 1
+  fi
+}
+
 collect_k8s() {
   git -C "$REPO" ls-files -- "$1" \
     | grep -E '\.ya?ml$' \
@@ -720,6 +730,7 @@ check_referenced_secrets() {
 }
 
 stage_validate() {
+  check_prune_mode || return 1
   cd "$REPO"
   select_manifests
   local m k cf
@@ -753,18 +764,16 @@ stage_validate() {
 }
 
 stage_apply_k8s() {
+  check_prune_mode || return 1
   cd "$REPO"
   select_manifests >/dev/null
-  local ns_files=() other_files=() m k prune_opts=()
+  local ns_files=() other_files=() m k
   for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do
     case "$m" in
       */namespace.y?ml) ns_files+=("$m") ;;
       *) other_files+=("$m") ;;
     esac
   done
-  if [ "$APPLY_PRUNE" = "true" ]; then
-    prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
-  fi
 
   # Record what is about to change, and publish it for the verify job, before
   # the first apply. Both are fatal on failure: see snapshot_dir.
@@ -789,7 +798,7 @@ stage_apply_k8s() {
   if [ "${#other_files[@]}" -gt 0 ]; then
     log "Applying resources (${#other_files[@]} files, our images pinned to digests)"
     for m in "${other_files[@]}"; do
-      if ! render_pinned <"$m" | kubectl apply "${prune_opts[@]}" -f -; then
+      if ! render_pinned <"$m" | kubectl apply -f -; then
         echo "ERROR: apply failed for ${m#"$REPO"/}" >&2
         exit 1
       fi

.gitea/workflows/deploy.yaml

@@ -5,6 +5,7 @@ on:
   # workflow_dispatch so a red lint/validate run can never reach the cluster.
   workflow_run:
     workflows: [ci]
+    branches: [main]
     types: [completed]
   workflow_dispatch:
 

.gitea/workflows/renovate-ci.yaml

@@ -2,9 +2,23 @@ name: renovate-ci
 
 on:
   pull_request:
+    paths:
+      - "renovate/**"
+      - ".gitea/workflows/renovate-ci.yaml"
+      - ".gitea/workflows/sync-renovate-configmap.sh"
+      - ".gitea/workflows/compose-lint.sh"
+      - ".gitea/workflows/install-ci-tools.sh"
+      - ".gitea/workflows/tool-versions.env"
   push:
     branches:
       - main
+    paths:
+      - "renovate/**"
+      - ".gitea/workflows/renovate-ci.yaml"
+      - ".gitea/workflows/sync-renovate-configmap.sh"
+      - ".gitea/workflows/compose-lint.sh"
+      - ".gitea/workflows/install-ci-tools.sh"
+      - ".gitea/workflows/tool-versions.env"
   workflow_dispatch:
 
 permissions:

adguardhome/compose.yaml

@@ -31,7 +31,7 @@ services:
       - "traefik.http.routers.adguard-dev.entrypoints=websecure"
       - "traefik.http.routers.adguard-dev.tls=true"
       # DoH Router
-      - "traefik.http.routers.dns-over-https.rule=(Host(`dns.forust.xyz` || Host(`adguard.forust.xyz`)) && PathPrefix(`/dns-query`))"
+      - "traefik.http.routers.dns-over-https.rule=(Host(`dns.forust.xyz`) || Host(`adguard.forust.xyz`)) && PathPrefix(`/dns-query`)"
       - "traefik.http.routers.dns-over-https.entrypoints=websecure"
       - "traefik.http.routers.dns-over-https.tls.certresolver=letsencrypt"
 

adguardhome/k8s/adguard.yaml

@@ -51,6 +51,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: adguard-deployment
   namespace: adguard
 spec:
@@ -64,8 +66,6 @@ spec:
     metadata:
       labels:
         app: adguard
-      annotations:
-        reloader.stakater.com/auto: "true"
     spec:
       containers:
         - name: adguard

authentik/k8s/authentik.yaml

@@ -27,6 +27,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: authentik-server-deployment
   namespace: authentik
 spec:
@@ -63,6 +65,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: authentik-worker-deployment
   namespace: authentik
 spec:

cfddns/k8s/deployment.yaml

@@ -1,6 +1,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: cfddns
   labels:
     app: cfddns

checkmk/k8s/checkmk.yaml

@@ -17,6 +17,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: checkmk-deployment
   namespace: checkmk
 spec:

cloudflared/k8s/deployment.yaml

@@ -1,6 +1,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: cloudflared
   labels:
     app: cloudflared

converters/k8s/convertx.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: convertx-deployment
   namespace: converters
 spec:

edu_master/k8s/session-keeper.yaml

@@ -1,6 +1,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: session-keeper
   namespace: edu-master
   labels:

edu_master/k8s/webinar-checker.yaml

@@ -1,6 +1,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: webinar-checker
   namespace: edu-master
   labels:

gitea/k8s/gitea.yaml

@@ -17,6 +17,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: gitea-deployment
   namespace: gitea
 spec:

glance/k8s/glance.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: glance-deployment
   namespace: glance
 spec:

homarr/k8s/homarr.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: homarr-deployment
   namespace: homarr
 spec:

immich/k8s/immich.yaml

@@ -14,6 +14,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: immich-deployment
   namespace: immich
   labels:

immich/k8s/machine-learning.yaml

@@ -14,6 +14,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: immich-machine-learning-deployment
   namespace: immich
   labels:

immich/k8s/valkey.yaml

@@ -17,6 +17,8 @@ spec:
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: immich-valkey
   namespace: immich
   labels:

kener/k8s/kener.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: kener-deployment
   namespace: kener
 spec:

metube/k8s/metube.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: metube-deployment
   namespace: metube
 spec:

n8n/k8s/n8n.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: n8n-deployment
   namespace: n8n
 spec:

netbird/entrypoint.sh

@@ -0,0 +1,109 @@
+#!/bin/sh
+set -eu
+
+umask 077
+
+TEMPLATE_PATH=/opt/netbird/config.template.yaml
+RENDERED_PATH=/run/netbird/config.yaml
+RELAY_SECRET_PATH=/run/secrets/relay_auth_secret
+ENCRYPTION_KEY_PATH=/run/secrets/datastore_encryption_key
+
+is_valid_proxy_subnet() {
+  candidate="$1"
+  case "$candidate" in
+    0.0.0.0/0)
+      return 1
+      ;;
+    */*)
+      address="${candidate%%/*}"
+      prefix="${candidate#*/}"
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+
+  case "$prefix" in
+    0|[1-9]|[1-2][0-9]|3[0-2]) ;;
+    *)
+      return 1
+      ;;
+  esac
+
+  old_ifs="$IFS"
+  IFS=.
+  # shellcheck disable=SC2086
+  set -- $address
+  IFS="$old_ifs"
+  [ "$#" -eq 4 ] || return 1
+
+  for octet do
+    case "$octet" in
+      0|[1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5]) ;;
+      *)
+        return 1
+        ;;
+    esac
+  done
+}
+
+read_secret() {
+  secret_path="$1"
+
+  if [ ! -r "$secret_path" ]; then
+    echo "Required secret is not readable: $secret_path" >&2
+    exit 1
+  fi
+
+  secret_value="$(cat "$secret_path")"
+  if [ -z "$secret_value" ]; then
+    echo "Required secret is empty: $secret_path" >&2
+    exit 1
+  fi
+
+  printf '%s' "$secret_value"
+}
+
+if [ -z "${NETBIRD_DOMAIN:-}" ]; then
+  echo "NETBIRD_DOMAIN must be set" >&2
+  exit 1
+fi
+
+case "$NETBIRD_DOMAIN" in
+  *[!A-Za-z0-9.-]*)
+    echo "NETBIRD_DOMAIN contains unsupported characters" >&2
+    exit 1
+    ;;
+esac
+
+if [ -z "${NETBIRD_PROXY_SUBNET:-}" ] || [ "$NETBIRD_PROXY_SUBNET" = "auto" ]; then
+  echo "NETBIRD_PROXY_SUBNET must be an explicit IPv4 CIDR; run netbird/setup.sh first" >&2
+  exit 1
+fi
+if ! is_valid_proxy_subnet "$NETBIRD_PROXY_SUBNET"; then
+  echo "NETBIRD_PROXY_SUBNET must be a non-default IPv4 CIDR, for example 172.20.0.0/16" >&2
+  exit 1
+fi
+
+if [ "$#" -ne 2 ] || [ "$1" != "--config" ] || [ "$2" != "$RENDERED_PATH" ]; then
+  echo "Expected: --config $RENDERED_PATH" >&2
+  exit 1
+fi
+
+relay_secret="$(read_secret "$RELAY_SECRET_PATH")"
+encryption_key="$(read_secret "$ENCRYPTION_KEY_PATH")"
+
+mkdir -p "$(dirname "$RENDERED_PATH")"
+sed \
+  -e "s|__NETBIRD_DOMAIN__|${NETBIRD_DOMAIN}|g" \
+  -e "s|__NETBIRD_AUTH_SECRET__|${relay_secret}|g" \
+  -e "s|__NETBIRD_ENCRYPTION_KEY__|${encryption_key}|g" \
+  -e "s|__NETBIRD_PROXY_SUBNET__|${NETBIRD_PROXY_SUBNET}|g" \
+  "$TEMPLATE_PATH" >"$RENDERED_PATH"
+
+if grep -q '__NETBIRD_' "$RENDERED_PATH"; then
+  echo "Rendered NetBird configuration still contains unresolved placeholders" >&2
+  exit 1
+fi
+
+exec /go/bin/netbird-server "$@"

netbird/k8s/netbird.yaml

@@ -32,6 +32,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: netbird-server-deployment
   namespace: netbird
 spec:
@@ -126,6 +128,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: netbird-dashboard-deployment
   namespace: netbird
 spec:

netbird/setup.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Prepare local Compose configuration without replacing existing credentials.
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")"
+umask 077
+if [ ! -f .env ]; then
+  cp .env.example .env
+fi
+
+if grep -q '^NETBIRD_PROXY_SUBNET=auto$' .env; then
+  subnet="$(docker network inspect proxy --format '{{range .IPAM.Config}}{{println .Subnet}}{{end}}' | awk '/^[0-9]+\./ { print; exit }')"
+  if [ -z "$subnet" ]; then
+    echo "No IPv4 subnet found on the Docker proxy network. Set NETBIRD_PROXY_SUBNET in .env." >&2
+    exit 1
+  fi
+  # The detected value must be safe to substitute into the env file.
+  if [[ ! "$subnet" =~ ^[0-9.]+/[0-9]+$ ]]; then
+    echo "Unexpected Docker network subnet: $subnet" >&2
+    exit 1
+  fi
+  sed -i "s|^NETBIRD_PROXY_SUBNET=auto$|NETBIRD_PROXY_SUBNET=$subnet|" .env
+fi
+
+mkdir -p secrets
+chmod 700 secrets
+for name in relay-auth-secret datastore-encryption-key; do
+  path="secrets/$name"
+  if [ -e "$path" ]; then
+    if [ ! -s "$path" ]; then
+      echo "Existing secret is empty: $path. Restore it before continuing." >&2
+      exit 1
+    fi
+  else
+    openssl rand -base64 32 >"$path"
+  fi
+  chmod 600 "$path"
+done
+printf '%s\n' 'Local files are ready. Review .env, then run docker compose config --quiet.'

netbox/k8s/netbox.yaml

@@ -14,6 +14,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: netbox-deployment
   namespace: netbox
   labels:
@@ -118,6 +120,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: netbox-worker-deployment
   namespace: netbox
   labels:

netbox/k8s/valkey.yaml

@@ -17,6 +17,8 @@ spec:
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: netbox-valkey
   namespace: netbox
   labels:

netronome/k8s/netronome.yaml

@@ -14,6 +14,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: netronome-deployment
   namespace: netronome
   labels:

reloader/k8s/reloader-values.yaml

@@ -1,11 +1,17 @@
 # Pinned chart: stakater/reloader 2.2.17 (app v1.4.22).
 # Deployed by the deploy workflow, namespace reloader.
 # Restarts pods when a ConfigMap or Secret they consume changes. Opt-in per workload
-# via the reloader.stakater.com/auto: "true" pod annotation; watchGlobally because
+# via the reloader.stakater.com/auto: "true" workload annotation; watchGlobally because
 # the workloads that need it are spread across a few dozen namespaces.
 
 reloader:
   watchGlobally: true
+  # Only opted-in workloads are restarted. Keep scheduled jobs on their schedule.
+  autoReloadAll: false
+  ignoreJobs: true
+  ignoreCronJobs: true
+  # Change pod-template annotations rather than injecting STAKATER_* env vars.
+  reloadStrategy: annotations
 
   deployment:
     replicas: 1

searxng/compose.yaml

@@ -12,11 +12,11 @@ services:
       - "traefik.enable=true"
       - "traefik.http.services.searxng.loadbalancer.server.port=8080"
       # Prod Router
-      - "traefik.http.routers.searxng.rule=Host(`s.forust.xyz` || `search.forust.xyz`)"
+      - "traefik.http.routers.searxng.rule=Host(`s.forust.xyz`) || Host(`search.forust.xyz`)"
       - "traefik.http.routers.searxng.entrypoints=websecure"
       - "traefik.http.routers.searxng.tls.certresolver=letsencrypt"
       # Local Router
-      - "traefik.http.routers.searxng-local.rule=Host(`s.workstation.internal` || `searxng.workstation.internal`)"
+      - "traefik.http.routers.searxng-local.rule=Host(`s.workstation.internal`) || Host(`searxng.workstation.internal`)"
       - "traefik.http.routers.searxng-local.entrypoints=websecure"
       - "traefik.http.routers.searxng-local.tls=true"
       # Dev Router

searxng/k8s/searxng.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: searxng-deployment
   namespace: searxng
 spec:

termix/k8s/termix.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: termix-deployment
   namespace: termix
 spec:

tests/test_netbird_runtime.py

@@ -0,0 +1,87 @@
+"""Exercise local setup and config rendering without a Docker daemon."""
+
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class NetbirdRuntimeTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.stack = self.root / 'netbird'
+        self.stack.mkdir()
+        for name in ('setup.sh', '.env.example', 'config.template.yaml'):
+            shutil.copy(ROOT / 'netbird' / name, self.stack / name)
+        binary = self.root / 'bin'
+        binary.mkdir()
+        docker = binary / 'docker'
+        docker.write_text('#!/bin/sh\nprintf "%s\\n" 172.20.0.0/16\n')
+        docker.chmod(0o755)
+        self.env = dict(os.environ, PATH=f'{binary}:{os.environ["PATH"]}')
+
+    def setup(self):
+        return subprocess.run(  # noqa: S603 - executes the repository script copied into this test's temp dir
+            ['/bin/bash', str(self.stack / 'setup.sh')], env=self.env, capture_output=True, check=False
+        )
+
+    def test_setup_preserves_existing_secrets_and_env(self):
+        self.assertEqual(self.setup().returncode, 0)
+        paths = [self.stack / '.env', *sorted((self.stack / 'secrets').iterdir())]
+        before = [p.read_bytes() for p in paths]
+        self.assertIn(b'NETBIRD_PROXY_SUBNET=172.20.0.0/16', before[0])
+        self.assertEqual(self.setup().returncode, 0)
+        self.assertEqual(before, [p.read_bytes() for p in paths])
+        for p in paths[1:]:
+            self.assertEqual(p.stat().st_mode & 0o777, 0o600)
+
+    def test_setup_rejects_empty_existing_secret(self):
+        (self.stack / 'secrets').mkdir()
+        secret = self.stack / 'secrets/datastore-encryption-key'
+        secret.touch()
+        self.assertNotEqual(self.setup().returncode, 0)
+        self.assertEqual(secret.read_bytes(), b'')
+
+    def render(self, subnet):
+        self.assertEqual(self.setup().returncode, 0)
+        rendered = self.root / 'run/config.yaml'
+        script = (ROOT / 'netbird/entrypoint.sh').read_text()
+        replacements = {
+            '/opt/netbird/config.template.yaml': str(self.stack / 'config.template.yaml'),
+            '/run/netbird/config.yaml': str(rendered),
+            '/run/secrets/relay_auth_secret': str(self.stack / 'secrets/relay-auth-secret'),
+            '/run/secrets/datastore_encryption_key': str(self.stack / 'secrets/datastore-encryption-key'),
+            '/go/bin/netbird-server': '/bin/true',
+        }
+        for original, local in replacements.items():
+            script = script.replace(original, local)
+        result = subprocess.run(  # noqa: S603 - repository renderer, with test-local paths
+            ['/bin/sh', '-c', script, 'entrypoint', '--config', str(rendered)],
+            env=dict(self.env, NETBIRD_DOMAIN='nb.example.com', NETBIRD_PROXY_SUBNET=subnet),
+            capture_output=True,
+            check=False,
+        )
+        return result, rendered
+
+    def test_renderer_replaces_placeholders_and_restricts_file_permissions(self):
+        result, rendered = self.render('172.20.0.0/16')
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn('__NETBIRD_', rendered.read_text())
+        self.assertIn('nb.example.com', rendered.read_text())
+        self.assertEqual(rendered.stat().st_mode & 0o777, 0o600)
+
+    def test_renderer_rejects_auto_and_default_route(self):
+        for subnet in ('auto', '0.0.0.0/0', '999.1.1.1/24'):
+            with self.subTest(subnet=subnet):
+                result, _ = self.render(subnet)
+                self.assertNotEqual(result.returncode, 0)
+
+
+if __name__ == '__main__':
+    unittest.main()

vaultwarden/k8s/vaultwarden.yaml

@@ -13,6 +13,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: vaultwarden-deployment
   namespace: vaultwarden
 spec:

vpn/xui/k8s/xui.yaml

@@ -20,6 +20,8 @@ spec:
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+    reloader.stakater.com/auto: "true"
   name: xui-deployment
   namespace: xui
 spec: