fix(deploy): correct service selection and recovery validation

.gitea/runner/README.md

@@ -157,3 +157,25 @@ run first. Restore the runner config/unit from `.before-<timestamp>` backups,
 reload systemd and restart the runner. Restore the prior workflows from Git.
 Production data and persistent volumes stay where they were. Do not remove run
 state or Compose recovery files until recovery is confirmed.
+
+### Compose configuration recovery
+
+Successful deploys save the complete resolved Compose configuration in
+`~/.local/state/homelab-deploy/compose-configs/`. These files can contain secrets.
+Keep them private and do not commit or upload them.
+The next deploy uses this configuration for its recovery file, including old
+commands, environment, mounts, ports, and removed services. The recovery command
+uses `--remove-orphans` to remove services added by the failed deploy. It does
+not restore volume data or reverse database migrations.
+
+On the first run after this update, the controller can use the Compose file
+from the previous successful run. If that file is absent, it reads the persistent
+checkout and checks its service configuration hashes against existing containers.
+A mismatch stops preflight. Restore the previous configuration before retrying.
+Update the installed controller with `bash .gitea/runner/setup-workstation.sh`
+from the reviewed checkout before using this change.
+
+New namespaces are checked during preflight. Server validation of their resources
+runs after namespace creation and before application resources are applied.
+Plan mode does not create namespaces. A failed deferred check can leave an empty
+namespace; inspect it before removing it.

.gitea/tests/deploy-validation.sh

@@ -92,3 +92,44 @@ if check_referenced_secrets >"$scratch/secrets.log"; then
   exit 1
 fi
 printf '%s\n' 'Deploy validation regressions passed.'
+
+# New declared namespaces defer only their own resources during preflight.
+render_selected_resources() {
+  cat <<'JSON'
+{"apiVersion":"v1","kind":"List","items":[
+ {"apiVersion":"v1","kind":"Namespace","metadata":{"name":"new"}},
+ {"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"new-config","namespace":"new"}},
+ {"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"existing-config","namespace":"default"}}
+]}
+JSON
+}
+kubectl() {
+  case "$1" in
+    get) printf '%s\n' '{"items":[{"metadata":{"name":"default"}}]}' ;;
+    apply) cat >"$scratch/server-input.json" ;;
+    *) return 1 ;;
+  esac
+}
+validate_server_resources true
+jq -e '.items | length == 2 and all(.metadata.name != "new-config")' "$scratch/server-input.json" >/dev/null
+if validate_server_resources false 2>"$scratch/deferred.log"; then
+  echo 'Post-namespace validation accepted a missing namespace' >&2
+  exit 1
+fi
+kubectl() {
+  case "$1" in
+    get) printf '%s\n' '{"items":[{"metadata":{"name":"default"}},{"metadata":{"name":"new"}}]}' ;;
+    apply) cat >"$scratch/server-input.json" ;;
+    *) return 1 ;;
+  esac
+}
+validate_server_resources false
+jq -e '.items | length == 3' "$scratch/server-input.json" >/dev/null
+render_selected_resources() {
+  printf '%s\n' '{"items":[{"kind":"ConfigMap","metadata":{"name":"bad","namespace":"undeclared"}}]}'
+}
+if validate_server_resources true 2>"$scratch/undeclared.log"; then
+  echo 'Preflight accepted an undeclared missing namespace' >&2
+  exit 1
+fi
+printf '%s\n' 'Namespace validation regressions passed.'

.gitea/workflows/compose-release.py

@@ -42,7 +42,37 @@ def prepare(source_file):
     images_file = directory / 'compose-images.json'
     locks = json.loads(images_file.read_text()) if images_file.exists() else previous.get('compose-images', {})
     release = json.loads((directory / 'release.json').read_text())
-    before = json.loads(json.dumps(config))
+    state = Path(os.environ.get('HOMELAB_STATE', Path.home() / '.local/state/homelab-deploy'))
+    baseline = state / 'compose-configs' / f'{relative.parent.name}.json'
+    if not baseline.exists() and re.fullmatch(r'[0-9]+-[0-9]+', previous.get('run_id', '')):
+        baseline = state / 'runs' / previous['run_id'] / 'compose' / baseline.name
+    bootstrap = not baseline.exists()
+    if not bootstrap:
+        before = json.loads(baseline.read_text())
+    else:
+        # Bootstrap from the persistent configuration, never from the new source.
+        persistent_file = config_repo / relative
+        if persistent_file.exists():
+            before = json.loads(
+                output(
+                    'docker',
+                    'compose',
+                    '--project-directory',
+                    str(project_dir),
+                    '-f',
+                    str(persistent_file),
+                    'config',
+                    '--format',
+                    'json',
+                    cwd=config_repo,
+                )
+            )
+        elif output('docker', 'ps', '-aq', '--filter', f'label=com.docker.compose.project={project}'):
+            raise ValueError(f'{project}: no previous Compose configuration; restore it before deploy')
+        else:
+            before = {'name': project, 'services': {}}
+    if before['name'] != project:
+        raise ValueError('Compose project name changed; manual migration is required')
     for service, settings in config['services'].items():
         reference = settings.get('image')
         nextcloud_aio_master = project == 'nextcloud' and service == 'nextcloud-aio-mastercontainer'
@@ -63,6 +93,12 @@ def prepare(source_file):
             pinned = resolve(reference)
         settings['image'] = pinned
         locks[reference] = pinned
+    for service, settings in before['services'].items():
+        reference = settings['image']
+        image_repo = reference.split('@')[0].rsplit('/', 1)
+        image_repo[-1] = image_repo[-1].split(':')[0]
+        image_repo = '/'.join(image_repo)
+        nextcloud_aio_master = project == 'nextcloud' and service == 'nextcloud-aio-mastercontainer'
         # Capture what is running, not the current value of its mutable tag.
         ids = output(
             'docker',
@@ -74,6 +110,31 @@ def prepare(source_file):
             f'label=com.docker.compose.service={service}',
         ).splitlines()
         actual = set()
+        if bootstrap and ids:
+            expected_hash = output(
+                'docker',
+                'compose',
+                '--project-directory',
+                str(project_dir),
+                '-f',
+                str(persistent_file),
+                'config',
+                '--hash',
+                service,
+                cwd=config_repo,
+            ).split()[-1]
+            for container in ids:
+                running_hash = output(
+                    'docker',
+                    'inspect',
+                    container,
+                    '--format',
+                    '{{ index .Config.Labels "com.docker.compose.config-hash" }}',
+                )
+                if running_hash != expected_hash:
+                    raise ValueError(
+                        f'{project}/{service}: persistent config differs from running config; restore the previous config'
+                    )
         for container in ids:
             image_id = output('docker', 'inspect', container, '--format', '{{.Image}}')
             digests = json.loads(output('docker', 'image', 'inspect', image_id, '--format', '{{json .RepoDigests}}'))
@@ -95,7 +156,7 @@ def prepare(source_file):
     images_file.write_text(json.dumps(locks, indent=2) + '\n')
     print(f'Compose {project}: images pinned; local paths preserved')
     print(
-        f'Recovery: docker compose --project-directory {project_dir} -p {project} -f {directory}/compose-before/{relative.parent.name}.json up -d --pull never'
+        f'Recovery: docker compose --project-directory {project_dir} -p {project} -f {directory}/compose-before/{relative.parent.name}.json up -d --pull never --remove-orphans'
     )
 
 

.gitea/workflows/deploy-controller.py

@@ -165,6 +165,10 @@ def finish_success(directory, plan):
         if previous.exists()
         else {}
     )
+    configs = STATE / 'compose-configs'
+    configs.mkdir(mode=0o700, exist_ok=True)
+    for config in (directory / 'compose').glob('*.json'):
+        atomic_json(configs / config.name, json.loads(config.read_text()))
     atomic_json(STATE / 'last-success.json', plan)
     status = json.loads((directory / 'status.json').read_text())
     status['state'] = 'success'

.gitea/workflows/deploy-lib.sh

@@ -571,6 +571,41 @@ skip_uninstalled_vmagent_crd() {
   return 1
 }
 
+# Render one complete resource list so new namespaces can be identified across
+# files and Kustomize apps. A missing undeclared namespace remains an error.
+render_selected_resources() {
+  local m k
+  {
+    for m in "${K8S_MANIFESTS[@]}"; do
+      if skip_uninstalled_vmagent_crd "$m" >/dev/null; then continue; fi
+      kubectl create --dry-run=client --validate=false -f "$m" -o json || return 1
+    done
+    for k in "${KUSTOMIZE_APPS[@]}"; do
+      kubectl kustomize "$k" | kubectl create --dry-run=client --validate=false -f - -o json || return 1
+    done
+  } | jq -s '{apiVersion: "v1", kind: "List", items: [ .[] | if .kind == "List" then .items[] else . end ]}'
+}
+
+validate_server_resources() {
+  local defer_new="$1" resources existing filtered
+  resources="$(render_selected_resources)" || return 1
+  existing="$(kubectl get namespaces -o json)" || return 1
+  filtered="$(jq --argjson existing "$existing" --argjson defer "$defer_new" '
+    [.items[] | select(.kind == "Namespace") | .metadata.name] as $declared
+    | [$existing.items[].metadata.name] as $present
+    | .items |= map(
+        (.metadata.namespace // "default") as $ns
+        | if .kind == "Namespace" or ($present | index($ns)) != null then .
+          elif ($declared | index($ns)) == null then error("Undeclared missing namespace: " + $ns)
+          elif $defer then empty
+          else error("Namespace still missing after namespace apply: " + $ns)
+          end)
+  ' <<<"$resources")" || return 1
+  if [ "$(jq '.items | length' <<<"$filtered")" -gt 0 ]; then
+    kubectl apply --dry-run=server -f - <<<"$filtered" >/dev/null
+  fi
+}
+
 stage_validate() {
   check_prune_mode || return 1
   cd "$REPO"
@@ -597,15 +632,7 @@ stage_validate() {
     kubectl apply -k "$k" --dry-run=client >/dev/null
   done
   log "Validate k8s manifests (kubectl dry-run=server)"
-  for m in ${K8S_MANIFESTS[@]+"${K8S_MANIFESTS[@]}"}; do
-    if skip_uninstalled_vmagent_crd "$m"; then
-      continue
-    fi
-    kubectl apply --dry-run=server -f "$m" >/dev/null
-  done
-  for k in ${KUSTOMIZE_APPS[@]+"${KUSTOMIZE_APPS[@]}"}; do
-    kubectl apply -k "$k" --dry-run=server >/dev/null
-  done
+  validate_server_resources true
   log "Checking referenced Secrets exist"
   echo "  (deploy never applies *secret*.yaml; create missing ones manually)"
   check_referenced_secrets
@@ -657,6 +684,14 @@ stage_apply_k8s() {
       record_apply kubectl "${m#"$REPO"/}" success
     done
   fi
+  # Kustomize may declare namespaces inside its rendered resources too.
+  local namespace_resources
+  namespace_resources="$(render_selected_resources | jq '.items |= map(select(.kind == "Namespace"))')" || return 1
+  if [ "$(jq '.items | length' <<<"$namespace_resources")" -gt 0 ]; then
+    kubectl apply -f - <<<"$namespace_resources" || return 1
+  fi
+  # Complete the deferred server checks before Helm or application resources change.
+  validate_server_resources false || return 1
   if selected_service k8s prometheus-stack && [ -f "$REPO/prometheus-stack/k8s/active" ]; then
     if [ ! -f "$CONFIG_REPO/prometheus-stack/k8s/grafana-values.yaml" ]; then
       echo "ERROR: prometheus-stack/k8s/grafana-values.yaml (gitignored) missing on workstation, restore it first."

.gitea/workflows/deploy-plan.py

@@ -83,20 +83,20 @@ def make_plan(repo, config_repo, release, previous, mode, live_helm):
         removed = []
     else:
         paths = output('git', '-C', str(repo), 'diff', '--name-only', previous['sha'], release['sha']).splitlines()
-        changed = {path.split('/')[0] for path in paths}
+        changed = {service for service in all_services for path in paths if path.startswith(service + '/')}
         if any(path.startswith('.gitea/') for path in paths):
             changed |= all_services
         changed |= {s for s in all_services if previous.get('local_inputs', {}).get(s) != local_inputs[s]}
         for file in tracked(repo):
-            service = file.split('/')[0]
-            if service not in all_services or not file.endswith(('.yaml', '.yml')):
+            owners = {service for service in all_services if file.startswith(service + '/')}
+            if not owners or not file.endswith(('.yaml', '.yml')):
                 continue
             text = (repo / file).read_text()
             if any(
                 image in text and previous.get('images', {}).get(image) != digest
                 for image, digest in release['images'].items()
             ):
-                changed.add(service)
+                changed |= owners
         removed = sorted(
             set(previous.get('active', {}).get('k8s', []) + previous.get('active', {}).get('compose', []))
             - all_services

paperless/.env.example

@@ -1,16 +0,0 @@
-PAPERLESS_URL=https://papers.forust.xyz
-PAPERLESS_ALLOWED_HOSTS=papers.forust.xyz,papers.workstation.internal
-PAPERLESS_CSRF_TRUSTED_ORIGINS=https://papers.forust.xyz,https://papers.workstation.internal
-PAPERLESS_TIME_ZONE=Europe/Bratislava
-PAPERLESS_REDIS=redis://valkey:6379
-PAPERLESS_DBENGINE=postgresql
-PAPERLESS_DBHOST=homelab-postgres
-PAPERLESS_DBNAME=paperless
-PAPERLESS_DBUSER=paperless
-PAPERLESS_DBPASS=<SET_THE_SAME_VALUE_AS_SHARED_POSTGRES_PAPERLESS_DB_PASSWORD>
-PAPERLESS_OCR_LANGUAGE=rus+eng
-PAPERLESS_OCR_LANGUAGES=rus
-PAPERLESS_TASK_WORKERS=1
-PAPERLESS_ADMIN_USER=admin
-PAPERLESS_SECRET_KEY=<GENERATE_WITH_python3_-c_import_secrets;_print(secrets.token_urlsafe(64))>
-PAPERLESS_ADMIN_PASSWORD=<SET_A_LONG_UNIQUE_PASSWORD>

paperless/README.md

@@ -1,79 +0,0 @@
-# Paperless-ngx
-
-Paperless-ngx runs in the `paperless` namespace. It uses the shared PostgreSQL
-service in the `database` namespace and Valkey for its task queue. The document
-library, exports, and consume folder are stored on the `local-path-retain`
-volume. The PVC size is fixed at 50 GiB because this storage class does not
-support volume expansion.
-
-The local route is `https://papers.workstation.internal`; the public route is
-`https://papers.forust.xyz`. Both use TLS. Paperless keeps its own login and
-password authentication. OCR is configured for Russian and English documents.
-Keep `papers.forust.xyz` in the existing Cloudflare DDNS `DOMAINS` setting so
-the public record follows the workstation address.
-
-## Compose alternative
-
-`compose.yaml` is an alternative to the active Kubernetes deployment. Do not
-run both at the same time: they use the same Paperless database and route
-names, but have separate document volumes.
-
-The Compose variant uses the shared Compose PostgreSQL service on the
-`homelab-database` Docker network. It does not start a PostgreSQL container.
-The shared Compose database must be running and must have the `paperless`
-database and role. Set `PAPERLESS_DBPASS` to the same password as
-`PAPERLESS_DB_PASSWORD` in the shared PostgreSQL configuration.
-
-To prepare and start the Compose variant:
-
-```sh
-cp paperless/.env.example paperless/.env
-cd paperless
-docker compose -f compose.yaml config --quiet
-docker compose -f compose.yaml up -d
-```
-
-Create unique values for `PAPERLESS_SECRET_KEY` and
-`PAPERLESS_ADMIN_PASSWORD` in `.env`. This directory has no Compose `active`
-marker, so the repository deploy workflow does not start this alternative.
-Stop the Kubernetes Paperless deployment before switching to Compose. Back up
-and migrate the media files as well as the database; the Compose named volumes
-are separate from the Kubernetes PVC.
-
-## Prepare the secret
-
-Create `k8s/secrets.yaml` on the workstation from
-`k8s/secrets.yaml.example`. Set a unique random `PAPERLESS_SECRET_KEY`, a long
-`PAPERLESS_ADMIN_PASSWORD`, and `PAPERLESS_DB_PASSWORD`.
-
-Add the same `PAPERLESS_DB_PASSWORD` value to the local
-`postgres/k8s/secrets.yaml` file. Keep both secret files out of Git. The
-database bootstrap Job creates the `paperless` role and database from the
-shared PostgreSQL secret. The job runs in the `database` namespace and needs
-that namespace's existing `postgres-shared-secrets` Secret.
-
-For example, generate a key with:
-
-```sh
-python3 -c 'import secrets; print(secrets.token_urlsafe(64))'
-```
-
-Then apply the secret before enabling the service:
-
-```sh
-kubectl apply -f paperless/k8s/namespace.yaml
-kubectl apply -f postgres/k8s/secrets.yaml
-kubectl apply -f paperless/k8s/secrets.yaml
-```
-
-The normal deploy workflow applies the remaining manifests when
-`paperless/k8s/active` is present. Verify the rollout and ingress after deploy:
-
-```sh
-kubectl -n paperless rollout status deployment/paperless
-kubectl -n paperless get pods,pvc,services
-```
-
-Back up the `paperless-data` PVC and the shared PostgreSQL database. The PVC
-contains the originals, archived PDFs, and export/consume folders. Valkey has
-no persistent volume; queued tasks are recreated after a restart.

paperless/compose.yaml

@@ -1,77 +0,0 @@
-services:
-  paperless:
-    image: ghcr.io/paperless-ngx/paperless-ngx:3.2.1
-    container_name: paperless
-    restart: unless-stopped
-    env_file:
-      - .env
-    depends_on:
-      valkey:
-        condition: service_healthy
-    deploy:
-      resources:
-        limits:
-          cpus: "2.0"
-          memory: 2G
-        reservations:
-          cpus: "0.10"
-          memory: 512M
-    volumes:
-      - paperless-data:/usr/src/paperless/data
-      - paperless-media:/usr/src/paperless/media
-      - paperless-export:/usr/src/paperless/export
-      - paperless-consume:/usr/src/paperless/consume
-    networks:
-      - default
-      - proxy
-      - database
-    labels:
-      - "traefik.enable=true"
-      - "traefik.docker.network=proxy"
-      - "traefik.http.services.paperless-compose.loadbalancer.server.port=8000"
-      - "traefik.http.routers.paperless-compose.rule=Host(`papers.forust.xyz`)"
-      - "traefik.http.routers.paperless-compose.entrypoints=websecure"
-      - "traefik.http.routers.paperless-compose.tls.certresolver=letsencrypt"
-      - "traefik.http.routers.paperless-compose-local.rule=Host(`papers.workstation.internal`)"
-      - "traefik.http.routers.paperless-compose-local.entrypoints=websecure"
-      - "traefik.http.routers.paperless-compose-local.tls=true"
-
-  valkey:
-    image: valkey/valkey:9.0.3-alpine
-    container_name: paperless-valkey
-    restart: unless-stopped
-    command:
-      - valkey-server
-      - --save
-      - ""
-      - --appendonly
-      - "no"
-    healthcheck:
-      test: ["CMD", "valkey-cli", "ping"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    deploy:
-      resources:
-        limits:
-          cpus: "0.25"
-          memory: 256M
-        reservations:
-          cpus: "0.025"
-          memory: 64M
-    networks:
-      - default
-
-volumes:
-  paperless-data:
-  paperless-media:
-  paperless-export:
-  paperless-consume:
-
-networks:
-  default:
-  proxy:
-    external: true
-  database:
-    name: homelab-database
-    external: true

paperless/k8s/active

@@ -1 +0,0 @@
-

paperless/k8s/certificates.yaml

@@ -1,28 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: paperless-prod-tls
-  namespace: paperless
-spec:
-  secretName: paperless-prod-tls
-  dnsNames:
-    - papers.forust.xyz
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: internal-wildcard-tls
-  namespace: paperless
-spec:
-  secretName: internal-wildcard-tls
-  dnsNames:
-    - "*.workstation.internal"
-    - "*.gigaforust.internal"
-    - workstation.internal
-    - gigaforust.internal
-  issuerRef:
-    name: internal-ca
-    kind: ClusterIssuer

paperless/k8s/database-init-job.yaml

@@ -1,58 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: paperless-database-init
-  namespace: database
-spec:
-  backoffLimit: 5
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless-database-init
-    spec:
-      restartPolicy: OnFailure
-      containers:
-        - name: create-database
-          image: postgres:17.11-alpine
-          command:
-            - /bin/sh
-            - -ec
-            - |
-              PGPASSWORD="$POSTGRES_ADMIN_PASSWORD" psql \
-                --host postgres \
-                --username postgres \
-                --dbname postgres \
-                --set ON_ERROR_STOP=1 \
-                --set paperless_password="$PAPERLESS_DB_PASSWORD" <<'SQL'
-              SELECT format(
-                'CREATE ROLE paperless LOGIN PASSWORD %L',
-                :'paperless_password'
-              )
-              WHERE NOT EXISTS (
-                SELECT FROM pg_roles WHERE rolname = 'paperless'
-              )
-              \gexec
-              SELECT format('CREATE DATABASE paperless OWNER paperless')
-              WHERE NOT EXISTS (
-                SELECT FROM pg_database WHERE datname = 'paperless'
-              )
-              \gexec
-              SQL
-          env:
-            - name: POSTGRES_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: postgres-shared-secrets
-                  key: POSTGRES_ADMIN_PASSWORD
-            - name: PAPERLESS_DB_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: postgres-shared-secrets
-                  key: PAPERLESS_DB_PASSWORD
-          resources:
-            requests:
-              cpu: 10m
-              memory: 32Mi
-            limits:
-              cpu: 100m
-              memory: 128Mi

paperless/k8s/ingress.yaml

@@ -1,33 +0,0 @@
-apiVersion: traefik.io/v1alpha1
-kind: IngressRoute
-metadata:
-  name: paperless-prod
-  namespace: paperless
-spec:
-  entryPoints:
-    - websecure
-  routes:
-    - match: Host(`papers.forust.xyz`)
-      kind: Rule
-      services:
-        - name: paperless
-          port: 8000
-  tls:
-    secretName: paperless-prod-tls
----
-apiVersion: traefik.io/v1alpha1
-kind: IngressRoute
-metadata:
-  name: paperless-local
-  namespace: paperless
-spec:
-  entryPoints:
-    - websecure
-  routes:
-    - match: Host(`papers.workstation.internal`)
-      kind: Rule
-      services:
-        - name: paperless
-          port: 8000
-  tls:
-    secretName: internal-wildcard-tls

paperless/k8s/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: paperless

paperless/k8s/network-policy.yaml

@@ -1,39 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: paperless-ingress
-  namespace: paperless
-spec:
-  podSelector:
-    matchLabels:
-      app.kubernetes.io/name: paperless
-  policyTypes:
-    - Ingress
-  ingress:
-    - from:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: traefik
-      ports:
-        - protocol: TCP
-          port: 8000
----
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: paperless-valkey-ingress
-  namespace: paperless
-spec:
-  podSelector:
-    matchLabels:
-      app.kubernetes.io/name: paperless-valkey
-  policyTypes:
-    - Ingress
-  ingress:
-    - from:
-        - podSelector:
-            matchLabels:
-              app.kubernetes.io/name: paperless
-      ports:
-        - protocol: TCP
-          port: 6379

paperless/k8s/paperless.yaml

@@ -1,210 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: paperless
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless
-spec:
-  selector:
-    app.kubernetes.io/name: paperless
-  ports:
-    - name: http
-      port: 8000
-      targetPort: http
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless
-    spec:
-      enableServiceLinks: false
-      containers:
-        - name: paperless
-          image: ghcr.io/paperless-ngx/paperless-ngx:3.2.1
-          ports:
-            - name: http
-              containerPort: 8000
-          env:
-            - name: PAPERLESS_URL
-              value: https://papers.forust.xyz
-            - name: PAPERLESS_ALLOWED_HOSTS
-              value: papers.forust.xyz,papers.workstation.internal
-            - name: PAPERLESS_CSRF_TRUSTED_ORIGINS
-              value: https://papers.forust.xyz,https://papers.workstation.internal
-            - name: PAPERLESS_TIME_ZONE
-              value: Europe/Bratislava
-            - name: PAPERLESS_REDIS
-              value: redis://paperless-valkey:6379
-            - name: PAPERLESS_DBENGINE
-              value: postgresql
-            - name: PAPERLESS_DBHOST
-              value: postgres.database.svc.cluster.local
-            - name: PAPERLESS_DBNAME
-              value: paperless
-            - name: PAPERLESS_DBUSER
-              value: paperless
-            - name: PAPERLESS_DBPASS
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-secrets
-                  key: PAPERLESS_DB_PASSWORD
-            - name: PAPERLESS_OCR_LANGUAGE
-              value: rus+eng
-            - name: PAPERLESS_OCR_LANGUAGES
-              value: rus
-            - name: PAPERLESS_TASK_WORKERS
-              value: "1"
-            - name: PAPERLESS_ADMIN_USER
-              value: admin
-            - name: PAPERLESS_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-secrets
-                  key: PAPERLESS_SECRET_KEY
-            - name: PAPERLESS_ADMIN_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-secrets
-                  key: PAPERLESS_ADMIN_PASSWORD
-          volumeMounts:
-            - name: documents
-              mountPath: /usr/src/paperless/data
-              subPath: data
-            - name: documents
-              mountPath: /usr/src/paperless/media
-              subPath: media
-            - name: documents
-              mountPath: /usr/src/paperless/export
-              subPath: export
-            - name: documents
-              mountPath: /usr/src/paperless/consume
-              subPath: consume
-          startupProbe:
-            httpGet:
-              path: /
-              port: http
-              httpHeaders:
-                - name: Host
-                  value: papers.workstation.internal
-            failureThreshold: 60
-            periodSeconds: 10
-            timeoutSeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /
-              port: http
-              httpHeaders:
-                - name: Host
-                  value: papers.workstation.internal
-            periodSeconds: 10
-            timeoutSeconds: 5
-          livenessProbe:
-            httpGet:
-              path: /
-              port: http
-              httpHeaders:
-                - name: Host
-                  value: papers.workstation.internal
-            periodSeconds: 30
-            timeoutSeconds: 5
-          resources:
-            requests:
-              cpu: 100m
-              memory: 512Mi
-            limits:
-              cpu: "2"
-              memory: 2Gi
-      volumes:
-        - name: documents
-          persistentVolumeClaim:
-            claimName: paperless-data
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-data
-  namespace: paperless
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: local-path-retain
-  resources:
-    requests:
-      storage: 50Gi
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: paperless-valkey
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless-valkey
-spec:
-  selector:
-    app.kubernetes.io/name: paperless-valkey
-  ports:
-    - name: redis
-      port: 6379
-      targetPort: redis
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless-valkey
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless-valkey
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless-valkey
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless-valkey
-    spec:
-      containers:
-        - name: valkey
-          image: valkey/valkey:9.0.3-alpine
-          args:
-            - valkey-server
-            - --save
-            - ""
-            - --appendonly
-            - "no"
-          ports:
-            - name: redis
-              containerPort: 6379
-          readinessProbe:
-            exec:
-              command: ["valkey-cli", "ping"]
-            periodSeconds: 10
-          livenessProbe:
-            exec:
-              command: ["valkey-cli", "ping"]
-            periodSeconds: 30
-          resources:
-            requests:
-              cpu: 25m
-              memory: 64Mi
-            limits:
-              cpu: 250m
-              memory: 256Mi

paperless/k8s/secrets.yaml.example

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: paperless-secrets
-  namespace: paperless
-type: Opaque
-stringData:
-  PAPERLESS_SECRET_KEY: "<GENERATE_WITH_python3_-c_import_secrets;_print(secrets.token_urlsafe(64))>"
-  PAPERLESS_ADMIN_PASSWORD: "<SET_A_LONG_UNIQUE_PASSWORD>"
-  PAPERLESS_DB_PASSWORD: "<SET_THE_SAME_VALUE_AS_database_PAPERLESS_DB_PASSWORD>"

postgres/.env.example

@@ -4,5 +4,4 @@ GITEA_DB_PASSWORD=
 NETBOX_DB_PASSWORD=
 NETRONOME_DB_PASSWORD=
 PENPOT_DB_PASSWORD=
-PAPERLESS_DB_PASSWORD=
 STATUSPAGE_DB_PASSWORD=

postgres/README.md

@@ -1,21 +1,20 @@
 # Shared PostgreSQL
 
 This directory contains the shared PostgreSQL 17 deployment for Authentik,
-Gitea, NetBox, Netronome, Paperless-ngx, and Statuspage. It creates one database
-and one login role per service. Per-service standalone databases were removed
-after the migration (Sep 2026); Penpot stays on its own compose PostgreSQL
-(archived, not part of the shared instance).
+Gitea, NetBox, Netronome, and Statuspage. It creates one database and one login role
+per service. Per-service standalone databases were removed after the
+migration (Sep 2026); Penpot stays on its own compose PostgreSQL (archived,
+not part of the shared instance).
 
 ## Compatibility baseline
 
-| Service       | Current application | Shared PostgreSQL 17                   |
-| ------------- | ------------------- | -------------------------------------- |
-| Authentik     | 2025.10.x           | Supported (Authentik requires 14+)     |
-| Gitea         | 1.27.3              | Supported (Gitea requires 12+)         |
-| NetBox        | 4.7.x               | Supported (NetBox 4.x requires 13+)    |
-| Netronome     | 0.14.0              | Supported (upstream's example uses 17) |
-| Paperless-ngx | 3.2.1               | Supported                              |
-| Statuspage    | custom              | Supported                              |
+| Service    | Current application | Shared PostgreSQL 17                   |
+| ---------- | ------------------- | -------------------------------------- |
+| Authentik  | 2025.10.x           | Supported (Authentik requires 14+)     |
+| Gitea      | 1.27.3              | Supported (Gitea requires 12+)         |
+| NetBox     | 4.7.x               | Supported (NetBox 4.x requires 13+)    |
+| Netronome  | 0.14.0              | Supported (upstream's example uses 17) |
+| Statuspage | custom              | Supported                              |
 
 A major-version change must use a logical dump/restore; changing only the
 image tag while keeping a data directory is not supported.

postgres/initdb/01-create-databases.sh

@@ -6,7 +6,6 @@ set -euo pipefail
 : "${NETBOX_DB_PASSWORD:?NETBOX_DB_PASSWORD is required}"
 : "${NETRONOME_DB_PASSWORD:?NETRONOME_DB_PASSWORD is required}"
 : "${PENPOT_DB_PASSWORD:?PENPOT_DB_PASSWORD is required}"
-: "${PAPERLESS_DB_PASSWORD:?PAPERLESS_DB_PASSWORD is required}"
 : "${STATUSPAGE_DB_PASSWORD:?STATUSPAGE_DB_PASSWORD is required}"
 
 create_role_and_database() {
@@ -28,5 +27,4 @@ create_role_and_database gitea gitea "$GITEA_DB_PASSWORD"
 create_role_and_database netbox netbox "$NETBOX_DB_PASSWORD"
 create_role_and_database netronome netronome "$NETRONOME_DB_PASSWORD"
 create_role_and_database penpot penpot "$PENPOT_DB_PASSWORD"
-create_role_and_database paperless paperless "$PAPERLESS_DB_PASSWORD"
 create_role_and_database statuspage statuspage "$STATUSPAGE_DB_PASSWORD"

postgres/k8s/network-policy.yaml

@@ -26,18 +26,6 @@ spec:
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: penpot
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: paperless
-          podSelector:
-            matchLabels:
-              app.kubernetes.io/name: paperless
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: database
-          podSelector:
-            matchLabels:
-              app.kubernetes.io/name: paperless-database-init
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: statuspage

postgres/k8s/postgres.yaml

@@ -126,7 +126,6 @@ data:
     : "${NETBOX_DB_PASSWORD:?NETBOX_DB_PASSWORD is required}"
     : "${NETRONOME_DB_PASSWORD:?NETRONOME_DB_PASSWORD is required}"
     : "${PENPOT_DB_PASSWORD:?PENPOT_DB_PASSWORD is required}"
-    : "${PAPERLESS_DB_PASSWORD:?PAPERLESS_DB_PASSWORD is required}"
     : "${STATUSPAGE_DB_PASSWORD:?STATUSPAGE_DB_PASSWORD is required}"
 
     create_role_and_database() {
@@ -148,5 +147,4 @@ data:
     create_role_and_database netbox netbox "$NETBOX_DB_PASSWORD"
     create_role_and_database netronome netronome "$NETRONOME_DB_PASSWORD"
     create_role_and_database penpot penpot "$PENPOT_DB_PASSWORD"
-    create_role_and_database paperless paperless "$PAPERLESS_DB_PASSWORD"
     create_role_and_database statuspage statuspage "$STATUSPAGE_DB_PASSWORD"

postgres/k8s/secrets.yaml.example

@@ -11,5 +11,4 @@ stringData:
   NETBOX_DB_PASSWORD: ""
   NETRONOME_DB_PASSWORD: ""
   PENPOT_DB_PASSWORD: ""
-  PAPERLESS_DB_PASSWORD: ""
   STATUSPAGE_DB_PASSWORD: ""

tests/test_cicd.py

@@ -128,6 +128,23 @@ class SelectionTests(unittest.TestCase):
         self.assertEqual(result['selected']['k8s'], ['one'])
         self.assertEqual(result['helm'], [])
 
+    def test_nested_service_change_and_owned_image_are_selected(self):
+        directory = self.repo / 'vpn/xui/k8s'
+        directory.mkdir(parents=True)
+        (directory / 'active').touch()
+        image = next(iter(release()['images']))
+        (directory / 'app.yaml').write_text('image: ' + image + ':main\n')
+        baseline_sha = self.commit()
+        baseline = planner.make_plan(self.repo, self.repo, release(baseline_sha), None, 'full', [])
+        (directory / 'app.yaml').write_text('image: ' + image + ':prod\n')
+        result = planner.make_plan(self.repo, self.repo, release(self.commit()), baseline, 'changed', [])
+        self.assertEqual(result['selected']['k8s'], ['vpn/xui'])
+        baseline = result
+        updated = release(result['sha'])
+        updated['images'][image] = 'sha256:' + 'e' * 64
+        result = planner.make_plan(self.repo, self.repo, updated, baseline, 'changed', [])
+        self.assertEqual(result['selected']['k8s'], ['vpn/xui'])
+
     def test_failed_intermediate_deploy_does_not_lose_changes(self):
         (self.repo / 'one/k8s/app.yaml').write_text('kind: StatefulSet\n')
         self.commit()  # This commit failed deploy: baseline must remain initial.
@@ -162,7 +179,8 @@ class ComposeConfigurationTests(unittest.TestCase):
             source = run / 'source'
             config_repo = root / 'persistent'
             (source / 'headscale').mkdir(parents=True)
-            config_repo.mkdir()
+            (config_repo / 'headscale').mkdir(parents=True)
+            (config_repo / 'headscale/compose.yaml').touch()
             (run / 'release.json').write_text(json.dumps(release()))
             old = 'busybox@sha256:' + 'd' * 64
             new = 'busybox@sha256:' + 'e' * 64
@@ -180,19 +198,41 @@ class ComposeConfigurationTests(unittest.TestCase):
                 'volumes': {'data': {'name': 'headscale_data'}},
             }
 
+            previous_config = json.loads(json.dumps(config))
+            previous_config['services']['app']['command'] = ['old-command']
+            previous_config['services']['app']['environment'] = {'VALUE': 'old'}
+            previous_config['services']['removed'] = {'image': 'busybox:latest'}
+            config['services']['app']['command'] = ['new-command']
+            config['services']['app']['environment'] = {'VALUE': 'new'}
+            config['services']['added'] = {'image': 'busybox:latest'}
+
             def fake_output(*args, **kwargs):
                 if args[:2] == ('docker', 'compose'):
                     self.assertEqual(kwargs['cwd'], config_repo)
                     self.assertIn(str(config_repo / 'headscale'), args)
-                    return json.dumps(config)
+                    if '--hash' in args:
+                        return 'app matching-hash'
+                    return json.dumps(
+                        previous_config if str(config_repo / 'headscale/compose.yaml') in args else config
+                    )
                 if args[:2] == ('docker', 'ps'):
                     return 'container'
                 if args[:2] == ('docker', 'inspect'):
+                    if 'com.docker.compose.config-hash' in args[-1]:
+                        return 'matching-hash'
                     return 'sha256:' + 'f' * 64
                 return json.dumps([old])
 
             with (
-                patch.dict(os.environ, {'CONFIG_REPO': str(config_repo), 'REPO': str(source), 'RUN_DIR': str(run)}),
+                patch.dict(
+                    os.environ,
+                    {
+                        'CONFIG_REPO': str(config_repo),
+                        'REPO': str(source),
+                        'RUN_DIR': str(run),
+                        'HOMELAB_STATE': str(root / 'state'),
+                    },
+                ),
                 patch.object(compose_module, 'output', side_effect=fake_output),
                 patch.object(compose_module, 'resolve', return_value=new),
             ):
@@ -204,6 +244,57 @@ class ComposeConfigurationTests(unittest.TestCase):
             self.assertEqual(pinned['services']['app']['volumes'], config['services']['app']['volumes'])
             self.assertEqual(pinned['services']['app']['image'], new)
             self.assertEqual(before['services']['app']['image'], old)
+            self.assertEqual(before['services']['app']['command'], ['old-command'])
+            self.assertEqual(before['services']['app']['environment'], {'VALUE': 'old'})
+            self.assertIn('removed', before['services'])
+            self.assertNotIn('added', before['services'])
+
+            def mismatched_output(*args, **kwargs):
+                if args[:2] == ('docker', 'inspect') and 'com.docker.compose.config-hash' in args[-1]:
+                    return 'different-hash'
+                return fake_output(*args, **kwargs)
+
+            with (
+                patch.dict(
+                    os.environ,
+                    {
+                        'CONFIG_REPO': str(config_repo),
+                        'REPO': str(source),
+                        'RUN_DIR': str(run),
+                        'HOMELAB_STATE': str(root / 'state'),
+                    },
+                ),
+                patch.object(compose_module, 'output', side_effect=mismatched_output),
+                patch.object(compose_module, 'resolve', return_value=new),
+                self.assertRaisesRegex(ValueError, 'differs from running config'),
+            ):
+                compose_module.prepare(source / 'headscale/compose.yaml')
+            state = root / 'state'
+            with patch.object(controller, 'STATE', state):
+                state.mkdir()
+                (run / 'status.json').write_text('{"state": "running", "stages": {}}')
+                with patch.object(controller, 'retain_completed'):
+                    controller.finish_success(run, {})
+            self.assertEqual(json.loads((state / 'compose-configs/headscale.json').read_text()), pinned)
+            # A stale persistent checkout must not replace the successful baseline.
+            with (
+                patch.dict(
+                    os.environ,
+                    {
+                        'CONFIG_REPO': str(config_repo),
+                        'REPO': str(source),
+                        'RUN_DIR': str(run),
+                        'HOMELAB_STATE': str(state),
+                    },
+                ),
+                patch.object(compose_module, 'output', side_effect=fake_output),
+                patch.object(compose_module, 'resolve', return_value=new),
+            ):
+                compose_module.prepare(source / 'headscale/compose.yaml')
+            before = json.loads((run / 'compose-before/headscale.json').read_text())
+            self.assertEqual(before['services']['app']['command'], ['new-command'])
+            self.assertIn('added', before['services'])
+            self.assertNotIn('removed', before['services'])
             self.assertEqual((run / 'compose/headscale.json').stat().st_mode & 0o777, 0o600)
 
     def test_registry_index_and_single_image_descriptors(self):