diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index c7ed6cc..70bde19 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -387,9 +387,6 @@ jobs: homepages/*) add_service homepages ;; - edu_master/phpsessid-bot/*|edu_master/webinar-checker/*|edu_master/compose.yaml) - add_service edu_master - ;; esac done @@ -496,32 +493,6 @@ jobs: done done ;; - edu_master) - for variant in session-keeper webinar-checker; do - case "$variant" in - session-keeper) - context="edu_master/phpsessid-bot" - image="${REGISTRY}/forust/session-keeper" - ;; - webinar-checker) - context="edu_master/webinar-checker" - image="${REGISTRY}/forust/webinar-checker" - ;; - esac - set_tags - build_args=() - for tag in "${tags[@]}"; do - build_args+=(-t "${image}:${tag}") - done - docker build \ - --cache-from "type=registry,ref=${image}:buildcache" \ - --cache-to "type=registry,ref=${image}:buildcache,mode=max" \ - "${build_args[@]}" "$context" - for tag in "${tags[@]}"; do - docker push "${image}:${tag}" - done - done - ;; esac done @@ -552,6 +523,11 @@ jobs: fi echo "pinning ${#repos[@]} image(s) to $commit_tag" for repo in "${repos[@]}"; do + # EDU images are released by the application repository and pinned + # directly by digest in edu_master manifests. Never retag them here. + case "$repo" in + */session-keeper|*/webinar-checker) continue ;; + esac if docker buildx imagetools inspect "$repo:$commit_tag" >/dev/null 2>&1; then echo " already built by this push: ${repo##*/}" continue diff --git a/edu_master/README.md b/edu_master/README.md new file mode 100644 index 0000000..093d47c --- /dev/null +++ b/edu_master/README.md @@ -0,0 +1,14 @@ +# EDU deployment ownership + +Application source and release builds: `forust/edu-master`. +The homelab pipeline deploys `edu_master/k8s` and preserves explicit image digests. +The application copies in this directory are legacy and are not build inputs. +Do not publish EDU `prod` images from homelab or resolve releases from moving tags. + +For an EDU release, validate both images, select their digests in the keeper and +checker manifests, and run the existing homelab validation/apply/verification +helpers against this service. Keep the existing Secret and Redis PVC. +Coordinate Redis authentication changes with both clients and all init/probes; +keep a pre-rollout Redis backup and both previous compatible image references. +The current HTTP checker does not depend on Playwright; check other consumers +before removing the separate browser service. diff --git a/edu_master/k8s/alerts.yaml b/edu_master/k8s/alerts.yaml index dce9760..82c62dc 100644 --- a/edu_master/k8s/alerts.yaml +++ b/edu_master/k8s/alerts.yaml @@ -50,7 +50,36 @@ spec: severity: critical annotations: summary: "Webinar checker failing consecutively" - description: 'edu-master/webinar-checker: {{ $value }} consecutive webinar check failures (timeout / playwright error / page error). Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).' + description: 'edu-master/webinar-checker: {{ $value }} consecutive webinar check failures (timeout / http error / page error). Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).' + + - alert: WebinarCheckerNeverStarted + expr: | + (time() - edu_process_start > 120) + and (webinar_check_last_run_timestamp_seconds == 0) + for: 2m + labels: + severity: critical + annotations: + summary: "Webinar checker job has not started" + description: "The process exposes metrics but its webinar job has never started." + + - alert: WebinarDeliveryPending + expr: edu_delivery_pending > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Webinar notifications await delivery" + description: "Telegram delivery has pending recipients. Check delivery failures and retry status." + + - alert: EduRedisUnavailable + expr: edu_redis_connected == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "EDU checker cannot reach Redis" + description: "Redis health checks are failing; checker commands and delivery may be unavailable." # Metrics endpoint not scraped for 10m: pod down, metrics server dead, or ServiceMonitor broken. - alert: WebinarCheckerScrapeDown @@ -74,7 +103,7 @@ spec: summary: "EDU_PHPSESSID missing" description: "edu-master: EDU_PHPSESSID absent from redis for 10m. Webinar/diari/schedule checks are all skipped. Check session-keeper logs and EDU credentials." - # Hard deps: checker and playwright deployments unavailable. + # Hard deps: checker deployment unavailable. - alert: WebinarCheckerDeploymentDown expr: | kube_deployment_status_replicas_unavailable{deployment="webinar-checker", namespace="edu-master"} > 0 @@ -84,13 +113,3 @@ spec: annotations: summary: "Webinar checker deployment unavailable" description: "edu-master/webinar-checker deployment has {{ $value }} unavailable replica(s) for 10m." - - - alert: PlaywrightServiceDown - expr: | - kube_deployment_status_replicas_unavailable{deployment="playwright-service", namespace="edu-master"} > 0 - for: 10m - labels: - severity: critical - annotations: - summary: "Playwright service unavailable" - description: "edu-master/playwright-service deployment has {{ $value }} unavailable replica(s) for 10m. All webinar/diari/schedule checks fail without it." diff --git a/edu_master/k8s/redis-networkpolicy.yaml b/edu_master/k8s/redis-networkpolicy.yaml new file mode 100644 index 0000000..e7e4ec1 --- /dev/null +++ b/edu_master/k8s/redis-networkpolicy.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: redis-clients-only + namespace: edu-master +spec: + podSelector: + matchLabels: + app: edu-master-redis + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + app: edu-master-session-keeper + - podSelector: + matchLabels: + app: edu-master-webinar-checker + ports: + - protocol: TCP + port: 6379 diff --git a/edu_master/k8s/redis.yaml b/edu_master/k8s/redis.yaml index e0fca76..e4f139c 100644 --- a/edu_master/k8s/redis.yaml +++ b/edu_master/k8s/redis.yaml @@ -20,6 +20,27 @@ spec: - name: redis image: redis:8.10.2-alpine imagePullPolicy: IfNotPresent + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: edu-master-secrets + key: REDIS_PASSWORD + - name: REDISCLI_AUTH + valueFrom: + secretKeyRef: + name: edu-master-secrets + key: REDIS_PASSWORD + command: + - /bin/sh + - -ec + - | + case "$REDIS_PASSWORD" in *[!0-9a-fA-F]*|'') echo 'REDIS_PASSWORD must be 64 hex characters' >&2; exit 1;; esac + [ "${#REDIS_PASSWORD}" -eq 64 ] || { echo 'REDIS_PASSWORD must be 64 hex characters' >&2; exit 1; } + umask 077 + printf 'requirepass "%s"\n' "$REDIS_PASSWORD" > /tmp/redis-auth.conf + chown redis:redis /tmp/redis-auth.conf + exec docker-entrypoint.sh redis-server /tmp/redis-auth.conf ports: - containerPort: 6379 volumeMounts: diff --git a/edu_master/k8s/session-keeper.yaml b/edu_master/k8s/session-keeper.yaml index f3c330d..65bfd97 100644 --- a/edu_master/k8s/session-keeper.yaml +++ b/edu_master/k8s/session-keeper.yaml @@ -16,12 +16,20 @@ spec: type: Recreate template: metadata: + annotations: + edu.forust.xyz/source-commit: "90829d6c8080b9928f9da23587678e640939e10a" labels: app: edu-master-session-keeper spec: initContainers: - name: wait-redis image: redis:8.10.2-alpine + env: + - name: REDISCLI_AUTH + valueFrom: + secretKeyRef: + name: edu-master-secrets + key: REDIS_PASSWORD command: - /bin/sh - -ec @@ -35,10 +43,16 @@ spec: echo "redis is ready" containers: - name: session-keeper - image: gcr.forust.xyz/forust/session-keeper:prod + image: gcr.forust.xyz/forust/session-keeper@sha256:49285e87cc5bc4cf4ffe190813d87927916c2df8a206daac0aeb7d227c636450 envFrom: - secretRef: name: edu-master-secrets + env: + - name: REDISCLI_AUTH + valueFrom: + secretKeyRef: + name: edu-master-secrets + key: REDIS_PASSWORD resources: requests: cpu: 25m diff --git a/edu_master/k8s/webinar-checker.yaml b/edu_master/k8s/webinar-checker.yaml index 3c53074..6280de1 100644 --- a/edu_master/k8s/webinar-checker.yaml +++ b/edu_master/k8s/webinar-checker.yaml @@ -16,14 +16,22 @@ spec: type: Recreate template: metadata: + annotations: + edu.forust.xyz/source-commit: "90829d6c8080b9928f9da23587678e640939e10a" labels: app: edu-master-webinar-checker spec: # Enforces dependency order like compose depends_on: - # redis healthy -> session-keeper healthy (EXISTS EDU_PHPSESSID) -> playwright started + # redis healthy -> session-keeper healthy (EXISTS EDU_PHPSESSID) initContainers: - name: wait-deps image: redis:8.10.2-alpine + env: + - name: REDISCLI_AUTH + valueFrom: + secretKeyRef: + name: edu-master-secrets + key: REDIS_PASSWORD command: - /bin/sh - -ec @@ -41,15 +49,9 @@ spec: sleep 2 done echo "PHPSESSID ok" - until nc -z playwright-service 3000; do - i=$((i+1)) - [ "$i" -ge 300 ] && echo "TIMEOUT: playwright-service not reachable" && exit 1 - sleep 2 - done - echo "playwright ok" containers: - name: webinar-checker - image: gcr.forust.xyz/forust/webinar-checker:prod + image: gcr.forust.xyz/forust/webinar-checker@sha256:66c146f7b43cb9f0dc31ba9aa36d217e01df42ddafba5971b79c12ec215b2c01 ports: - name: metrics containerPort: 8000 @@ -62,6 +64,14 @@ spec: timeoutSeconds: 3 failureThreshold: 12 initialDelaySeconds: 10 + livenessProbe: + httpGet: + path: /live + port: metrics + initialDelaySeconds: 60 + periodSeconds: 15 + timeoutSeconds: 3 + failureThreshold: 4 envFrom: - secretRef: name: edu-master-secrets