fix(edu): deploy reviewed application release with Redis authentication
renovate-ci / validate-renovate (push) Skipped
ci / lint-compose (push) Successful in 14s
ci / lint-actionlint (push) Successful in 6s
ci / lint-shellcheck (push) Successful in 13s
ci / lint-prettier (push) Successful in 19s
ci / lint-ruff (push) Successful in 9s
ci / lint-yaml (push) Successful in 10s
ci / lint-dockerfiles (push) Successful in 6s
ci / validate (push) Successful in 10s
ci / build (push) Successful in 30s
renovate-ci / validate-renovate (push) Skipped
ci / lint-compose (push) Successful in 14s
ci / lint-actionlint (push) Successful in 6s
ci / lint-shellcheck (push) Successful in 13s
ci / lint-prettier (push) Successful in 19s
ci / lint-ruff (push) Successful in 9s
ci / lint-yaml (push) Successful in 10s
ci / lint-dockerfiles (push) Successful in 6s
ci / validate (push) Successful in 10s
ci / build (push) Successful in 30s
This commit is contained in:
1 parent
fcd16f6128
commit
5f9354b9a8
7 files changed
+126
-50
No files matched your search
+31
-12
@@ -50,7 +50,36 @@ spec:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Webinar checker failing consecutively"
|
||||
description: 'edu-master/webinar-checker: {{ $value }} consecutive webinar check failures (timeout / playwright error / page error). Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).'
|
||||
description: 'edu-master/webinar-checker: {{ $value }} consecutive webinar check failures (timeout / http error / page error). Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).'
|
||||
|
||||
- alert: WebinarCheckerNeverStarted
|
||||
expr: |
|
||||
(time() - edu_process_start > 120)
|
||||
and (webinar_check_last_run_timestamp_seconds == 0)
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Webinar checker job has not started"
|
||||
description: "The process exposes metrics but its webinar job has never started."
|
||||
|
||||
- alert: WebinarDeliveryPending
|
||||
expr: edu_delivery_pending > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Webinar notifications await delivery"
|
||||
description: "Telegram delivery has pending recipients. Check delivery failures and retry status."
|
||||
|
||||
- alert: EduRedisUnavailable
|
||||
expr: edu_redis_connected == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "EDU checker cannot reach Redis"
|
||||
description: "Redis health checks are failing; checker commands and delivery may be unavailable."
|
||||
|
||||
# Metrics endpoint not scraped for 10m: pod down, metrics server dead, or ServiceMonitor broken.
|
||||
- alert: WebinarCheckerScrapeDown
|
||||
@@ -74,7 +103,7 @@ spec:
|
||||
summary: "EDU_PHPSESSID missing"
|
||||
description: "edu-master: EDU_PHPSESSID absent from redis for 10m. Webinar/diari/schedule checks are all skipped. Check session-keeper logs and EDU credentials."
|
||||
|
||||
# Hard deps: checker and playwright deployments unavailable.
|
||||
# Hard deps: checker deployment unavailable.
|
||||
- alert: WebinarCheckerDeploymentDown
|
||||
expr: |
|
||||
kube_deployment_status_replicas_unavailable{deployment="webinar-checker", namespace="edu-master"} > 0
|
||||
@@ -84,13 +113,3 @@ spec:
|
||||
annotations:
|
||||
summary: "Webinar checker deployment unavailable"
|
||||
description: "edu-master/webinar-checker deployment has {{ $value }} unavailable replica(s) for 10m."
|
||||
|
||||
- alert: PlaywrightServiceDown
|
||||
expr: |
|
||||
kube_deployment_status_replicas_unavailable{deployment="playwright-service", namespace="edu-master"} > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Playwright service unavailable"
|
||||
description: "edu-master/playwright-service deployment has {{ $value }} unavailable replica(s) for 10m. All webinar/diari/schedule checks fail without it."
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: redis-clients-only
|
||||
namespace: edu-master
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: edu-master-redis
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: edu-master-session-keeper
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app: edu-master-webinar-checker
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 6379
|
||||
@@ -20,6 +20,27 @@ spec:
|
||||
- name: redis
|
||||
image: redis:8.10.2-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: REDIS_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: edu-master-secrets
|
||||
key: REDIS_PASSWORD
|
||||
- name: REDISCLI_AUTH
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: edu-master-secrets
|
||||
key: REDIS_PASSWORD
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- |
|
||||
case "$REDIS_PASSWORD" in *[!0-9a-fA-F]*|'') echo 'REDIS_PASSWORD must be 64 hex characters' >&2; exit 1;; esac
|
||||
[ "${#REDIS_PASSWORD}" -eq 64 ] || { echo 'REDIS_PASSWORD must be 64 hex characters' >&2; exit 1; }
|
||||
umask 077
|
||||
printf 'requirepass "%s"\n' "$REDIS_PASSWORD" > /tmp/redis-auth.conf
|
||||
chown redis:redis /tmp/redis-auth.conf
|
||||
exec docker-entrypoint.sh redis-server /tmp/redis-auth.conf
|
||||
ports:
|
||||
- containerPort: 6379
|
||||
volumeMounts:
|
||||
|
||||
@@ -16,12 +16,20 @@ spec:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
edu.forust.xyz/source-commit: "90829d6c8080b9928f9da23587678e640939e10a"
|
||||
labels:
|
||||
app: edu-master-session-keeper
|
||||
spec:
|
||||
initContainers:
|
||||
- name: wait-redis
|
||||
image: redis:8.10.2-alpine
|
||||
env:
|
||||
- name: REDISCLI_AUTH
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: edu-master-secrets
|
||||
key: REDIS_PASSWORD
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
@@ -35,10 +43,16 @@ spec:
|
||||
echo "redis is ready"
|
||||
containers:
|
||||
- name: session-keeper
|
||||
image: gcr.forust.xyz/forust/session-keeper:prod
|
||||
image: gcr.forust.xyz/forust/session-keeper@sha256:49285e87cc5bc4cf4ffe190813d87927916c2df8a206daac0aeb7d227c636450
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: edu-master-secrets
|
||||
env:
|
||||
- name: REDISCLI_AUTH
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: edu-master-secrets
|
||||
key: REDIS_PASSWORD
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
|
||||
@@ -16,14 +16,22 @@ spec:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
edu.forust.xyz/source-commit: "90829d6c8080b9928f9da23587678e640939e10a"
|
||||
labels:
|
||||
app: edu-master-webinar-checker
|
||||
spec:
|
||||
# Enforces dependency order like compose depends_on:
|
||||
# redis healthy -> session-keeper healthy (EXISTS EDU_PHPSESSID) -> playwright started
|
||||
# redis healthy -> session-keeper healthy (EXISTS EDU_PHPSESSID)
|
||||
initContainers:
|
||||
- name: wait-deps
|
||||
image: redis:8.10.2-alpine
|
||||
env:
|
||||
- name: REDISCLI_AUTH
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: edu-master-secrets
|
||||
key: REDIS_PASSWORD
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
@@ -41,15 +49,9 @@ spec:
|
||||
sleep 2
|
||||
done
|
||||
echo "PHPSESSID ok"
|
||||
until nc -z playwright-service 3000; do
|
||||
i=$((i+1))
|
||||
[ "$i" -ge 300 ] && echo "TIMEOUT: playwright-service not reachable" && exit 1
|
||||
sleep 2
|
||||
done
|
||||
echo "playwright ok"
|
||||
containers:
|
||||
- name: webinar-checker
|
||||
image: gcr.forust.xyz/forust/webinar-checker:prod
|
||||
image: gcr.forust.xyz/forust/webinar-checker@sha256:66c146f7b43cb9f0dc31ba9aa36d217e01df42ddafba5971b79c12ec215b2c01
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 8000
|
||||
@@ -62,6 +64,14 @@ spec:
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 12
|
||||
initialDelaySeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /live
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 15
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 4
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: edu-master-secrets
|
||||
|
||||
Reference in new issue
Block a user