diff --git a/edu_master/k8s/alerts.yaml b/edu_master/k8s/alerts.yaml index 831d138..dce9760 100644 --- a/edu_master/k8s/alerts.yaml +++ b/edu_master/k8s/alerts.yaml @@ -11,9 +11,14 @@ spec: rules: # No successful webinar check for 5m (~2-3 missed 2-min checks). # Catches: playwright hangs/timeouts, version skew, site changes, hung job. + # The last_success > 0 guard is mandatory: checker.py initialises + # last_success to 0, so without it `time() - 0` equals the current epoch + # and humanizeDuration renders ~20722d on every pod restart. Keep the + # duration expression on the left so $value stays the real gap. - alert: WebinarCheckerNoSuccessfulCheck expr: | - (time() - webinar_check_last_success_timestamp_seconds > 300) + ((time() - webinar_check_last_success_timestamp_seconds) > 300) + and (webinar_check_last_success_timestamp_seconds > 0) and (webinar_check_last_run_timestamp_seconds > 0) for: 2m labels: @@ -22,6 +27,20 @@ spec: summary: "Webinar checker has no successful check for 5m" description: "edu-master/webinar-checker: last successful webinar check was {{ $value | humanizeDuration }} ago. Checks are failing or hanging (see consecutive failures alert). Notifications about new webinars are NOT being sent." + # Checks are running but none has ever succeeded since pod start. + # Split out from the rule above so a zeroed gauge never feeds + # humanizeDuration. + - alert: WebinarCheckerNeverSucceeded + expr: | + (webinar_check_last_success_timestamp_seconds == 0) + and (webinar_check_last_run_timestamp_seconds > 0) + for: 10m + labels: + severity: critical + annotations: + summary: "Webinar checker has never completed a successful check" + description: 'edu-master/webinar-checker: checks have been running for 10m but not one has ever succeeded since the pod started, so every check is failing. Check pod logs (Loki: {namespace="edu-master", container="webinar-checker"}).' + # Fast path: 3 consecutive failures (~6+ min at 2-min interval). - alert: WebinarCheckerConsecutiveFailures expr: |