chore(deploy): rework k8s pipeline, monitoring and postgres 17

Deploy workflow uses git-tracked manifests, DISABLED flag and kustomize overlays; add webinar-checker metrics with ServiceMonitor and alerts; upgrade shared postgres to 17 with statuspage DB and probes/resources.
This commit is contained in:
forust committed 2026-09-23 15:47:26 +02:00
1 parent 8b2cf29771
commit 46c7e99b1d
19 files changed
+629 -227

No files matched your search

+1
View File
@@ -0,0 +1 @@
1.56.0
+1 -1
View File
@@ -11,7 +11,7 @@ services:
retries: 5
playwright-service:
image: mcr.microsoft.com/playwright:v1.63.0-jammy
image: mcr.microsoft.com/playwright:v1.56.0-jammy
restart: unless-stopped
command: npx -y playwright@1.56.0 run-server --port 3000 --path /ws
+77
View File
@@ -0,0 +1,77 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: edu-master-webinar
namespace: edu-master
labels:
release: prometheus-stack
spec:
groups:
- name: edu_master.webinar
rules:
# No successful webinar check for 5m (~2-3 missed 2-min checks).
# Catches: playwright hangs/timeouts, version skew, site changes, hung job.
- alert: WebinarCheckerNoSuccessfulCheck
expr: |
(time() - webinar_check_last_success_timestamp_seconds > 300)
and (webinar_check_last_run_timestamp_seconds > 0)
for: 2m
labels:
severity: critical
annotations:
summary: "Webinar checker has no successful check for 5m"
description: "edu-master/webinar-checker: last successful webinar check was {{ $value | humanizeDuration }} ago. Checks are failing or hanging (see consecutive failures alert). Notifications about new webinars are NOT being sent."
# Fast path: 3 consecutive failures (~6+ min at 2-min interval).
- alert: WebinarCheckerConsecutiveFailures
expr: |
webinar_check_consecutive_failures >= 3
for: 5m
labels:
severity: critical
annotations:
summary: "Webinar checker failing consecutively"
description: "edu-master/webinar-checker: {{ $value }} consecutive webinar check failures (timeout / playwright error / page error). Check pod logs (Loki: {namespace=\"edu-master\", container=\"webinar-checker\"})."
# Metrics endpoint not scraped for 10m: pod down, metrics server dead, or ServiceMonitor broken.
- alert: WebinarCheckerScrapeDown
expr: |
absent(webinar_check_last_run_timestamp_seconds) == 1
for: 10m
labels:
severity: critical
annotations:
summary: "Webinar checker metrics missing"
description: "edu-master/webinar-checker: no metrics series for 10m. Pod may be down, metrics server dead, or ServiceMonitor/Service broken. Webinar checks are unobserved."
# EDU session lost: session-keeper down or credentials expired. Without PHPSESSID every check is skipped.
- alert: EduPhpsessidMissing
expr: |
edu_phpsessid_present == 0
for: 10m
labels:
severity: critical
annotations:
summary: "EDU_PHPSESSID missing"
description: "edu-master: EDU_PHPSESSID absent from redis for 10m. Webinar/diari/schedule checks are all skipped. Check session-keeper logs and EDU credentials."
# Hard deps: checker and playwright deployments unavailable.
- alert: WebinarCheckerDeploymentDown
expr: |
kube_deployment_status_replicas_unavailable{deployment="webinar-checker", namespace="edu-master"} > 0
for: 10m
labels:
severity: critical
annotations:
summary: "Webinar checker deployment unavailable"
description: "edu-master/webinar-checker deployment has {{ $value }} unavailable replica(s) for 10m."
- alert: PlaywrightServiceDown
expr: |
kube_deployment_status_replicas_unavailable{deployment="playwright-service", namespace="edu-master"} > 0
for: 10m
labels:
severity: critical
annotations:
summary: "Playwright service unavailable"
description: "edu-master/playwright-service deployment has {{ $value }} unavailable replica(s) for 10m. All webinar/diari/schedule checks fail without it."
+2 -1
View File
@@ -17,7 +17,8 @@ spec:
spec:
containers:
- name: playwright
image: mcr.microsoft.com/playwright:v1.63.0-jammy
# renovate: datasource=docker depName=mcr.microsoft.com/playwright versioning=docker
image: mcr.microsoft.com/playwright:v1.56.0-jammy
imagePullPolicy: IfNotPresent
command:
- npx
+2
View File
@@ -21,6 +21,8 @@ stringData:
WEBINAR_TELEGRAM_TOKEN: ""
WEBINAR_ADMIN_ID: ""
WEBINAR_CHECK_INTERVAL: "60"
# Prometheus metrics endpoint (scraped via ServiceMonitor, alerts in k8s/alerts.yaml)
METRICS_PORT: "8000"
# Database
REDIS_HOST: "redis"
REDIS_PORT: "6379"
+15
View File
@@ -0,0 +1,15 @@
apiVersion: v1
kind: Service
metadata:
name: webinar-checker
namespace: edu-master
labels:
app: edu-master-webinar-checker
spec:
selector:
app: edu-master-webinar-checker
ports:
- name: metrics
port: 8000
targetPort: metrics
protocol: TCP
+16
View File
@@ -0,0 +1,16 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: webinar-checker
namespace: edu-master
labels:
release: prometheus-stack
spec:
selector:
matchLabels:
app: edu-master-webinar-checker
endpoints:
- port: metrics
path: /metrics
interval: 30s
scrapeTimeout: 10s
+4
View File
@@ -47,6 +47,10 @@ spec:
- name: webinar-checker
image: gcr.forust.xyz/forust/webinar-checker:latest
imagePullPolicy: Always
ports:
- name: metrics
containerPort: 8000
protocol: TCP
envFrom:
- secretRef:
name: edu-master-secrets
+5 -2
View File
@@ -2,8 +2,11 @@ FROM python:3.11-slim
WORKDIR /app
# Install dependencies
RUN pip install --no-cache-dir pip==25.0.1 && pip install --no-cache-dir playwright==1.56.0 redis==5.2.1 requests==2.32.3 "python-telegram-bot[job-queue]==21.10"
# renovate: datasource=pypi depName=playwright versioning=pep440
ARG PLAYWRIGHT_VERSION=1.56.0
# Install dependencies - PLAYWRIGHT_VERSION is single-source, renovate updates ARG above and all other places via regexManagers
RUN pip install --no-cache-dir pip==25.0.1 && pip install --no-cache-dir playwright==${PLAYWRIGHT_VERSION} redis==5.2.1 requests==2.32.3 "python-telegram-bot[job-queue]==21.10"
COPY checker.py .
+275 -142
View File
@@ -1,12 +1,15 @@
import asyncio
import contextlib
import json
import logging
import os
import re
import tempfile
import threading
import time
from datetime import datetime, timedelta
from html import escape
from http.server import BaseHTTPRequestHandler, HTTPServer
import redis
from playwright.async_api import async_playwright
@@ -48,6 +51,106 @@ USER_AGENT = _env(
)
WEBINAR_TELEGRAM_TOKEN = _env('WEBINAR_TELEGRAM_TOKEN')
ADMIN_ID = int(_env('WEBINAR_ADMIN_ID', '0'))
METRICS_PORT = int(_env('METRICS_PORT', '8000'))
# --- Prometheus metrics (stdlib only, no extra deps) ---
# Scraped by prometheus-stack via ServiceMonitor (edu_master/k8s/servicemonitor.yaml).
# Critical alerts in edu_master/k8s/alerts.yaml fire to Telegram via Alertmanager.
_METRICS_LOCK = threading.Lock()
_METRICS = {
'last_run': 0.0, # Unix ts of last check start
'last_success': 0.0, # Unix ts of last successful check
'last_duration': 0.0, # Duration of last check in seconds
'success_total': 0,
'failure_total': 0,
'consecutive_failures': 0,
'phpsessid_present': 1, # 1 if EDU_PHPSESSID found in redis, else 0
}
def _metric_check_start():
with _METRICS_LOCK:
_METRICS['last_run'] = time.time()
def _metric_check_ok(duration: float):
now = time.time()
with _METRICS_LOCK:
_METRICS['last_success'] = now
_METRICS['last_duration'] = duration
_METRICS['success_total'] += 1
_METRICS['consecutive_failures'] = 0
_METRICS['phpsessid_present'] = 1
def _metric_check_fail(duration: float, phpsessid_missing: bool = False):
with _METRICS_LOCK:
_METRICS['last_duration'] = duration
_METRICS['failure_total'] += 1
_METRICS['consecutive_failures'] += 1
_METRICS['phpsessid_present'] = 0 if phpsessid_missing else 1
def _metrics_render() -> bytes:
with _METRICS_LOCK:
m = dict(_METRICS)
lines = [
'# HELP webinar_check_last_run_timestamp_seconds Unix timestamp of last webinar check start.',
'# TYPE webinar_check_last_run_timestamp_seconds gauge',
f'webinar_check_last_run_timestamp_seconds {m["last_run"]}',
'# HELP webinar_check_last_success_timestamp_seconds Unix timestamp of last successful webinar check.',
'# TYPE webinar_check_last_success_timestamp_seconds gauge',
f'webinar_check_last_success_timestamp_seconds {m["last_success"]}',
'# HELP webinar_check_last_duration_seconds Duration of last webinar check in seconds.',
'# TYPE webinar_check_last_duration_seconds gauge',
f'webinar_check_last_duration_seconds {m["last_duration"]}',
'# HELP webinar_check_success_total Total successful webinar checks.',
'# TYPE webinar_check_success_total counter',
f'webinar_check_success_total {m["success_total"]}',
'# HELP webinar_check_failure_total Total failed webinar checks (timeout, playwright error, page error).',
'# TYPE webinar_check_failure_total counter',
f'webinar_check_failure_total {m["failure_total"]}',
'# HELP webinar_check_consecutive_failures Consecutive failed webinar checks (reset on success).',
'# TYPE webinar_check_consecutive_failures gauge',
f'webinar_check_consecutive_failures {m["consecutive_failures"]}',
'# HELP edu_phpsessid_present 1 if EDU_PHPSESSID exists in redis, 0 otherwise.',
'# TYPE edu_phpsessid_present gauge',
f'edu_phpsessid_present {m["phpsessid_present"]}',
]
return ('\n'.join(lines) + '\n').encode()
class _MetricsHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/metrics':
body = _metrics_render()
self.send_response(200)
self.send_header('Content-Type', 'text/plain; version=0.0.4')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
elif self.path in ('/healthz', '/health'):
body = b'ok\n'
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
else:
self.send_response(404)
self.end_headers()
def log_message(self, *args):
pass # keep bot logs clean
def start_metrics_server(port: int = METRICS_PORT):
server = HTTPServer(('0.0.0.0', port), _MetricsHandler) # noqa: S104 - k8s ServiceMonitor scrapes pod IP
thread = threading.Thread(target=server.serve_forever, name='metrics-server', daemon=True)
thread.start()
logger.info(f'Metrics server listening on :{port}/metrics')
return server
# Redis Keys
KEY_WHITELIST = 'bot:whitelist'
@@ -597,59 +700,66 @@ async def _collect_event_times(page) -> dict:
async def fetch_diary_data(phpsessid: str) -> dict | None:
logger.info('Fetching diary data via Playwright...')
try:
async with async_playwright() as p:
browser = await p.chromium.connect(PLAYWRIGHT_WS)
try:
context_browser = await browser.new_context(user_agent=USER_AGENT)
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
page = await context_browser.new_page()
async with asyncio.timeout(60):
async with async_playwright() as p:
browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
try:
await page.goto(DIARY_URL, wait_until='domcontentloaded')
await page.wait_for_selector('table.calendar', timeout=10000)
await page.wait_for_timeout(1500)
table_html = await page.evaluate("""
() => {
const t = document.querySelector('table.calendar');
return t ? t.outerHTML : null;
}
""")
if not table_html:
logger.error('table.calendar not found in DOM')
return None
# Debug: save HTML for troubleshooting
with contextlib.suppress(Exception), open('/tmp/diary_debug.html', 'w', encoding='utf-8') as f: # noqa: S108
f.write(table_html)
month_text, days = _parse_calendar_html(table_html)
# Read event times by opening each event's AJAX popup.
times_by_id = await _collect_event_times(page)
if times_by_id:
for day_data in days.values():
for ev in day_data.get('events', []):
eid = ev.get('id')
if eid and eid in times_by_id:
ev['time'] = times_by_id[eid]
logger.info(
f'Diary parsed: month={month_text!r}, days_with_events={sum(1 for d in days.values() if d["events"])}/{len(days)}'
context_browser = await browser.new_context(user_agent=USER_AGENT)
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
page = await context_browser.new_page()
return {'monthFullText': month_text, 'days': days}
try:
await asyncio.wait_for(page.goto(DIARY_URL, wait_until='domcontentloaded'), timeout=30)
await page.wait_for_selector('table.calendar', timeout=10000)
await page.wait_for_timeout(1500)
except Exception as e:
logger.error(f'Error parsing diary: {e}')
return None
table_html = await page.evaluate("""
() => {
const t = document.querySelector('table.calendar');
return t ? t.outerHTML : null;
}
""")
if not table_html:
logger.error('table.calendar not found in DOM')
return None
# Debug: save HTML for troubleshooting
with contextlib.suppress(Exception), open('/tmp/diary_debug.html', 'w', encoding='utf-8') as f: # noqa: S108
f.write(table_html)
month_text, days = _parse_calendar_html(table_html)
# Read event times by opening each event's AJAX popup.
times_by_id = await _collect_event_times(page)
if times_by_id:
for day_data in days.values():
for ev in day_data.get('events', []):
eid = ev.get('id')
if eid and eid in times_by_id:
ev['time'] = times_by_id[eid]
logger.info(
f'Diary parsed: month={month_text!r}, days_with_events={sum(1 for d in days.values() if d["events"])}/{len(days)}'
)
return {'monthFullText': month_text, 'days': days}
except Exception as e:
logger.error(f'Error parsing diary: {e}')
return None
finally:
with contextlib.suppress(Exception):
await asyncio.wait_for(page.close(), timeout=5)
with contextlib.suppress(Exception):
await asyncio.wait_for(context_browser.close(), timeout=5)
finally:
await page.close()
await context_browser.close()
finally:
await browser.close()
with contextlib.suppress(Exception):
await asyncio.wait_for(browser.close(), timeout=5)
except TimeoutError:
logger.error('Diary fetch timed out (60s)')
return None
except Exception as e:
logger.error(f'Playwright error in diary fetch: {e}')
return None
@@ -931,48 +1041,55 @@ def _parse_schedule_html(table_html: str) -> dict:
async def fetch_schedule_data(phpsessid: str) -> dict | None:
logger.info('Fetching schedule data via Playwright...')
try:
async with async_playwright() as p:
browser = await p.chromium.connect(PLAYWRIGHT_WS)
try:
context_browser = await browser.new_context(user_agent=USER_AGENT)
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
page = await context_browser.new_page()
async with asyncio.timeout(60):
async with async_playwright() as p:
browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
try:
await page.goto(SCHEDULE_URL, wait_until='domcontentloaded')
await page.wait_for_selector('table.schedule-table', timeout=10000)
await page.wait_for_timeout(1500)
context_browser = await browser.new_context(user_agent=USER_AGENT)
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
page = await context_browser.new_page()
table_html = await page.evaluate("""
() => {
const t = document.querySelector('table.schedule-table');
return t ? t.outerHTML : null;
}
""")
if not table_html:
logger.error('table.schedule-table not found in DOM')
try:
await asyncio.wait_for(page.goto(SCHEDULE_URL, wait_until='domcontentloaded'), timeout=30)
await page.wait_for_selector('table.schedule-table', timeout=10000)
await page.wait_for_timeout(1500)
table_html = await page.evaluate("""
() => {
const t = document.querySelector('table.schedule-table');
return t ? t.outerHTML : null;
}
""")
if not table_html:
logger.error('table.schedule-table not found in DOM')
return None
debug_path = os.path.join(tempfile.gettempdir(), 'schedule_debug.html')
with contextlib.suppress(Exception), open(debug_path, 'w', encoding='utf-8') as f:
f.write(table_html)
data = _parse_schedule_html(table_html)
logger.info(list(data['weekdays'].keys()))
logger.info(f'Schedule parsed: {len(data["weekdays"])} days, classes={data["classes"]}')
return data
except Exception as e:
logger.error(f'Error parsing schedule: {e}')
return None
debug_path = os.path.join(tempfile.gettempdir(), 'schedule_debug.html')
with contextlib.suppress(Exception), open(debug_path, 'w', encoding='utf-8') as f:
f.write(table_html)
data = _parse_schedule_html(table_html)
logger.info(list(data['weekdays'].keys()))
logger.info(f'Schedule parsed: {len(data["weekdays"])} days, classes={data["classes"]}')
return data
except Exception as e:
logger.error(f'Error parsing schedule: {e}')
return None
finally:
with contextlib.suppress(Exception):
await asyncio.wait_for(page.close(), timeout=5)
with contextlib.suppress(Exception):
await asyncio.wait_for(context_browser.close(), timeout=5)
finally:
await page.close()
await context_browser.close()
finally:
await browser.close()
with contextlib.suppress(Exception):
await asyncio.wait_for(browser.close(), timeout=5)
except TimeoutError:
logger.error('Schedule fetch timed out (60s)')
return None
except Exception as e:
logger.error(f'Playwright error in schedule fetch: {e}')
return None
@@ -1485,10 +1602,13 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
int: Number of webinars found, or None if check failed
"""
logger.info('Running webinar check...')
_t0 = time.time()
_metric_check_start()
phpsessid = redis_client.get(KEY_PHPSESSID)
if not phpsessid:
logger.warning('PHPSESSID missing. Skipping check.')
_metric_check_fail(time.time() - _t0, phpsessid_missing=True)
# --- DEBUG LOGGING ---
try:
with open('phpsessid_missing.log', 'a') as f:
@@ -1502,78 +1622,88 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
content = ''
try:
async with async_playwright() as p:
# Connect to remote Playwright service
browser = await p.chromium.connect(PLAYWRIGHT_WS)
try:
# Create browser context with user agent
context_browser = await browser.new_context(user_agent=USER_AGENT)
# Add PHPSESSID cookie
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
# Create new page
page = await context_browser.new_page()
async with asyncio.timeout(90):
async with async_playwright() as p:
# Connect to remote Playwright service
browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
try:
# Navigate to webinar page
await page.goto(WEBINAR_URL, wait_until='domcontentloaded')
# Create browser context with user agent
context_browser = await browser.new_context(user_agent=USER_AGENT)
# Wait for the table to load
await page.wait_for_selector('#meetings table', timeout=10000)
await page.wait_for_timeout(2000)
# Add PHPSESSID cookie
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
# Get page content
content = await page.content()
# Create new page
page = await context_browser.new_page()
# Check if "no webinar" message is present
if NO_WEBINAR_MARKER not in content:
logger.info('!!! WEBINAR FOUND !!!')
try:
# Navigate to webinar page
await asyncio.wait_for(page.goto(WEBINAR_URL, wait_until='domcontentloaded'), timeout=30)
# Extract webinar details from table rows
rows = page.locator('#meetings table tbody tr')
count = await rows.count()
# Wait for the table to load
await page.wait_for_selector('#meetings table', timeout=10000)
await page.wait_for_timeout(2000)
for i in range(count):
row = rows.nth(i)
text = await row.inner_text()
# Get page content
content = await page.content()
if NO_WEBINAR_MARKER not in text:
# Extract name (topic) from first column
name_elem = row.locator('td').nth(0)
name = await name_elem.inner_text()
name = name.strip()
# Check if "no webinar" message is present
if NO_WEBINAR_MARKER not in content:
logger.info('!!! WEBINAR FOUND !!!')
# Extract join URL from fourth column
url_elem = row.locator('td').nth(3).locator('a[href*="/webinar/join/"]').first
url = await url_elem.get_attribute('href')
# Extract webinar details from table rows
rows = page.locator('#meetings table tbody tr')
count = await rows.count()
if name and url:
current_webinars.append({'name': name, 'url': url, 'text': text.strip()})
logger.info(f'Found webinar: {name} -> {url}')
else:
logger.info('No webinars found (expected message present)')
for i in range(count):
row = rows.nth(i)
text = await row.inner_text()
except Exception as e:
logger.error(f'Error checking page: {e}. Saving content for debug.')
# If page content is available, save it on error
with contextlib.suppress(Exception):
if page and not content:
content = await page.content()
if NO_WEBINAR_MARKER not in text:
# Extract name (topic) from first column
name_elem = row.locator('td').nth(0)
name = await name_elem.inner_text()
name = name.strip()
# Extract join URL from fourth column
url_elem = row.locator('td').nth(3).locator('a[href*="/webinar/join/"]').first
url = await url_elem.get_attribute('href')
if name and url:
current_webinars.append({'name': name, 'url': url, 'text': text.strip()})
logger.info(f'Found webinar: {name} -> {url}')
else:
logger.info('No webinars found (expected message present)')
except Exception as e:
logger.error(f'Error checking page: {e}. Saving content for debug.')
# If page content is available, save it on error
with contextlib.suppress(Exception):
if page and not content:
content = await page.content()
_metric_check_fail(time.time() - _t0)
return None
finally:
with contextlib.suppress(Exception):
await asyncio.wait_for(page.close(), timeout=5)
with contextlib.suppress(Exception):
await asyncio.wait_for(context_browser.close(), timeout=5)
return None
finally:
await page.close()
await context_browser.close()
finally:
await browser.close()
with contextlib.suppress(Exception):
await asyncio.wait_for(browser.close(), timeout=5)
except TimeoutError:
logger.error('Webinar check timed out after 90s (playwright hang)')
_metric_check_fail(time.time() - _t0)
return None
except Exception as e:
logger.error(f'Playwright error: {e}')
_metric_check_fail(time.time() - _t0)
return None
# --- DEBUG LOGGING (Saving last response content) ---
@@ -1637,6 +1767,7 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
else:
logger.info(f'Found {len(current_webinars)} webinar(s), but all are already known')
_metric_check_ok(time.time() - _t0)
return len(current_webinars)
@@ -1680,6 +1811,8 @@ def main():
job_queue = app.job_queue
job_queue.run_repeating(check_webinars_job, interval=WEBINAR_CHECK_INTERVAL, first=10)
start_metrics_server()
logger.info('Bot started polling...')
app.run_polling()