forust/homelab
0
0 Fork 0
Code Issues 1 Pull Requests 3 Actions Packages Wiki Activity

chore(deploy): rework k8s pipeline, monitoring and postgres 17

Browse Source 
Deploy workflow uses git-tracked manifests, DISABLED flag and kustomize overlays; add webinar-checker metrics with ServiceMonitor and alerts; upgrade shared postgres to 17 with statuspage DB and probes/resources.
This commit is contained in:
forust committed 2026-09-23 15:47:26 +02:00
1 parent 8b2cf29771
commit 46c7e99b1d
19 files changed
+629 -227
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

edu_master/webinar-checker/checker.py
+275 -142
View File
@@ -1,12 +1,15 @@
import asyncio
import contextlib
import json
import logging
import os
import re
import tempfile
import threading
import time
from datetime import datetime, timedelta
from html import escape
from http.server import BaseHTTPRequestHandler, HTTPServer
import redis
from playwright.async_api import async_playwright
@@ -48,6 +51,106 @@ USER_AGENT = _env(
)
WEBINAR_TELEGRAM_TOKEN = _env('WEBINAR_TELEGRAM_TOKEN')
ADMIN_ID = int(_env('WEBINAR_ADMIN_ID', '0'))
METRICS_PORT = int(_env('METRICS_PORT', '8000'))
# --- Prometheus metrics (stdlib only, no extra deps) ---
# Scraped by prometheus-stack via ServiceMonitor (edu_master/k8s/servicemonitor.yaml).
# Critical alerts in edu_master/k8s/alerts.yaml fire to Telegram via Alertmanager.
_METRICS_LOCK = threading.Lock()
_METRICS = {
'last_run': 0.0, # Unix ts of last check start
'last_success': 0.0, # Unix ts of last successful check
'last_duration': 0.0, # Duration of last check in seconds
'success_total': 0,
'failure_total': 0,
'consecutive_failures': 0,
'phpsessid_present': 1, # 1 if EDU_PHPSESSID found in redis, else 0
}
def _metric_check_start():
with _METRICS_LOCK:
_METRICS['last_run'] = time.time()
def _metric_check_ok(duration: float):
now = time.time()
with _METRICS_LOCK:
_METRICS['last_success'] = now
_METRICS['last_duration'] = duration
_METRICS['success_total'] += 1
_METRICS['consecutive_failures'] = 0
_METRICS['phpsessid_present'] = 1
def _metric_check_fail(duration: float, phpsessid_missing: bool = False):
with _METRICS_LOCK:
_METRICS['last_duration'] = duration
_METRICS['failure_total'] += 1
_METRICS['consecutive_failures'] += 1
_METRICS['phpsessid_present'] = 0 if phpsessid_missing else 1
def _metrics_render() -> bytes:
with _METRICS_LOCK:
m = dict(_METRICS)
lines = [
'# HELP webinar_check_last_run_timestamp_seconds Unix timestamp of last webinar check start.',
'# TYPE webinar_check_last_run_timestamp_seconds gauge',
f'webinar_check_last_run_timestamp_seconds {m["last_run"]}',
'# HELP webinar_check_last_success_timestamp_seconds Unix timestamp of last successful webinar check.',
'# TYPE webinar_check_last_success_timestamp_seconds gauge',
f'webinar_check_last_success_timestamp_seconds {m["last_success"]}',
'# HELP webinar_check_last_duration_seconds Duration of last webinar check in seconds.',
'# TYPE webinar_check_last_duration_seconds gauge',
f'webinar_check_last_duration_seconds {m["last_duration"]}',
'# HELP webinar_check_success_total Total successful webinar checks.',
'# TYPE webinar_check_success_total counter',
f'webinar_check_success_total {m["success_total"]}',
'# HELP webinar_check_failure_total Total failed webinar checks (timeout, playwright error, page error).',
'# TYPE webinar_check_failure_total counter',
f'webinar_check_failure_total {m["failure_total"]}',
'# HELP webinar_check_consecutive_failures Consecutive failed webinar checks (reset on success).',
'# TYPE webinar_check_consecutive_failures gauge',
f'webinar_check_consecutive_failures {m["consecutive_failures"]}',
'# HELP edu_phpsessid_present 1 if EDU_PHPSESSID exists in redis, 0 otherwise.',
'# TYPE edu_phpsessid_present gauge',
f'edu_phpsessid_present {m["phpsessid_present"]}',
]
return ('\n'.join(lines) + '\n').encode()
class _MetricsHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/metrics':
body = _metrics_render()
self.send_response(200)
self.send_header('Content-Type', 'text/plain; version=0.0.4')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
elif self.path in ('/healthz', '/health'):
body = b'ok\n'
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
else:
self.send_response(404)
self.end_headers()
def log_message(self, *args):
pass # keep bot logs clean
def start_metrics_server(port: int = METRICS_PORT):
server = HTTPServer(('0.0.0.0', port), _MetricsHandler) # noqa: S104 - k8s ServiceMonitor scrapes pod IP
thread = threading.Thread(target=server.serve_forever, name='metrics-server', daemon=True)
thread.start()
logger.info(f'Metrics server listening on :{port}/metrics')
return server
# Redis Keys
KEY_WHITELIST = 'bot:whitelist'
@@ -597,59 +700,66 @@ async def _collect_event_times(page) -> dict:
async def fetch_diary_data(phpsessid: str) -> dict | None:
logger.info('Fetching diary data via Playwright...')
try:
async with async_playwright() as p:
browser = await p.chromium.connect(PLAYWRIGHT_WS)
try:
context_browser = await browser.new_context(user_agent=USER_AGENT)
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
page = await context_browser.new_page()
async with asyncio.timeout(60):
async with async_playwright() as p:
browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
try:
await page.goto(DIARY_URL, wait_until='domcontentloaded')
await page.wait_for_selector('table.calendar', timeout=10000)
await page.wait_for_timeout(1500)
table_html = await page.evaluate("""
() => {
const t = document.querySelector('table.calendar');
return t ? t.outerHTML : null;
}
""")
if not table_html:
logger.error('table.calendar not found in DOM')
return None
# Debug: save HTML for troubleshooting
with contextlib.suppress(Exception), open('/tmp/diary_debug.html', 'w', encoding='utf-8') as f: # noqa: S108
f.write(table_html)
month_text, days = _parse_calendar_html(table_html)
# Read event times by opening each event's AJAX popup.
times_by_id = await _collect_event_times(page)
if times_by_id:
for day_data in days.values():
for ev in day_data.get('events', []):
eid = ev.get('id')
if eid and eid in times_by_id:
ev['time'] = times_by_id[eid]
logger.info(
f'Diary parsed: month={month_text!r}, days_with_events={sum(1 for d in days.values() if d["events"])}/{len(days)}'
context_browser = await browser.new_context(user_agent=USER_AGENT)
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
page = await context_browser.new_page()
return {'monthFullText': month_text, 'days': days}
try:
await asyncio.wait_for(page.goto(DIARY_URL, wait_until='domcontentloaded'), timeout=30)
await page.wait_for_selector('table.calendar', timeout=10000)
await page.wait_for_timeout(1500)
except Exception as e:
logger.error(f'Error parsing diary: {e}')
return None
table_html = await page.evaluate("""
() => {
const t = document.querySelector('table.calendar');
return t ? t.outerHTML : null;
}
""")
if not table_html:
logger.error('table.calendar not found in DOM')
return None
# Debug: save HTML for troubleshooting
with contextlib.suppress(Exception), open('/tmp/diary_debug.html', 'w', encoding='utf-8') as f: # noqa: S108
f.write(table_html)
month_text, days = _parse_calendar_html(table_html)
# Read event times by opening each event's AJAX popup.
times_by_id = await _collect_event_times(page)
if times_by_id:
for day_data in days.values():
for ev in day_data.get('events', []):
eid = ev.get('id')
if eid and eid in times_by_id:
ev['time'] = times_by_id[eid]
logger.info(
f'Diary parsed: month={month_text!r}, days_with_events={sum(1 for d in days.values() if d["events"])}/{len(days)}'
)
return {'monthFullText': month_text, 'days': days}
except Exception as e:
logger.error(f'Error parsing diary: {e}')
return None
finally:
with contextlib.suppress(Exception):
await asyncio.wait_for(page.close(), timeout=5)
with contextlib.suppress(Exception):
await asyncio.wait_for(context_browser.close(), timeout=5)
finally:
await page.close()
await context_browser.close()
finally:
await browser.close()
with contextlib.suppress(Exception):
await asyncio.wait_for(browser.close(), timeout=5)
except TimeoutError:
logger.error('Diary fetch timed out (60s)')
return None
except Exception as e:
logger.error(f'Playwright error in diary fetch: {e}')
return None
@@ -931,48 +1041,55 @@ def _parse_schedule_html(table_html: str) -> dict:
async def fetch_schedule_data(phpsessid: str) -> dict | None:
logger.info('Fetching schedule data via Playwright...')
try:
async with async_playwright() as p:
browser = await p.chromium.connect(PLAYWRIGHT_WS)
try:
context_browser = await browser.new_context(user_agent=USER_AGENT)
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
page = await context_browser.new_page()
async with asyncio.timeout(60):
async with async_playwright() as p:
browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
try:
await page.goto(SCHEDULE_URL, wait_until='domcontentloaded')
await page.wait_for_selector('table.schedule-table', timeout=10000)
await page.wait_for_timeout(1500)
context_browser = await browser.new_context(user_agent=USER_AGENT)
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
page = await context_browser.new_page()
table_html = await page.evaluate("""
() => {
const t = document.querySelector('table.schedule-table');
return t ? t.outerHTML : null;
}
""")
if not table_html:
logger.error('table.schedule-table not found in DOM')
try:
await asyncio.wait_for(page.goto(SCHEDULE_URL, wait_until='domcontentloaded'), timeout=30)
await page.wait_for_selector('table.schedule-table', timeout=10000)
await page.wait_for_timeout(1500)
table_html = await page.evaluate("""
() => {
const t = document.querySelector('table.schedule-table');
return t ? t.outerHTML : null;
}
""")
if not table_html:
logger.error('table.schedule-table not found in DOM')
return None
debug_path = os.path.join(tempfile.gettempdir(), 'schedule_debug.html')
with contextlib.suppress(Exception), open(debug_path, 'w', encoding='utf-8') as f:
f.write(table_html)
data = _parse_schedule_html(table_html)
logger.info(list(data['weekdays'].keys()))
logger.info(f'Schedule parsed: {len(data["weekdays"])} days, classes={data["classes"]}')
return data
except Exception as e:
logger.error(f'Error parsing schedule: {e}')
return None
debug_path = os.path.join(tempfile.gettempdir(), 'schedule_debug.html')
with contextlib.suppress(Exception), open(debug_path, 'w', encoding='utf-8') as f:
f.write(table_html)
data = _parse_schedule_html(table_html)
logger.info(list(data['weekdays'].keys()))
logger.info(f'Schedule parsed: {len(data["weekdays"])} days, classes={data["classes"]}')
return data
except Exception as e:
logger.error(f'Error parsing schedule: {e}')
return None
finally:
with contextlib.suppress(Exception):
await asyncio.wait_for(page.close(), timeout=5)
with contextlib.suppress(Exception):
await asyncio.wait_for(context_browser.close(), timeout=5)
finally:
await page.close()
await context_browser.close()
finally:
await browser.close()
with contextlib.suppress(Exception):
await asyncio.wait_for(browser.close(), timeout=5)
except TimeoutError:
logger.error('Schedule fetch timed out (60s)')
return None
except Exception as e:
logger.error(f'Playwright error in schedule fetch: {e}')
return None
@@ -1485,10 +1602,13 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
int: Number of webinars found, or None if check failed
"""
logger.info('Running webinar check...')
_t0 = time.time()
_metric_check_start()
phpsessid = redis_client.get(KEY_PHPSESSID)
if not phpsessid:
logger.warning('PHPSESSID missing. Skipping check.')
_metric_check_fail(time.time() - _t0, phpsessid_missing=True)
# --- DEBUG LOGGING ---
try:
with open('phpsessid_missing.log', 'a') as f:
@@ -1502,78 +1622,88 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
content = ''
try:
async with async_playwright() as p:
# Connect to remote Playwright service
browser = await p.chromium.connect(PLAYWRIGHT_WS)
try:
# Create browser context with user agent
context_browser = await browser.new_context(user_agent=USER_AGENT)
# Add PHPSESSID cookie
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
# Create new page
page = await context_browser.new_page()
async with asyncio.timeout(90):
async with async_playwright() as p:
# Connect to remote Playwright service
browser = await asyncio.wait_for(p.chromium.connect(PLAYWRIGHT_WS), timeout=15)
try:
# Navigate to webinar page
await page.goto(WEBINAR_URL, wait_until='domcontentloaded')
# Create browser context with user agent
context_browser = await browser.new_context(user_agent=USER_AGENT)
# Wait for the table to load
await page.wait_for_selector('#meetings table', timeout=10000)
await page.wait_for_timeout(2000)
# Add PHPSESSID cookie
await context_browser.add_cookies(
[{'name': 'PHPSESSID', 'value': phpsessid, 'domain': 'edu.edu.vn.ua', 'path': '/'}]
)
# Get page content
content = await page.content()
# Create new page
page = await context_browser.new_page()
# Check if "no webinar" message is present
if NO_WEBINAR_MARKER not in content:
logger.info('!!! WEBINAR FOUND !!!')
try:
# Navigate to webinar page
await asyncio.wait_for(page.goto(WEBINAR_URL, wait_until='domcontentloaded'), timeout=30)
# Extract webinar details from table rows
rows = page.locator('#meetings table tbody tr')
count = await rows.count()
# Wait for the table to load
await page.wait_for_selector('#meetings table', timeout=10000)
await page.wait_for_timeout(2000)
for i in range(count):
row = rows.nth(i)
text = await row.inner_text()
# Get page content
content = await page.content()
if NO_WEBINAR_MARKER not in text:
# Extract name (topic) from first column
name_elem = row.locator('td').nth(0)
name = await name_elem.inner_text()
name = name.strip()
# Check if "no webinar" message is present
if NO_WEBINAR_MARKER not in content:
logger.info('!!! WEBINAR FOUND !!!')
# Extract join URL from fourth column
url_elem = row.locator('td').nth(3).locator('a[href*="/webinar/join/"]').first
url = await url_elem.get_attribute('href')
# Extract webinar details from table rows
rows = page.locator('#meetings table tbody tr')
count = await rows.count()
if name and url:
current_webinars.append({'name': name, 'url': url, 'text': text.strip()})
logger.info(f'Found webinar: {name} -> {url}')
else:
logger.info('No webinars found (expected message present)')
for i in range(count):
row = rows.nth(i)
text = await row.inner_text()
except Exception as e:
logger.error(f'Error checking page: {e}. Saving content for debug.')
# If page content is available, save it on error
with contextlib.suppress(Exception):
if page and not content:
content = await page.content()
if NO_WEBINAR_MARKER not in text:
# Extract name (topic) from first column
name_elem = row.locator('td').nth(0)
name = await name_elem.inner_text()
name = name.strip()
# Extract join URL from fourth column
url_elem = row.locator('td').nth(3).locator('a[href*="/webinar/join/"]').first
url = await url_elem.get_attribute('href')
if name and url:
current_webinars.append({'name': name, 'url': url, 'text': text.strip()})
logger.info(f'Found webinar: {name} -> {url}')
else:
logger.info('No webinars found (expected message present)')
except Exception as e:
logger.error(f'Error checking page: {e}. Saving content for debug.')
# If page content is available, save it on error
with contextlib.suppress(Exception):
if page and not content:
content = await page.content()
_metric_check_fail(time.time() - _t0)
return None
finally:
with contextlib.suppress(Exception):
await asyncio.wait_for(page.close(), timeout=5)
with contextlib.suppress(Exception):
await asyncio.wait_for(context_browser.close(), timeout=5)
return None
finally:
await page.close()
await context_browser.close()
finally:
await browser.close()
with contextlib.suppress(Exception):
await asyncio.wait_for(browser.close(), timeout=5)
except TimeoutError:
logger.error('Webinar check timed out after 90s (playwright hang)')
_metric_check_fail(time.time() - _t0)
return None
except Exception as e:
logger.error(f'Playwright error: {e}')
_metric_check_fail(time.time() - _t0)
return None
# --- DEBUG LOGGING (Saving last response content) ---
@@ -1637,6 +1767,7 @@ async def check_webinars_job(context: ContextTypes.DEFAULT_TYPE):
else:
logger.info(f'Found {len(current_webinars)} webinar(s), but all are already known')
_metric_check_ok(time.time() - _t0)
return len(current_webinars)
@@ -1680,6 +1811,8 @@ def main():
job_queue = app.job_queue
job_queue.run_repeating(check_webinars_job, interval=WEBINAR_CHECK_INTERVAL, first=10)
start_metrics_server()
logger.info('Bot started polling...')
app.run_polling()