Refactor server state and add health checks

webui_server.py

@@ -21,6 +21,9 @@ from typing import Any, Dict, List, Optional
 
 import qrcode
 import qrcode.image.svg
+from app_state import StateStore
+from health import health_payload
+from scraper_jobs import ScraperJobService
 from telethon import TelegramClient
 from telethon.errors import SessionPasswordNeededError
 
@@ -31,8 +34,11 @@ WEBUI_DIR = BASE_DIR / "webui"
 STATE_FILE = DATA_DIR / "state.json"
 DEFAULT_HOST = os.environ.get("TELEGRAM_SCRAPER_HOST", "0.0.0.0")
 DEFAULT_PORT = int(os.environ.get("TELEGRAM_SCRAPER_PORT", "8080"))
+START_CONTINUOUS = os.environ.get("TELEGRAM_SCRAPER_START_CONTINUOUS", "1") != "0"
 SESSION_DIR = BASE_DIR / "session"
 SESSION_DIR.mkdir(exist_ok=True)
+STATE_STORE = StateStore(STATE_FILE)
+SCRAPER_JOBS = ScraperJobService(STATE_STORE)
 
 
 def utc_now_iso() -> str:
@@ -40,23 +46,11 @@ def utc_now_iso() -> str:
 
 
 def load_state() -> Dict[str, Any]:
-    if STATE_FILE.exists():
-        with STATE_FILE.open("r", encoding="utf-8") as handle:
-            return json.load(handle)
-    return {
-        "api_id": None,
-        "api_hash": None,
-        "channels": {},
-        "channel_names": {},
-        "scrape_media": True,
-        "forwarding_rules": [],
-    }
+    return STATE_STORE.load()
 
 
 def save_state(state: Dict[str, Any]) -> None:
-    DATA_DIR.mkdir(parents=True, exist_ok=True)
-    with STATE_FILE.open("w", encoding="utf-8") as handle:
-        json.dump(state, handle, ensure_ascii=False, indent=2)
+    STATE_STORE.save(state)
 
 
 def channel_db_path(channel_id: str) -> Path:
@@ -496,12 +490,7 @@ class ContinuousScrapeManager:
         self.lock = threading.RLock()
         self.thread: Optional[threading.Thread] = None
         self.stop_event = threading.Event()
-        self.config: Dict[str, Any] = {
-            "enabled": True,
-            "interval_minutes": 1,
-            "channels": [],
-            "run_all_tracked": True,
-        }
+        self.config: Dict[str, Any] = STATE_STORE.continuous_config()
         self.status: Dict[str, Any] = {
             "running": False,
             "last_started_at": None,
@@ -546,12 +535,14 @@ class ContinuousScrapeManager:
     ) -> Dict[str, Any]:
         interval_minutes = max(1, int(interval_minutes))
         with self.lock:
-            self.config = {
-                "enabled": bool(enabled),
-                "interval_minutes": interval_minutes,
-                "channels": channels,
-                "run_all_tracked": bool(run_all_tracked),
-            }
+            self.config = STATE_STORE.save_continuous_config(
+                {
+                    "enabled": bool(enabled),
+                    "interval_minutes": interval_minutes,
+                    "channels": channels,
+                    "run_all_tracked": bool(run_all_tracked),
+                }
+            )
         if enabled:
             self.start()
         else:
@@ -641,63 +632,7 @@ def import_scraper_class():
 
 
 def run_job(job_type: str, payload: Dict[str, Any]) -> None:
-    if job_type == "set_scrape_media":
-        state = load_state()
-        state["scrape_media"] = bool(payload["value"])
-        save_state(state)
-        print(f"Media scraping set to {state['scrape_media']}")
-        return
-
-    async def _async_job() -> None:
-        ScraperClass = import_scraper_class()
-        scraper = ScraperClass()
-        initialized = await scraper.initialize_client(interactive=False)
-        if not initialized:
-            raise RuntimeError(
-                "Telegram client is not ready. Check credentials, session, and write access to /app/session."
-            )
-        try:
-            if job_type == "scrape_channel":
-                channel_id = payload["channel_id"]
-                state = scraper.load_state()
-                offset = int(state.get("channels", {}).get(channel_id, 0) or 0)
-                await scraper.scrape_channel(channel_id, offset)
-            elif job_type == "scrape_all":
-                channels = list(scraper.state.get("channels", {}).keys())
-                for channel_id in channels:
-                    offset = int(
-                        scraper.state.get("channels", {}).get(channel_id, 0) or 0
-                    )
-                    await scraper.scrape_channel(channel_id, offset)
-            elif job_type == "scrape_selected":
-                channels = [
-                    str(channel_id) for channel_id in payload.get("channels", [])
-                ]
-                for channel_id in channels:
-                    offset = int(
-                        scraper.state.get("channels", {}).get(channel_id, 0) or 0
-                    )
-                    await scraper.scrape_channel(channel_id, offset)
-            elif job_type == "export_all":
-                await scraper.export_data()
-            elif job_type == "export_channel":
-                channel_id = payload["channel_id"]
-                scraper.export_to_csv(channel_id)
-                scraper.export_to_json(channel_id)
-            elif job_type == "rescrape_media":
-                await scraper.rescrape_media(payload["channel_id"])
-            elif job_type == "fix_missing_media":
-                await scraper.fix_missing_media(payload["channel_id"])
-            elif job_type == "refresh_dialogs":
-                await scraper.list_channels()
-            else:
-                raise RuntimeError(f"Unsupported job type: {job_type}")
-        finally:
-            scraper.close_db_connections()
-            if scraper.client:
-                await scraper.client.disconnect()
-
-    asyncio.run(_async_job())
+    SCRAPER_JOBS.run(job_type, payload)
 
 
 def auth_status() -> Dict[str, Any]:
@@ -882,6 +817,13 @@ def openapi_payload() -> Dict[str, Any]:
                     "responses": json_response,
                 }
             },
+            "/health": {
+                "get": {
+                    "summary": "Application health",
+                    "description": "Checks data/session write access, state loading, SQLite, continuous status, and job queue size.",
+                    "responses": json_response,
+                }
+            },
             "/api/channels": {
                 "get": {
                     "summary": "List tracked channels",
@@ -1066,6 +1008,16 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
             return self.send_json(self.app.continuous_manager.snapshot())
         if path == "/health/continuous":
             return self.send_json(self.app.continuous_manager.snapshot())
+        if path == "/health":
+            return self.send_json(
+                health_payload(
+                    DATA_DIR,
+                    SESSION_DIR,
+                    STATE_STORE,
+                    self.app.continuous_manager.snapshot(),
+                    self.app.job_runner.queue.qsize(),
+                )
+            )
         if path == "/api/jobs":
             return self.send_json(self.app.job_runner.recent_jobs())
         if path.startswith("/api/jobs/"):
@@ -1131,6 +1083,7 @@ class TelegramScraperRequestHandler(BaseHTTPRequestHandler):
                 "/api/continuous",
                 "/api/jobs",
                 "/api/channels",
+                "/health",
                 "/health/continuous",
                 "/openapi.json",
             }
@@ -1397,7 +1350,8 @@ class TelegramScraperWebServer(ThreadingHTTPServer):
         self.job_runner = JobRunner()
         self.auth_manager = TelegramAuthManager()
         self.continuous_manager = ContinuousScrapeManager()
-        self.continuous_manager.start()
+        if START_CONTINUOUS and self.continuous_manager.snapshot()["config"].get("enabled", True):
+            self.continuous_manager.start()
 
 
 def run_server(host: str = DEFAULT_HOST, port: int = DEFAULT_PORT) -> None: