from __future__ import annotations import json import sqlite3 from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any, Iterator SCHEMA = """ CREATE TABLE IF NOT EXISTS channels ( channel_id TEXT PRIMARY KEY, handle TEXT, name TEXT, last_scraped TEXT, video_count INTEGER DEFAULT 0 ); CREATE TABLE IF NOT EXISTS videos ( video_id TEXT PRIMARY KEY, channel_id TEXT NOT NULL, title TEXT, url TEXT NOT NULL, upload_date TEXT, duration INTEGER, status TEXT NOT NULL DEFAULT 'pending', error_msg TEXT, transcript_lang TEXT, transcript_src TEXT, has_chapters INTEGER DEFAULT 0, markdown_path TEXT, discovered_at TEXT NOT NULL, processed_at TEXT, FOREIGN KEY (channel_id) REFERENCES channels(channel_id) ); CREATE INDEX IF NOT EXISTS idx_videos_status ON videos(status); CREATE INDEX IF NOT EXISTS idx_videos_channel ON videos(channel_id); """ # New columns added by the platform migration (idempotent ALTERs) _VIDEO_COLUMNS: dict[str, str] = { "view_count": "INTEGER", "like_count": "INTEGER", "tags": "TEXT", "thumbnail": "TEXT", "description": "TEXT", "chapters_json": "TEXT", "segments_json": "TEXT", } _EXTRA_SCHEMA = """ CREATE TABLE IF NOT EXISTS transcript_segments ( video_id TEXT NOT NULL, idx INTEGER NOT NULL, start_sec REAL NOT NULL, end_sec REAL NOT NULL, text TEXT NOT NULL, PRIMARY KEY (video_id, idx), FOREIGN KEY (video_id) REFERENCES videos(video_id) ); CREATE INDEX IF NOT EXISTS idx_segments_video ON transcript_segments(video_id); CREATE VIRTUAL TABLE IF NOT EXISTS transcript_fts USING fts5( video_id UNINDEXED, idx UNINDEXED, start_sec UNINDEXED, end_sec UNINDEXED, text ); CREATE TABLE IF NOT EXISTS cookies_meta ( id TEXT PRIMARY KEY, filename TEXT NOT NULL, label TEXT, added_at TEXT NOT NULL, expires_at TEXT, is_active INTEGER DEFAULT 0, has_session INTEGER DEFAULT 0, cookie_count INTEGER DEFAULT 0 ); CREATE TABLE IF NOT EXISTS scrape_jobs ( id TEXT PRIMARY KEY, channel_id TEXT, opts_json TEXT, status TEXT NOT NULL, started_at TEXT, finished_at TEXT, total INTEGER DEFAULT 0, completed INTEGER DEFAULT 0, last_error TEXT ); """ @dataclass class VideoRef: video_id: str channel_id: str title: str url: str upload_date: str | None = None duration: int | None = None @dataclass class VideoRow: video_id: str channel_id: str title: str | None url: str upload_date: str | None duration: int | None status: str error_msg: str | None transcript_lang: str | None transcript_src: str | None has_chapters: int markdown_path: str | None view_count: int | None = None like_count: int | None = None tags: str | None = None thumbnail: str | None = None description: str | None = None chapters_json: str | None = None segments_json: str | None = None @dataclass class SegmentRow: video_id: str idx: int start_sec: float end_sec: float text: str @dataclass class SearchHit: video_id: str channel_id: str title: str | None idx: int start_sec: float end_sec: float snippet: str rank: float @dataclass class CookieRow: id: str filename: str label: str | None added_at: str expires_at: str | None is_active: bool has_session: bool cookie_count: int @dataclass class JobRow: id: str channel_id: str | None opts_json: str | None status: str started_at: str | None finished_at: str | None total: int completed: int last_error: str | None class Store: def __init__(self, db_path: str | Path): self.db_path = Path(db_path) self.db_path.parent.mkdir(parents=True, exist_ok=True) self._init_schema() def _connect(self) -> sqlite3.Connection: conn = sqlite3.connect(str(self.db_path)) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA foreign_keys=ON") return conn def _init_schema(self) -> None: with self._connect() as conn: conn.executescript(SCHEMA) # idempotent column adds existing = {row["name"] for row in conn.execute("PRAGMA table_info(videos)")} for col, coltype in _VIDEO_COLUMNS.items(): if col not in existing: conn.execute(f"ALTER TABLE videos ADD COLUMN {col} {coltype}") conn.executescript(_EXTRA_SCHEMA) @contextmanager def _cursor(self) -> Iterator[sqlite3.Cursor]: conn = self._connect() try: yield conn.cursor() conn.commit() finally: conn.close() # ------------------------------------------------------------------ channels def upsert_channel(self, channel_id: str, handle: str | None, name: str | None, video_count: int = 0) -> None: now = _now_iso() with self._cursor() as cur: cur.execute( """INSERT INTO channels (channel_id, handle, name, last_scraped, video_count) VALUES (?, ?, ?, ?, ?) ON CONFLICT(channel_id) DO UPDATE SET handle = excluded.handle, name = excluded.name, last_scraped = excluded.last_scraped, video_count = excluded.video_count""", (channel_id, handle, name, now, video_count), ) def list_channels(self) -> list[dict]: with self._cursor() as cur: cur.execute("SELECT * FROM channels ORDER BY name") return [dict(r) for r in cur.fetchall()] def get_channel(self, channel_id: str) -> dict | None: with self._cursor() as cur: cur.execute("SELECT * FROM channels WHERE channel_id = ?", (channel_id,)) row = cur.fetchone() return dict(row) if row else None def delete_channel(self, channel_id: str) -> None: with self._cursor() as cur: cur.execute("DELETE FROM transcript_segments WHERE video_id IN (SELECT video_id FROM videos WHERE channel_id = ?)", (channel_id,)) cur.execute("DELETE FROM transcript_fts WHERE video_id IN (SELECT video_id FROM videos WHERE channel_id = ?)", (channel_id,)) cur.execute("DELETE FROM videos WHERE channel_id = ?", (channel_id,)) cur.execute("DELETE FROM channels WHERE channel_id = ?", (channel_id,)) # ------------------------------------------------------------------ videos def upsert_videos(self, refs: list[VideoRef]) -> int: now = _now_iso() inserted = 0 with self._cursor() as cur: for r in refs: cur.execute( """INSERT INTO videos (video_id, channel_id, title, url, upload_date, duration, status, discovered_at) VALUES (?, ?, ?, ?, ?, ?, 'pending', ?) ON CONFLICT(video_id) DO UPDATE SET title = excluded.title, upload_date = excluded.upload_date, duration = excluded.duration""", (r.video_id, r.channel_id, r.title, r.url, r.upload_date, r.duration, now), ) if cur.rowcount > 0: inserted += 1 return inserted def get_pending(self, channel_id: str | None = None, limit: int | None = None) -> list[VideoRow]: sql = "SELECT * FROM videos WHERE status = 'pending'" params: list[Any] = [] if channel_id: sql += " AND channel_id = ?" params.append(channel_id) sql += " ORDER BY discovered_at ASC" if limit: sql += " LIMIT ?" params.append(limit) with self._cursor() as cur: cur.execute(sql, params) return [_row_to_videorow(row) for row in cur.fetchall()] def get_all(self, channel_id: str | None = None) -> list[VideoRow]: sql = "SELECT * FROM videos" params: list[Any] = [] if channel_id: sql += " WHERE channel_id = ?" params.append(channel_id) sql += " ORDER BY discovered_at ASC" with self._cursor() as cur: cur.execute(sql, params) return [_row_to_videorow(row) for row in cur.fetchall()] def get_video(self, video_id: str) -> VideoRow | None: with self._cursor() as cur: cur.execute("SELECT * FROM videos WHERE video_id = ?", (video_id,)) row = cur.fetchone() return _row_to_videorow(row) if row else None def query_videos( self, channel_id: str | None = None, status: str | None = None, date_from: str | None = None, date_to: str | None = None, min_duration: int | None = None, q: str | None = None, sort: str = "upload_date_desc", page: int = 1, size: int = 50, ) -> tuple[list[VideoRow], int]: where: list[str] = [] params: list[Any] = [] if channel_id: where.append("channel_id = ?"); params.append(channel_id) if status: where.append("status = ?"); params.append(status) if date_from: where.append("upload_date >= ?"); params.append(date_from.replace("-", "")) if date_to: where.append("upload_date <= ?"); params.append(date_to.replace("-", "")) if min_duration is not None: where.append("duration >= ?"); params.append(min_duration) if q: where.append("(title LIKE ? OR description LIKE ?)") params.extend([f"%{q}%", f"%{q}%"]) clause = ("WHERE " + " AND ".join(where)) if where else "" order = { "upload_date_desc": "upload_date DESC", "upload_date_asc": "upload_date ASC", "duration_desc": "duration DESC", "views_desc": "view_count DESC", "title_asc": "title ASC", }.get(sort, "upload_date DESC") offset = max(0, (page - 1) * size) with self._cursor() as cur: cur.execute(f"SELECT COUNT(*) AS n FROM videos {clause}", params) total = cur.fetchone()["n"] cur.execute( f"SELECT * FROM videos {clause} ORDER BY {order} LIMIT ? OFFSET ?", [*params, size, offset], ) rows = [_row_to_videorow(r) for r in cur.fetchall()] return rows, total def update_video_metadata( self, video_id: str, *, view_count: int | None = None, like_count: int | None = None, tags: list[str] | None = None, thumbnail: str | None = None, description: str | None = None, chapters_json: str | None = None, segments_json: str | None = None, ) -> None: sets: list[str] = [] params: list[Any] = [] if view_count is not None: sets.append("view_count = ?"); params.append(view_count) if like_count is not None: sets.append("like_count = ?"); params.append(like_count) if tags is not None: sets.append("tags = ?"); params.append(json.dumps(tags, ensure_ascii=False)) if thumbnail is not None: sets.append("thumbnail = ?"); params.append(thumbnail) if description is not None: sets.append("description = ?"); params.append(description[:4000]) if chapters_json is not None: sets.append("chapters_json = ?"); params.append(chapters_json) if segments_json is not None: sets.append("segments_json = ?"); params.append(segments_json) if not sets: return params.append(video_id) with self._cursor() as cur: cur.execute(f"UPDATE videos SET {', '.join(sets)} WHERE video_id = ?", params) def mark_done( self, video_id: str, markdown_path: str, transcript_lang: str | None, transcript_src: str | None, has_chapters: bool, ) -> None: now = _now_iso() with self._cursor() as cur: cur.execute( """UPDATE videos SET status = 'done', markdown_path = ?, transcript_lang = ?, transcript_src = ?, has_chapters = ?, processed_at = ?, error_msg = NULL WHERE video_id = ?""", (markdown_path, transcript_lang, transcript_src, int(has_chapters), now, video_id), ) def mark_status(self, video_id: str, status: str) -> None: now = _now_iso() with self._cursor() as cur: cur.execute( "UPDATE videos SET status = ?, processed_at = ? WHERE video_id = ?", (status, now, video_id), ) def set_upload_date(self, video_id: str, upload_date: str | None) -> None: if not upload_date: return with self._cursor() as cur: cur.execute( "UPDATE videos SET upload_date = ? WHERE video_id = ? AND upload_date IS NULL", (upload_date, video_id), ) def mark_error(self, video_id: str, error_msg: str) -> None: now = _now_iso() with self._cursor() as cur: cur.execute( """UPDATE videos SET status = 'error', error_msg = ?, processed_at = ? WHERE video_id = ?""", (error_msg[:500], now, video_id), ) # ------------------------------------------------------------------ segments / FTS def store_segments(self, video_id: str, segments: list) -> None: with self._cursor() as cur: cur.execute("DELETE FROM transcript_segments WHERE video_id = ?", (video_id,)) cur.execute("DELETE FROM transcript_fts WHERE video_id = ?", (video_id,)) rows = [] fts_rows = [] for i, seg in enumerate(segments): start = float(getattr(seg, "start", 0.0)) end = float(getattr(seg, "end", start)) text = (getattr(seg, "text", "") or "").strip() if not text: continue rows.append((video_id, i, start, end, text)) fts_rows.append((video_id, i, start, end, text)) if rows: cur.executemany( "INSERT INTO transcript_segments (video_id, idx, start_sec, end_sec, text) VALUES (?, ?, ?, ?, ?)", rows, ) cur.executemany( "INSERT INTO transcript_fts (video_id, idx, start_sec, end_sec, text) VALUES (?, ?, ?, ?, ?)", fts_rows, ) def get_segments(self, video_id: str) -> list[SegmentRow]: with self._cursor() as cur: cur.execute( "SELECT * FROM transcript_segments WHERE video_id = ? ORDER BY idx ASC", (video_id,), ) return [ SegmentRow( video_id=r["video_id"], idx=r["idx"], start_sec=r["start_sec"], end_sec=r["end_sec"], text=r["text"], ) for r in cur.fetchall() ] def has_segments(self, video_id: str) -> bool: with self._cursor() as cur: cur.execute("SELECT 1 FROM transcript_segments WHERE video_id = ? LIMIT 1", (video_id,)) return cur.fetchone() is not None def search_segments(self, query: str, channel_id: str | None = None, limit: int = 50) -> list[SearchHit]: # FTS5 MATCH; join videos for title + optional channel filter. fts_query = _sanitize_fts(query) if not fts_query: return [] sql = ( "SELECT f.video_id, f.idx, f.start_sec, f.end_sec, f.text, v.channel_id, v.title, bm25(transcript_fts) AS rank " "FROM transcript_fts f JOIN videos v ON v.video_id = f.video_id " "WHERE transcript_fts MATCH ?" ) params: list[Any] = [fts_query] if channel_id: sql += " AND v.channel_id = ?" params.append(channel_id) sql += " ORDER BY rank LIMIT ?" params.append(limit) with self._cursor() as cur: cur.execute(sql, params) return [ SearchHit( video_id=r["video_id"], channel_id=r["channel_id"], title=r["title"], idx=r["idx"], start_sec=r["start_sec"], end_sec=r["end_sec"], snippet=r["text"], rank=r["rank"], ) for r in cur.fetchall() ] # ------------------------------------------------------------------ cookies def upsert_cookie(self, cookie_id: str, filename: str, label: str | None, expires_at: str | None, has_session: bool, cookie_count: int) -> None: now = _now_iso() with self._cursor() as cur: cur.execute( """INSERT INTO cookies_meta (id, filename, label, added_at, expires_at, is_active, has_session, cookie_count) VALUES (?, ?, ?, ?, ?, 0, ?, ?) ON CONFLICT(id) DO UPDATE SET filename = excluded.filename, label = excluded.label, expires_at = excluded.expires_at, has_session = excluded.has_session, cookie_count = excluded.cookie_count""", (cookie_id, filename, label, now, expires_at, int(has_session), cookie_count), ) def list_cookies(self) -> list[CookieRow]: with self._cursor() as cur: cur.execute("SELECT * FROM cookies_meta ORDER BY added_at DESC") return [_row_to_cookierow(r) for r in cur.fetchall()] def get_cookie(self, cookie_id: str) -> CookieRow | None: with self._cursor() as cur: cur.execute("SELECT * FROM cookies_meta WHERE id = ?", (cookie_id,)) r = cur.fetchone() return _row_to_cookierow(r) if r else None def set_active_cookie(self, cookie_id: str | None) -> None: with self._cursor() as cur: cur.execute("UPDATE cookies_meta SET is_active = 0") if cookie_id: cur.execute("UPDATE cookies_meta SET is_active = 1 WHERE id = ?", (cookie_id,)) def get_active_cookie(self) -> CookieRow | None: with self._cursor() as cur: cur.execute("SELECT * FROM cookies_meta WHERE is_active = 1 LIMIT 1") r = cur.fetchone() return _row_to_cookierow(r) if r else None def delete_cookie(self, cookie_id: str) -> None: with self._cursor() as cur: cur.execute("DELETE FROM cookies_meta WHERE id = ?", (cookie_id,)) # ------------------------------------------------------------------ jobs def create_job(self, job_id: str, channel_id: str | None, opts: dict) -> None: now = _now_iso() with self._cursor() as cur: cur.execute( """INSERT INTO scrape_jobs (id, channel_id, opts_json, status, started_at, total, completed) VALUES (?, ?, ?, 'queued', ?, 0, 0)""", (job_id, channel_id, json.dumps(opts, ensure_ascii=False), now), ) def update_job(self, job_id: str, *, status: str | None = None, total: int | None = None, completed: int | None = None, last_error: str | None = None, finished: bool = False) -> None: sets: list[str] = [] params: list[Any] = [] if status: sets.append("status = ?"); params.append(status) if total is not None: sets.append("total = ?"); params.append(total) if completed is not None: sets.append("completed = ?"); params.append(completed) if last_error is not None: sets.append("last_error = ?"); params.append(last_error[:500]) if finished: sets.append("finished_at = ?"); params.append(_now_iso()) if not sets: return params.append(job_id) with self._cursor() as cur: cur.execute(f"UPDATE scrape_jobs SET {', '.join(sets)} WHERE id = ?", params) def get_job(self, job_id: str) -> JobRow | None: with self._cursor() as cur: cur.execute("SELECT * FROM scrape_jobs WHERE id = ?", (job_id,)) r = cur.fetchone() return _row_to_jobrow(r) if r else None def list_jobs(self, limit: int = 50) -> list[JobRow]: with self._cursor() as cur: cur.execute("SELECT * FROM scrape_jobs ORDER BY started_at DESC LIMIT ?", (limit,)) return [_row_to_jobrow(r) for r in cur.fetchall()] TERMINAL_STATUSES = ("done", "error", "cancelled") def delete_job(self, job_id: str) -> bool: with self._cursor() as cur: cur.execute("DELETE FROM scrape_jobs WHERE id = ?", (job_id,)) return cur.rowcount > 0 def delete_terminal_jobs(self) -> int: placeholders = ",".join("?" for _ in self.TERMINAL_STATUSES) with self._cursor() as cur: cur.execute( f"DELETE FROM scrape_jobs WHERE status IN ({placeholders})", list(self.TERMINAL_STATUSES), ) return cur.rowcount # ------------------------------------------------------------------ aggregates def stats(self, channel_id: str | None = None) -> dict[str, int]: sql = "SELECT status, COUNT(*) as n FROM videos" params: list[Any] = [] if channel_id: sql += " WHERE channel_id = ?" params.append(channel_id) sql += " GROUP BY status" result: dict[str, int] = {} with self._cursor() as cur: cur.execute(sql, params) for row in cur.fetchall(): result[row["status"]] = row["n"] return result def dashboard(self) -> dict: with self._cursor() as cur: channels = [dict(r) for r in cur.execute("SELECT * FROM channels ORDER BY name").fetchall()] for ch in channels: cid = ch["channel_id"] row = cur.execute( "SELECT COUNT(*) AS n, COALESCE(SUM(duration),0) AS dur, MIN(upload_date) AS mind, MAX(upload_date) AS maxd, COALESCE(SUM(view_count),0) AS views, COALESCE(SUM(like_count),0) AS likes FROM videos WHERE channel_id = ?", (cid,), ).fetchone() ch["video_count_db"] = row["n"] ch["total_duration"] = row["dur"] ch["date_min"] = row["mind"] ch["date_max"] = row["maxd"] ch["total_views"] = row["views"] ch["total_likes"] = row["likes"] status_breakdown = { r["status"]: r["n"] for r in cur.execute("SELECT status, COUNT(*) AS n FROM videos GROUP BY status").fetchall() } # uploads over time (by month) uploads = [ {"month": r["m"], "count": r["n"]} for r in cur.execute( "SELECT substr(upload_date,1,6) AS m, COUNT(*) AS n FROM videos WHERE upload_date IS NOT NULL GROUP BY m ORDER BY m" ).fetchall() ] # duration histogram (buckets) hist = [ {"bucket": r["b"], "count": r["n"]} for r in cur.execute( """SELECT CASE WHEN duration < 300 THEN '<5m' WHEN duration < 600 THEN '5-10m' WHEN duration < 1200 THEN '10-20m' WHEN duration < 2400 THEN '20-40m' ELSE '40m+' END AS b, COUNT(*) AS n FROM videos WHERE duration IS NOT NULL GROUP BY b""" ).fetchall() ] # top tags (tags is JSON array text) tag_rows = cur.execute("SELECT tags FROM videos WHERE tags IS NOT NULL AND tags != '[]'").fetchall() tag_counts: dict[str, int] = {} for tr in tag_rows: try: for t in json.loads(tr["tags"]): t = (t or "").strip().lower() if t: tag_counts[t] = tag_counts.get(t, 0) + 1 except (json.JSONDecodeError, TypeError): continue top_tags = sorted(tag_counts.items(), key=lambda x: x[1], reverse=True)[:20] return { "channels": channels, "status_breakdown": status_breakdown, "uploads_over_time": uploads, "duration_histogram": hist, "top_tags": [{"tag": t, "count": c} for t, c in top_tags], } def reset_errors(self, channel_id: str | None = None) -> int: sql = "UPDATE videos SET status = 'pending', error_msg = NULL WHERE status = 'error'" params: list[Any] = [] if channel_id: sql += " AND channel_id = ?" params.append(channel_id) with self._cursor() as cur: cur.execute(sql, params) return cur.rowcount def _sanitize_fts(query: str) -> str: # Build a safe AND FTS5 query from whitespace-separated terms. cleaned = (query or "").strip() if not cleaned: return "" terms = [] for tok in cleaned.split(): tok = tok.strip('"') if tok: terms.append(f'"{tok.replace("\"", "")}"') return " AND ".join(terms) def _now_iso() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds") def _row_to_videorow(row: sqlite3.Row) -> VideoRow: keys = row.keys() return VideoRow( video_id=row["video_id"], channel_id=row["channel_id"], title=row["title"], url=row["url"], upload_date=row["upload_date"], duration=row["duration"], status=row["status"], error_msg=row["error_msg"], transcript_lang=row["transcript_lang"], transcript_src=row["transcript_src"], has_chapters=row["has_chapters"], markdown_path=row["markdown_path"], view_count=row["view_count"] if "view_count" in keys else None, like_count=row["like_count"] if "like_count" in keys else None, tags=row["tags"] if "tags" in keys else None, thumbnail=row["thumbnail"] if "thumbnail" in keys else None, description=row["description"] if "description" in keys else None, chapters_json=row["chapters_json"] if "chapters_json" in keys else None, segments_json=row["segments_json"] if "segments_json" in keys else None, ) def _row_to_cookierow(row: sqlite3.Row) -> CookieRow: return CookieRow( id=row["id"], filename=row["filename"], label=row["label"], added_at=row["added_at"], expires_at=row["expires_at"], is_active=bool(row["is_active"]), has_session=bool(row["has_session"]), cookie_count=row["cookie_count"], ) def _row_to_jobrow(row: sqlite3.Row) -> JobRow: return JobRow( id=row["id"], channel_id=row["channel_id"], opts_json=row["opts_json"], status=row["status"], started_at=row["started_at"], finished_at=row["finished_at"], total=row["total"], completed=row["completed"], last_error=row["last_error"], )