feat: local content-mining platform for YouTube creator scraping
Scraper de canales de YouTube hacia notas Markdown para base de conocimiento (Obsidian-ready), con plataforma web local. Engine + CLI (Workstream A): - Modular pipeline: discover/extract/parse/chapters/render/store + ratelimit - SQLite store con migración idempotente: FTS5 (transcript search), columnas de metadata enriquecida, tablas cookies_meta y scrape_jobs - Módulos: segments, cookies (Netscape vault), export (json/csv/srt/html), analysis (word freq/timeline/wordcloud), monitor (watch loop), pipeline - CLI Click group: search, export, audio, channels, watch, analyze, re-render - Fix del bug de scoping de cookies en cli.py Webapp local (Workstream B): - FastAPI backend: dashboard, channels, videos facetado, transcript, search FTS, analysis, scrape jobs con SSE, cookies drag-and-drop, exports, folders (abrir en OS), tools (re-render, formato) - SPA no-build (Alpine.js + Tailwind + Chart.js por CDN): 9 vistas, tema dark command-center con acento rojo→rosa, cookie vault drag-drop, consola de scrapeo con progreso live vía SSE Launcher + subagentes (Workstream C): - start-server.bat / stop-server.bat con auto port-scan + browser open - .opencode/agent/webapp-builder.md + .opencode/goals/webapp-build.md Tests: 38 pytest verdes. Sin funcionalidad de IA (enfoque data-mining).
This commit is contained in:
urieljareth
@@ -0,0 +1,731 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS channels (
|
||||
channel_id TEXT PRIMARY KEY,
|
||||
handle TEXT,
|
||||
name TEXT,
|
||||
last_scraped TEXT,
|
||||
video_count INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS videos (
|
||||
video_id TEXT PRIMARY KEY,
|
||||
channel_id TEXT NOT NULL,
|
||||
title TEXT,
|
||||
url TEXT NOT NULL,
|
||||
upload_date TEXT,
|
||||
duration INTEGER,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
error_msg TEXT,
|
||||
transcript_lang TEXT,
|
||||
transcript_src TEXT,
|
||||
has_chapters INTEGER DEFAULT 0,
|
||||
markdown_path TEXT,
|
||||
discovered_at TEXT NOT NULL,
|
||||
processed_at TEXT,
|
||||
FOREIGN KEY (channel_id) REFERENCES channels(channel_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_videos_status ON videos(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_videos_channel ON videos(channel_id);
|
||||
"""
|
||||
|
||||
# New columns added by the platform migration (idempotent ALTERs)
|
||||
_VIDEO_COLUMNS: dict[str, str] = {
|
||||
"view_count": "INTEGER",
|
||||
"like_count": "INTEGER",
|
||||
"tags": "TEXT",
|
||||
"thumbnail": "TEXT",
|
||||
"description": "TEXT",
|
||||
"chapters_json": "TEXT",
|
||||
"segments_json": "TEXT",
|
||||
}
|
||||
|
||||
_EXTRA_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS transcript_segments (
|
||||
video_id TEXT NOT NULL,
|
||||
idx INTEGER NOT NULL,
|
||||
start_sec REAL NOT NULL,
|
||||
end_sec REAL NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
PRIMARY KEY (video_id, idx),
|
||||
FOREIGN KEY (video_id) REFERENCES videos(video_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_segments_video ON transcript_segments(video_id);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS transcript_fts USING fts5(
|
||||
video_id UNINDEXED, idx UNINDEXED, start_sec UNINDEXED, end_sec UNINDEXED, text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cookies_meta (
|
||||
id TEXT PRIMARY KEY,
|
||||
filename TEXT NOT NULL,
|
||||
label TEXT,
|
||||
added_at TEXT NOT NULL,
|
||||
expires_at TEXT,
|
||||
is_active INTEGER DEFAULT 0,
|
||||
has_session INTEGER DEFAULT 0,
|
||||
cookie_count INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scrape_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
channel_id TEXT,
|
||||
opts_json TEXT,
|
||||
status TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
total INTEGER DEFAULT 0,
|
||||
completed INTEGER DEFAULT 0,
|
||||
last_error TEXT
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoRef:
|
||||
video_id: str
|
||||
channel_id: str
|
||||
title: str
|
||||
url: str
|
||||
upload_date: str | None = None
|
||||
duration: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoRow:
|
||||
video_id: str
|
||||
channel_id: str
|
||||
title: str | None
|
||||
url: str
|
||||
upload_date: str | None
|
||||
duration: int | None
|
||||
status: str
|
||||
error_msg: str | None
|
||||
transcript_lang: str | None
|
||||
transcript_src: str | None
|
||||
has_chapters: int
|
||||
markdown_path: str | None
|
||||
view_count: int | None = None
|
||||
like_count: int | None = None
|
||||
tags: str | None = None
|
||||
thumbnail: str | None = None
|
||||
description: str | None = None
|
||||
chapters_json: str | None = None
|
||||
segments_json: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SegmentRow:
|
||||
video_id: str
|
||||
idx: int
|
||||
start_sec: float
|
||||
end_sec: float
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchHit:
|
||||
video_id: str
|
||||
channel_id: str
|
||||
title: str | None
|
||||
idx: int
|
||||
start_sec: float
|
||||
end_sec: float
|
||||
snippet: str
|
||||
rank: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class CookieRow:
|
||||
id: str
|
||||
filename: str
|
||||
label: str | None
|
||||
added_at: str
|
||||
expires_at: str | None
|
||||
is_active: bool
|
||||
has_session: bool
|
||||
cookie_count: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobRow:
|
||||
id: str
|
||||
channel_id: str | None
|
||||
opts_json: str | None
|
||||
status: str
|
||||
started_at: str | None
|
||||
finished_at: str | None
|
||||
total: int
|
||||
completed: int
|
||||
last_error: str | None
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, db_path: str | Path):
|
||||
self.db_path = Path(db_path)
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._init_schema()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(str(self.db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
return conn
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.executescript(SCHEMA)
|
||||
# idempotent column adds
|
||||
existing = {row["name"] for row in conn.execute("PRAGMA table_info(videos)")}
|
||||
for col, coltype in _VIDEO_COLUMNS.items():
|
||||
if col not in existing:
|
||||
conn.execute(f"ALTER TABLE videos ADD COLUMN {col} {coltype}")
|
||||
conn.executescript(_EXTRA_SCHEMA)
|
||||
|
||||
@contextmanager
|
||||
def _cursor(self) -> Iterator[sqlite3.Cursor]:
|
||||
conn = self._connect()
|
||||
try:
|
||||
yield conn.cursor()
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# ------------------------------------------------------------------ channels
|
||||
|
||||
def upsert_channel(self, channel_id: str, handle: str | None, name: str | None, video_count: int = 0) -> None:
|
||||
now = _now_iso()
|
||||
with self._cursor() as cur:
|
||||
cur.execute(
|
||||
"""INSERT INTO channels (channel_id, handle, name, last_scraped, video_count)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(channel_id) DO UPDATE SET
|
||||
handle = excluded.handle,
|
||||
name = excluded.name,
|
||||
last_scraped = excluded.last_scraped,
|
||||
video_count = excluded.video_count""",
|
||||
(channel_id, handle, name, now, video_count),
|
||||
)
|
||||
|
||||
def list_channels(self) -> list[dict]:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT * FROM channels ORDER BY name")
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
|
||||
def get_channel(self, channel_id: str) -> dict | None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT * FROM channels WHERE channel_id = ?", (channel_id,))
|
||||
row = cur.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def delete_channel(self, channel_id: str) -> None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("DELETE FROM transcript_segments WHERE video_id IN (SELECT video_id FROM videos WHERE channel_id = ?)", (channel_id,))
|
||||
cur.execute("DELETE FROM transcript_fts WHERE video_id IN (SELECT video_id FROM videos WHERE channel_id = ?)", (channel_id,))
|
||||
cur.execute("DELETE FROM videos WHERE channel_id = ?", (channel_id,))
|
||||
cur.execute("DELETE FROM channels WHERE channel_id = ?", (channel_id,))
|
||||
|
||||
# ------------------------------------------------------------------ videos
|
||||
|
||||
def upsert_videos(self, refs: list[VideoRef]) -> int:
|
||||
now = _now_iso()
|
||||
inserted = 0
|
||||
with self._cursor() as cur:
|
||||
for r in refs:
|
||||
cur.execute(
|
||||
"""INSERT INTO videos
|
||||
(video_id, channel_id, title, url, upload_date, duration, status, discovered_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'pending', ?)
|
||||
ON CONFLICT(video_id) DO UPDATE SET
|
||||
title = excluded.title,
|
||||
upload_date = excluded.upload_date,
|
||||
duration = excluded.duration""",
|
||||
(r.video_id, r.channel_id, r.title, r.url, r.upload_date, r.duration, now),
|
||||
)
|
||||
if cur.rowcount > 0:
|
||||
inserted += 1
|
||||
return inserted
|
||||
|
||||
def get_pending(self, channel_id: str | None = None, limit: int | None = None) -> list[VideoRow]:
|
||||
sql = "SELECT * FROM videos WHERE status = 'pending'"
|
||||
params: list[Any] = []
|
||||
if channel_id:
|
||||
sql += " AND channel_id = ?"
|
||||
params.append(channel_id)
|
||||
sql += " ORDER BY discovered_at ASC"
|
||||
if limit:
|
||||
sql += " LIMIT ?"
|
||||
params.append(limit)
|
||||
with self._cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return [_row_to_videorow(row) for row in cur.fetchall()]
|
||||
|
||||
def get_all(self, channel_id: str | None = None) -> list[VideoRow]:
|
||||
sql = "SELECT * FROM videos"
|
||||
params: list[Any] = []
|
||||
if channel_id:
|
||||
sql += " WHERE channel_id = ?"
|
||||
params.append(channel_id)
|
||||
sql += " ORDER BY discovered_at ASC"
|
||||
with self._cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return [_row_to_videorow(row) for row in cur.fetchall()]
|
||||
|
||||
def get_video(self, video_id: str) -> VideoRow | None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT * FROM videos WHERE video_id = ?", (video_id,))
|
||||
row = cur.fetchone()
|
||||
return _row_to_videorow(row) if row else None
|
||||
|
||||
def query_videos(
|
||||
self,
|
||||
channel_id: str | None = None,
|
||||
status: str | None = None,
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
min_duration: int | None = None,
|
||||
q: str | None = None,
|
||||
sort: str = "upload_date_desc",
|
||||
page: int = 1,
|
||||
size: int = 50,
|
||||
) -> tuple[list[VideoRow], int]:
|
||||
where: list[str] = []
|
||||
params: list[Any] = []
|
||||
if channel_id:
|
||||
where.append("channel_id = ?"); params.append(channel_id)
|
||||
if status:
|
||||
where.append("status = ?"); params.append(status)
|
||||
if date_from:
|
||||
where.append("upload_date >= ?"); params.append(date_from.replace("-", ""))
|
||||
if date_to:
|
||||
where.append("upload_date <= ?"); params.append(date_to.replace("-", ""))
|
||||
if min_duration is not None:
|
||||
where.append("duration >= ?"); params.append(min_duration)
|
||||
if q:
|
||||
where.append("(title LIKE ? OR description LIKE ?)")
|
||||
params.extend([f"%{q}%", f"%{q}%"])
|
||||
clause = ("WHERE " + " AND ".join(where)) if where else ""
|
||||
order = {
|
||||
"upload_date_desc": "upload_date DESC",
|
||||
"upload_date_asc": "upload_date ASC",
|
||||
"duration_desc": "duration DESC",
|
||||
"views_desc": "view_count DESC",
|
||||
"title_asc": "title ASC",
|
||||
}.get(sort, "upload_date DESC")
|
||||
offset = max(0, (page - 1) * size)
|
||||
with self._cursor() as cur:
|
||||
cur.execute(f"SELECT COUNT(*) AS n FROM videos {clause}", params)
|
||||
total = cur.fetchone()["n"]
|
||||
cur.execute(
|
||||
f"SELECT * FROM videos {clause} ORDER BY {order} LIMIT ? OFFSET ?",
|
||||
[*params, size, offset],
|
||||
)
|
||||
rows = [_row_to_videorow(r) for r in cur.fetchall()]
|
||||
return rows, total
|
||||
|
||||
def update_video_metadata(
|
||||
self,
|
||||
video_id: str,
|
||||
*,
|
||||
view_count: int | None = None,
|
||||
like_count: int | None = None,
|
||||
tags: list[str] | None = None,
|
||||
thumbnail: str | None = None,
|
||||
description: str | None = None,
|
||||
chapters_json: str | None = None,
|
||||
segments_json: str | None = None,
|
||||
) -> None:
|
||||
sets: list[str] = []
|
||||
params: list[Any] = []
|
||||
if view_count is not None:
|
||||
sets.append("view_count = ?"); params.append(view_count)
|
||||
if like_count is not None:
|
||||
sets.append("like_count = ?"); params.append(like_count)
|
||||
if tags is not None:
|
||||
sets.append("tags = ?"); params.append(json.dumps(tags, ensure_ascii=False))
|
||||
if thumbnail is not None:
|
||||
sets.append("thumbnail = ?"); params.append(thumbnail)
|
||||
if description is not None:
|
||||
sets.append("description = ?"); params.append(description[:4000])
|
||||
if chapters_json is not None:
|
||||
sets.append("chapters_json = ?"); params.append(chapters_json)
|
||||
if segments_json is not None:
|
||||
sets.append("segments_json = ?"); params.append(segments_json)
|
||||
if not sets:
|
||||
return
|
||||
params.append(video_id)
|
||||
with self._cursor() as cur:
|
||||
cur.execute(f"UPDATE videos SET {', '.join(sets)} WHERE video_id = ?", params)
|
||||
|
||||
def mark_done(
|
||||
self,
|
||||
video_id: str,
|
||||
markdown_path: str,
|
||||
transcript_lang: str | None,
|
||||
transcript_src: str | None,
|
||||
has_chapters: bool,
|
||||
) -> None:
|
||||
now = _now_iso()
|
||||
with self._cursor() as cur:
|
||||
cur.execute(
|
||||
"""UPDATE videos SET
|
||||
status = 'done',
|
||||
markdown_path = ?,
|
||||
transcript_lang = ?,
|
||||
transcript_src = ?,
|
||||
has_chapters = ?,
|
||||
processed_at = ?,
|
||||
error_msg = NULL
|
||||
WHERE video_id = ?""",
|
||||
(markdown_path, transcript_lang, transcript_src, int(has_chapters), now, video_id),
|
||||
)
|
||||
|
||||
def mark_status(self, video_id: str, status: str) -> None:
|
||||
now = _now_iso()
|
||||
with self._cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE videos SET status = ?, processed_at = ? WHERE video_id = ?",
|
||||
(status, now, video_id),
|
||||
)
|
||||
|
||||
def set_upload_date(self, video_id: str, upload_date: str | None) -> None:
|
||||
if not upload_date:
|
||||
return
|
||||
with self._cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE videos SET upload_date = ? WHERE video_id = ? AND upload_date IS NULL",
|
||||
(upload_date, video_id),
|
||||
)
|
||||
|
||||
def mark_error(self, video_id: str, error_msg: str) -> None:
|
||||
now = _now_iso()
|
||||
with self._cursor() as cur:
|
||||
cur.execute(
|
||||
"""UPDATE videos SET status = 'error', error_msg = ?, processed_at = ?
|
||||
WHERE video_id = ?""",
|
||||
(error_msg[:500], now, video_id),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ segments / FTS
|
||||
|
||||
def store_segments(self, video_id: str, segments: list) -> None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("DELETE FROM transcript_segments WHERE video_id = ?", (video_id,))
|
||||
cur.execute("DELETE FROM transcript_fts WHERE video_id = ?", (video_id,))
|
||||
rows = []
|
||||
fts_rows = []
|
||||
for i, seg in enumerate(segments):
|
||||
start = float(getattr(seg, "start", 0.0))
|
||||
end = float(getattr(seg, "end", start))
|
||||
text = (getattr(seg, "text", "") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
rows.append((video_id, i, start, end, text))
|
||||
fts_rows.append((video_id, i, start, end, text))
|
||||
if rows:
|
||||
cur.executemany(
|
||||
"INSERT INTO transcript_segments (video_id, idx, start_sec, end_sec, text) VALUES (?, ?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
cur.executemany(
|
||||
"INSERT INTO transcript_fts (video_id, idx, start_sec, end_sec, text) VALUES (?, ?, ?, ?, ?)",
|
||||
fts_rows,
|
||||
)
|
||||
|
||||
def get_segments(self, video_id: str) -> list[SegmentRow]:
|
||||
with self._cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT * FROM transcript_segments WHERE video_id = ? ORDER BY idx ASC",
|
||||
(video_id,),
|
||||
)
|
||||
return [
|
||||
SegmentRow(
|
||||
video_id=r["video_id"], idx=r["idx"], start_sec=r["start_sec"],
|
||||
end_sec=r["end_sec"], text=r["text"],
|
||||
)
|
||||
for r in cur.fetchall()
|
||||
]
|
||||
|
||||
def has_segments(self, video_id: str) -> bool:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT 1 FROM transcript_segments WHERE video_id = ? LIMIT 1", (video_id,))
|
||||
return cur.fetchone() is not None
|
||||
|
||||
def search_segments(self, query: str, channel_id: str | None = None, limit: int = 50) -> list[SearchHit]:
|
||||
# FTS5 MATCH; join videos for title + optional channel filter.
|
||||
fts_query = _sanitize_fts(query)
|
||||
if not fts_query:
|
||||
return []
|
||||
sql = (
|
||||
"SELECT f.video_id, f.idx, f.start_sec, f.end_sec, f.text, v.channel_id, v.title, bm25(transcript_fts) AS rank "
|
||||
"FROM transcript_fts f JOIN videos v ON v.video_id = f.video_id "
|
||||
"WHERE transcript_fts MATCH ?"
|
||||
)
|
||||
params: list[Any] = [fts_query]
|
||||
if channel_id:
|
||||
sql += " AND v.channel_id = ?"
|
||||
params.append(channel_id)
|
||||
sql += " ORDER BY rank LIMIT ?"
|
||||
params.append(limit)
|
||||
with self._cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return [
|
||||
SearchHit(
|
||||
video_id=r["video_id"], channel_id=r["channel_id"], title=r["title"],
|
||||
idx=r["idx"], start_sec=r["start_sec"], end_sec=r["end_sec"],
|
||||
snippet=r["text"], rank=r["rank"],
|
||||
)
|
||||
for r in cur.fetchall()
|
||||
]
|
||||
|
||||
# ------------------------------------------------------------------ cookies
|
||||
|
||||
def upsert_cookie(self, cookie_id: str, filename: str, label: str | None, expires_at: str | None, has_session: bool, cookie_count: int) -> None:
|
||||
now = _now_iso()
|
||||
with self._cursor() as cur:
|
||||
cur.execute(
|
||||
"""INSERT INTO cookies_meta (id, filename, label, added_at, expires_at, is_active, has_session, cookie_count)
|
||||
VALUES (?, ?, ?, ?, ?, 0, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
filename = excluded.filename,
|
||||
label = excluded.label,
|
||||
expires_at = excluded.expires_at,
|
||||
has_session = excluded.has_session,
|
||||
cookie_count = excluded.cookie_count""",
|
||||
(cookie_id, filename, label, now, expires_at, int(has_session), cookie_count),
|
||||
)
|
||||
|
||||
def list_cookies(self) -> list[CookieRow]:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT * FROM cookies_meta ORDER BY added_at DESC")
|
||||
return [_row_to_cookierow(r) for r in cur.fetchall()]
|
||||
|
||||
def get_cookie(self, cookie_id: str) -> CookieRow | None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT * FROM cookies_meta WHERE id = ?", (cookie_id,))
|
||||
r = cur.fetchone()
|
||||
return _row_to_cookierow(r) if r else None
|
||||
|
||||
def set_active_cookie(self, cookie_id: str | None) -> None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("UPDATE cookies_meta SET is_active = 0")
|
||||
if cookie_id:
|
||||
cur.execute("UPDATE cookies_meta SET is_active = 1 WHERE id = ?", (cookie_id,))
|
||||
|
||||
def get_active_cookie(self) -> CookieRow | None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT * FROM cookies_meta WHERE is_active = 1 LIMIT 1")
|
||||
r = cur.fetchone()
|
||||
return _row_to_cookierow(r) if r else None
|
||||
|
||||
def delete_cookie(self, cookie_id: str) -> None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("DELETE FROM cookies_meta WHERE id = ?", (cookie_id,))
|
||||
|
||||
# ------------------------------------------------------------------ jobs
|
||||
|
||||
def create_job(self, job_id: str, channel_id: str | None, opts: dict) -> None:
|
||||
now = _now_iso()
|
||||
with self._cursor() as cur:
|
||||
cur.execute(
|
||||
"""INSERT INTO scrape_jobs (id, channel_id, opts_json, status, started_at, total, completed)
|
||||
VALUES (?, ?, ?, 'queued', ?, 0, 0)""",
|
||||
(job_id, channel_id, json.dumps(opts, ensure_ascii=False), now),
|
||||
)
|
||||
|
||||
def update_job(self, job_id: str, *, status: str | None = None, total: int | None = None, completed: int | None = None, last_error: str | None = None, finished: bool = False) -> None:
|
||||
sets: list[str] = []
|
||||
params: list[Any] = []
|
||||
if status:
|
||||
sets.append("status = ?"); params.append(status)
|
||||
if total is not None:
|
||||
sets.append("total = ?"); params.append(total)
|
||||
if completed is not None:
|
||||
sets.append("completed = ?"); params.append(completed)
|
||||
if last_error is not None:
|
||||
sets.append("last_error = ?"); params.append(last_error[:500])
|
||||
if finished:
|
||||
sets.append("finished_at = ?"); params.append(_now_iso())
|
||||
if not sets:
|
||||
return
|
||||
params.append(job_id)
|
||||
with self._cursor() as cur:
|
||||
cur.execute(f"UPDATE scrape_jobs SET {', '.join(sets)} WHERE id = ?", params)
|
||||
|
||||
def get_job(self, job_id: str) -> JobRow | None:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT * FROM scrape_jobs WHERE id = ?", (job_id,))
|
||||
r = cur.fetchone()
|
||||
return _row_to_jobrow(r) if r else None
|
||||
|
||||
def list_jobs(self, limit: int = 50) -> list[JobRow]:
|
||||
with self._cursor() as cur:
|
||||
cur.execute("SELECT * FROM scrape_jobs ORDER BY started_at DESC LIMIT ?", (limit,))
|
||||
return [_row_to_jobrow(r) for r in cur.fetchall()]
|
||||
|
||||
# ------------------------------------------------------------------ aggregates
|
||||
|
||||
def stats(self, channel_id: str | None = None) -> dict[str, int]:
|
||||
sql = "SELECT status, COUNT(*) as n FROM videos"
|
||||
params: list[Any] = []
|
||||
if channel_id:
|
||||
sql += " WHERE channel_id = ?"
|
||||
params.append(channel_id)
|
||||
sql += " GROUP BY status"
|
||||
result: dict[str, int] = {}
|
||||
with self._cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
for row in cur.fetchall():
|
||||
result[row["status"]] = row["n"]
|
||||
return result
|
||||
|
||||
def dashboard(self) -> dict:
|
||||
with self._cursor() as cur:
|
||||
channels = [dict(r) for r in cur.execute("SELECT * FROM channels ORDER BY name").fetchall()]
|
||||
for ch in channels:
|
||||
cid = ch["channel_id"]
|
||||
row = cur.execute(
|
||||
"SELECT COUNT(*) AS n, COALESCE(SUM(duration),0) AS dur, MIN(upload_date) AS mind, MAX(upload_date) AS maxd, COALESCE(SUM(view_count),0) AS views, COALESCE(SUM(like_count),0) AS likes FROM videos WHERE channel_id = ?",
|
||||
(cid,),
|
||||
).fetchone()
|
||||
ch["video_count_db"] = row["n"]
|
||||
ch["total_duration"] = row["dur"]
|
||||
ch["date_min"] = row["mind"]
|
||||
ch["date_max"] = row["maxd"]
|
||||
ch["total_views"] = row["views"]
|
||||
ch["total_likes"] = row["likes"]
|
||||
status_breakdown = {
|
||||
r["status"]: r["n"]
|
||||
for r in cur.execute("SELECT status, COUNT(*) AS n FROM videos GROUP BY status").fetchall()
|
||||
}
|
||||
# uploads over time (by month)
|
||||
uploads = [
|
||||
{"month": r["m"], "count": r["n"]}
|
||||
for r in cur.execute(
|
||||
"SELECT substr(upload_date,1,6) AS m, COUNT(*) AS n FROM videos WHERE upload_date IS NOT NULL GROUP BY m ORDER BY m"
|
||||
).fetchall()
|
||||
]
|
||||
# duration histogram (buckets)
|
||||
hist = [
|
||||
{"bucket": r["b"], "count": r["n"]}
|
||||
for r in cur.execute(
|
||||
"""SELECT
|
||||
CASE WHEN duration < 300 THEN '<5m'
|
||||
WHEN duration < 600 THEN '5-10m'
|
||||
WHEN duration < 1200 THEN '10-20m'
|
||||
WHEN duration < 2400 THEN '20-40m'
|
||||
ELSE '40m+' END AS b,
|
||||
COUNT(*) AS n
|
||||
FROM videos WHERE duration IS NOT NULL GROUP BY b"""
|
||||
).fetchall()
|
||||
]
|
||||
# top tags (tags is JSON array text)
|
||||
tag_rows = cur.execute("SELECT tags FROM videos WHERE tags IS NOT NULL AND tags != '[]'").fetchall()
|
||||
tag_counts: dict[str, int] = {}
|
||||
for tr in tag_rows:
|
||||
try:
|
||||
for t in json.loads(tr["tags"]):
|
||||
t = (t or "").strip().lower()
|
||||
if t:
|
||||
tag_counts[t] = tag_counts.get(t, 0) + 1
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
top_tags = sorted(tag_counts.items(), key=lambda x: x[1], reverse=True)[:20]
|
||||
return {
|
||||
"channels": channels,
|
||||
"status_breakdown": status_breakdown,
|
||||
"uploads_over_time": uploads,
|
||||
"duration_histogram": hist,
|
||||
"top_tags": [{"tag": t, "count": c} for t, c in top_tags],
|
||||
}
|
||||
|
||||
def reset_errors(self, channel_id: str | None = None) -> int:
|
||||
sql = "UPDATE videos SET status = 'pending', error_msg = NULL WHERE status = 'error'"
|
||||
params: list[Any] = []
|
||||
if channel_id:
|
||||
sql += " AND channel_id = ?"
|
||||
params.append(channel_id)
|
||||
with self._cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
def _sanitize_fts(query: str) -> str:
|
||||
# Build a safe AND FTS5 query from whitespace-separated terms.
|
||||
cleaned = (query or "").strip()
|
||||
if not cleaned:
|
||||
return ""
|
||||
terms = []
|
||||
for tok in cleaned.split():
|
||||
tok = tok.strip('"')
|
||||
if tok:
|
||||
terms.append(f'"{tok.replace("\"", "")}"')
|
||||
return " AND ".join(terms)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _row_to_videorow(row: sqlite3.Row) -> VideoRow:
|
||||
keys = row.keys()
|
||||
return VideoRow(
|
||||
video_id=row["video_id"],
|
||||
channel_id=row["channel_id"],
|
||||
title=row["title"],
|
||||
url=row["url"],
|
||||
upload_date=row["upload_date"],
|
||||
duration=row["duration"],
|
||||
status=row["status"],
|
||||
error_msg=row["error_msg"],
|
||||
transcript_lang=row["transcript_lang"],
|
||||
transcript_src=row["transcript_src"],
|
||||
has_chapters=row["has_chapters"],
|
||||
markdown_path=row["markdown_path"],
|
||||
view_count=row["view_count"] if "view_count" in keys else None,
|
||||
like_count=row["like_count"] if "like_count" in keys else None,
|
||||
tags=row["tags"] if "tags" in keys else None,
|
||||
thumbnail=row["thumbnail"] if "thumbnail" in keys else None,
|
||||
description=row["description"] if "description" in keys else None,
|
||||
chapters_json=row["chapters_json"] if "chapters_json" in keys else None,
|
||||
segments_json=row["segments_json"] if "segments_json" in keys else None,
|
||||
)
|
||||
|
||||
|
||||
def _row_to_cookierow(row: sqlite3.Row) -> CookieRow:
|
||||
return CookieRow(
|
||||
id=row["id"],
|
||||
filename=row["filename"],
|
||||
label=row["label"],
|
||||
added_at=row["added_at"],
|
||||
expires_at=row["expires_at"],
|
||||
is_active=bool(row["is_active"]),
|
||||
has_session=bool(row["has_session"]),
|
||||
cookie_count=row["cookie_count"],
|
||||
)
|
||||
|
||||
|
||||
def _row_to_jobrow(row: sqlite3.Row) -> JobRow:
|
||||
return JobRow(
|
||||
id=row["id"],
|
||||
channel_id=row["channel_id"],
|
||||
opts_json=row["opts_json"],
|
||||
status=row["status"],
|
||||
started_at=row["started_at"],
|
||||
finished_at=row["finished_at"],
|
||||
total=row["total"],
|
||||
completed=row["completed"],
|
||||
last_error=row["last_error"],
|
||||
)
|
||||
Reference in New Issue
Block a user