feat: local content-mining platform for YouTube creator scraping
Scraper de canales de YouTube hacia notas Markdown para base de conocimiento (Obsidian-ready), con plataforma web local. Engine + CLI (Workstream A): - Modular pipeline: discover/extract/parse/chapters/render/store + ratelimit - SQLite store con migración idempotente: FTS5 (transcript search), columnas de metadata enriquecida, tablas cookies_meta y scrape_jobs - Módulos: segments, cookies (Netscape vault), export (json/csv/srt/html), analysis (word freq/timeline/wordcloud), monitor (watch loop), pipeline - CLI Click group: search, export, audio, channels, watch, analyze, re-render - Fix del bug de scoping de cookies en cli.py Webapp local (Workstream B): - FastAPI backend: dashboard, channels, videos facetado, transcript, search FTS, analysis, scrape jobs con SSE, cookies drag-and-drop, exports, folders (abrir en OS), tools (re-render, formato) - SPA no-build (Alpine.js + Tailwind + Chart.js por CDN): 9 vistas, tema dark command-center con acento rojo→rosa, cookie vault drag-drop, consola de scrapeo con progreso live vía SSE Launcher + subagentes (Workstream C): - start-server.bat / stop-server.bat con auto port-scan + browser open - .opencode/agent/webapp-builder.md + .opencode/goals/webapp-build.md Tests: 38 pytest verdes. Sin funcionalidad de IA (enfoque data-mining).
This commit is contained in:
urieljareth
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from typing import Any
|
||||
|
||||
from ..config import Config, load_config
|
||||
from ..cookies import resolve_active_path
|
||||
from ..discover import discover_channel
|
||||
from ..pipeline import process_video
|
||||
from ..ratelimit import polite_sleep
|
||||
from ..store import Store
|
||||
|
||||
|
||||
class JobManager:
|
||||
"""Single-worker scrape job runner with an in-memory event log per job (SSE-polled)."""
|
||||
|
||||
def __init__(self, store: Store, cfg: Config):
|
||||
self.store = store
|
||||
self.cfg = cfg
|
||||
self._lock = threading.Lock()
|
||||
self._events: dict[str, list[dict]] = {}
|
||||
self._cancel: set[str] = set()
|
||||
self._queue: deque[str] = deque()
|
||||
self._thread = threading.Thread(target=self._worker, daemon=True, name="scrape-worker")
|
||||
self._running = False
|
||||
|
||||
def start(self) -> None:
|
||||
if self._running:
|
||||
return
|
||||
self._running = True
|
||||
self._thread.start()
|
||||
|
||||
def enqueue(self, channel_id: str | None, opts: dict[str, Any]) -> str:
|
||||
job_id = uuid.uuid4().hex[:12]
|
||||
self.store.create_job(job_id, channel_id, opts)
|
||||
with self._lock:
|
||||
self._events[job_id] = []
|
||||
self._emit(job_id, "queued", {"job_id": job_id, "channel_id": channel_id, "opts": opts})
|
||||
self._queue.append(job_id)
|
||||
return job_id
|
||||
|
||||
def cancel(self, job_id: str) -> bool:
|
||||
job = self.store.get_job(job_id)
|
||||
if not job or job.status not in ("queued", "running"):
|
||||
return False
|
||||
self._cancel.add(job_id)
|
||||
self._emit(job_id, "log", {"msg": "cancel requested"})
|
||||
return True
|
||||
|
||||
def events_since(self, job_id: str, cursor: int) -> list[dict]:
|
||||
with self._lock:
|
||||
evs = self._events.get(job_id, [])
|
||||
return evs[cursor:]
|
||||
|
||||
def is_terminal(self, job_id: str) -> str | None:
|
||||
job = self.store.get_job(job_id)
|
||||
if not job:
|
||||
return None
|
||||
if job.status in ("done", "error", "cancelled"):
|
||||
return job.status
|
||||
return None
|
||||
|
||||
# ---------------------------------------------------------- internals
|
||||
|
||||
def _emit(self, job_id: str, event: str, data: dict) -> None:
|
||||
with self._lock:
|
||||
self._events.setdefault(job_id, []).append({"event": event, "data": data})
|
||||
|
||||
def _worker(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
job_id = self._queue.popleft()
|
||||
except IndexError:
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
try:
|
||||
self._run_job(job_id)
|
||||
except Exception as exc:
|
||||
self.store.update_job(job_id, status="error", last_error=str(exc), finished=True)
|
||||
self._emit(job_id, "error", {"message": str(exc)})
|
||||
|
||||
def _run_job(self, job_id: str) -> None:
|
||||
job = self.store.get_job(job_id)
|
||||
if not job:
|
||||
return
|
||||
opts = json.loads(job.opts_json) if job.opts_json else {}
|
||||
channel_id = job.channel_id
|
||||
cfg = _clone_config(self.cfg)
|
||||
channel_url = _resolve_channel_url(self.store, cfg, channel_id)
|
||||
if not channel_url:
|
||||
self.store.update_job(job_id, status="error", last_error="no channel url", finished=True)
|
||||
self._emit(job_id, "error", {"message": "no channel url"})
|
||||
return
|
||||
cfg.channel_url = channel_url
|
||||
|
||||
self.store.update_job(job_id, status="running")
|
||||
self._emit(job_id, "log", {"msg": f"discovering {channel_url}"})
|
||||
try:
|
||||
_channel_id, channel_name, refs = discover_channel(channel_url, sleep_subrequests=cfg.yt_dlp.sleep_subrequests)
|
||||
except Exception as exc:
|
||||
self.store.update_job(job_id, status="error", last_error=str(exc), finished=True)
|
||||
self._emit(job_id, "error", {"message": f"discovery failed: {exc}"})
|
||||
return
|
||||
|
||||
limit = opts.get("limit")
|
||||
since = opts.get("since")
|
||||
languages = opts.get("languages")
|
||||
include_shorts = opts.get("include_shorts", cfg.include_shorts)
|
||||
no_live = opts.get("no_live", not cfg.include_live)
|
||||
cookie_override = opts.get("cookies_file")
|
||||
|
||||
if languages:
|
||||
cfg.languages = languages
|
||||
if since:
|
||||
refs = [r for r in refs if (r.upload_date or "") >= since.replace("-", "")]
|
||||
if not include_shorts:
|
||||
refs = [r for r in refs if "/shorts/" not in (r.url or "")]
|
||||
if no_live:
|
||||
refs = [r for r in refs if not (r.url or "").startswith("https://www.youtube.com/live/")]
|
||||
if limit:
|
||||
refs = refs[: int(limit)]
|
||||
|
||||
self.store.upsert_channel(_channel_id, _handle(channel_url), channel_name, len(refs))
|
||||
self.store.upsert_videos(refs)
|
||||
|
||||
pending = [r for r in self.store.get_pending(_channel_id) if r.video_id in {x.video_id for x in refs}]
|
||||
total = len(pending)
|
||||
self.store.update_job(job_id, total=total)
|
||||
self._emit(job_id, "progress", {"completed": 0, "total": total})
|
||||
self._emit(job_id, "log", {"msg": f"{total} videos to process"})
|
||||
|
||||
cookie_path = cookie_override or resolve_active_path(self.store)
|
||||
completed = 0
|
||||
for row in pending:
|
||||
if job_id in self._cancel:
|
||||
self.store.update_job(job_id, status="cancelled", completed=completed, finished=True)
|
||||
self._emit(job_id, "cancelled", {"completed": completed, "total": total})
|
||||
return
|
||||
status = process_video(
|
||||
row, cfg, self.store, channel_name, _channel_id, channel_url,
|
||||
cookies_file=cookie_path,
|
||||
on_log=lambda m: self._emit(job_id, "log", {"msg": m}),
|
||||
)
|
||||
completed += 1
|
||||
self.store.update_job(job_id, completed=completed)
|
||||
self._emit(job_id, "progress", {"completed": completed, "total": total, "video_id": row.video_id, "status": status})
|
||||
if completed < total:
|
||||
polite_sleep(cfg.delay.min_seconds, cfg.delay.max_seconds)
|
||||
|
||||
self.store.update_job(job_id, status="done", completed=completed, finished=True)
|
||||
self._emit(job_id, "done", {"completed": completed, "total": total})
|
||||
|
||||
|
||||
def _clone_config(cfg: Config) -> Config:
|
||||
import copy
|
||||
return copy.deepcopy(cfg)
|
||||
|
||||
|
||||
def _resolve_channel_url(store: Store, cfg: Config, channel_id: str | None) -> str | None:
|
||||
if not channel_id:
|
||||
return cfg.channel_url or None
|
||||
ch = store.get_channel(channel_id)
|
||||
if not ch:
|
||||
return None
|
||||
handle = (ch.get("handle") or "").lstrip("@")
|
||||
if handle:
|
||||
return f"https://www.youtube.com/@{handle}/videos"
|
||||
return f"https://www.youtube.com/channel/{channel_id}"
|
||||
|
||||
|
||||
def _handle(url: str) -> str:
|
||||
if "@" in url:
|
||||
return "@" + url.split("@", 1)[1].split("/", 1)[0]
|
||||
return ""
|
||||
Reference in New Issue
Block a user