urieljareth
621bbc5f5c
Scraper de canales de YouTube hacia notas Markdown para base de conocimiento (Obsidian-ready), con plataforma web local. Engine + CLI (Workstream A): - Modular pipeline: discover/extract/parse/chapters/render/store + ratelimit - SQLite store con migración idempotente: FTS5 (transcript search), columnas de metadata enriquecida, tablas cookies_meta y scrape_jobs - Módulos: segments, cookies (Netscape vault), export (json/csv/srt/html), analysis (word freq/timeline/wordcloud), monitor (watch loop), pipeline - CLI Click group: search, export, audio, channels, watch, analyze, re-render - Fix del bug de scoping de cookies en cli.py Webapp local (Workstream B): - FastAPI backend: dashboard, channels, videos facetado, transcript, search FTS, analysis, scrape jobs con SSE, cookies drag-and-drop, exports, folders (abrir en OS), tools (re-render, formato) - SPA no-build (Alpine.js + Tailwind + Chart.js por CDN): 9 vistas, tema dark command-center con acento rojo→rosa, cookie vault drag-drop, consola de scrapeo con progreso live vía SSE Launcher + subagentes (Workstream C): - start-server.bat / stop-server.bat con auto port-scan + browser open - .opencode/agent/webapp-builder.md + .opencode/goals/webapp-build.md Tests: 38 pytest verdes. Sin funcionalidad de IA (enfoque data-mining).
103 lines
2.9 KiB
Python
103 lines
2.9 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from typing import Callable
|
|
|
|
from .config import Config
|
|
from .cookies import resolve_active_path
|
|
from .discover import discover_channel
|
|
from .pipeline import process_video
|
|
from .ratelimit import polite_sleep
|
|
from .store import Store
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def watch_loop(
|
|
cfg: Config,
|
|
store: Store,
|
|
interval_seconds: float,
|
|
channel_id: str | None = None,
|
|
once: bool = False,
|
|
cookies_file: str | None = None,
|
|
cookies_from_browser: str | None = None,
|
|
on_log: Callable[[str], None] | None = None,
|
|
) -> None:
|
|
"""Discover + process pending videos for a channel, optionally on an interval."""
|
|
|
|
def _emit(msg: str) -> None:
|
|
log.info(msg)
|
|
if on_log:
|
|
on_log(msg)
|
|
|
|
while True:
|
|
if not cfg.channel_url and not channel_id:
|
|
_emit("watch: no channel_url configured; sleeping")
|
|
if once:
|
|
return
|
|
time.sleep(interval_seconds)
|
|
continue
|
|
|
|
try:
|
|
_run_once(cfg, store, channel_id, cookies_file, cookies_from_browser, on_log)
|
|
except Exception as exc:
|
|
_emit(f"watch: iteration error: {exc}")
|
|
log.exception("watch iteration failed")
|
|
|
|
if once:
|
|
return
|
|
_emit(f"watch: sleeping {interval_seconds}s")
|
|
time.sleep(interval_seconds)
|
|
|
|
|
|
def _run_once(
|
|
cfg: Config,
|
|
store: Store,
|
|
channel_id: str | None,
|
|
cookies_file: str | None,
|
|
cookies_from_browser: str | None,
|
|
on_log: Callable[[str], None] | None,
|
|
) -> None:
|
|
def _emit(msg: str) -> None:
|
|
log.info(msg)
|
|
if on_log:
|
|
on_log(msg)
|
|
|
|
channel_id_found, channel_name, refs = discover_channel(
|
|
cfg.channel_url, sleep_subrequests=cfg.yt_dlp.sleep_subrequests
|
|
)
|
|
target_channel = channel_id or channel_id_found
|
|
store.upsert_channel(target_channel, _extract_handle(cfg.channel_url), channel_name, len(refs))
|
|
store.upsert_videos(refs)
|
|
_emit(f"watch: discovered {len(refs)} videos on {channel_name}")
|
|
|
|
pending = store.get_pending(target_channel)
|
|
if not pending:
|
|
_emit("watch: nothing pending")
|
|
return
|
|
|
|
# fallback to active vault cookie if no explicit cookie given
|
|
cookie_path = cookies_file
|
|
if not cookie_path and not cookies_from_browser:
|
|
vault = resolve_active_path(store)
|
|
if vault:
|
|
cookie_path = vault
|
|
_emit(f"watch: using active vault cookie {vault}")
|
|
|
|
for row in pending:
|
|
process_video(
|
|
row, cfg, store, channel_name, target_channel, cfg.channel_url,
|
|
cookies_file=cookie_path,
|
|
cookies_from_browser=cookies_from_browser,
|
|
on_log=on_log,
|
|
)
|
|
polite_sleep(cfg.delay.min_seconds, cfg.delay.max_seconds)
|
|
|
|
|
|
def _extract_handle(url: str) -> str:
|
|
if "@" in url:
|
|
return "@" + url.split("@", 1)[1].split("/", 1)[0]
|
|
return ""
|