feat: local content-mining platform for YouTube creator scraping
Scraper de canales de YouTube hacia notas Markdown para base de conocimiento (Obsidian-ready), con plataforma web local. Engine + CLI (Workstream A): - Modular pipeline: discover/extract/parse/chapters/render/store + ratelimit - SQLite store con migración idempotente: FTS5 (transcript search), columnas de metadata enriquecida, tablas cookies_meta y scrape_jobs - Módulos: segments, cookies (Netscape vault), export (json/csv/srt/html), analysis (word freq/timeline/wordcloud), monitor (watch loop), pipeline - CLI Click group: search, export, audio, channels, watch, analyze, re-render - Fix del bug de scoping de cookies en cli.py Webapp local (Workstream B): - FastAPI backend: dashboard, channels, videos facetado, transcript, search FTS, analysis, scrape jobs con SSE, cookies drag-and-drop, exports, folders (abrir en OS), tools (re-render, formato) - SPA no-build (Alpine.js + Tailwind + Chart.js por CDN): 9 vistas, tema dark command-center con acento rojo→rosa, cookie vault drag-drop, consola de scrapeo con progreso live vía SSE Launcher + subagentes (Workstream C): - start-server.bat / stop-server.bat con auto port-scan + browser open - .opencode/agent/webapp-builder.md + .opencode/goals/webapp-build.md Tests: 38 pytest verdes. Sin funcionalidad de IA (enfoque data-mining).
This commit is contained in:
urieljareth
@@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from .chapters import align_chapters, chapters_from_info
|
||||
from .config import Config
|
||||
from .extract import extract_video
|
||||
from .render import build_filename_stem, render_markdown
|
||||
from .store import Store, VideoRow
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def process_video(
|
||||
row: VideoRow,
|
||||
cfg: Config,
|
||||
store: Store,
|
||||
channel_name: str,
|
||||
channel_id: str,
|
||||
channel_url: str,
|
||||
cookies_file: str | None = None,
|
||||
cookies_from_browser: str | None = None,
|
||||
on_log: Callable[[str], None] | None = None,
|
||||
) -> str:
|
||||
"""Extract + parse + store + render a single video. Returns final status string."""
|
||||
|
||||
def _emit(msg: str) -> None:
|
||||
log.info(msg)
|
||||
if on_log:
|
||||
on_log(msg)
|
||||
|
||||
try:
|
||||
data = extract_video(
|
||||
row.url,
|
||||
cfg.languages,
|
||||
retries=cfg.yt_dlp.retries,
|
||||
sleep_subrequests=cfg.yt_dlp.sleep_subrequests,
|
||||
prefer_manual=cfg.prefer_manual,
|
||||
cookies_file=cookies_file,
|
||||
cookies_from_browser=cookies_from_browser,
|
||||
)
|
||||
except Exception as exc:
|
||||
log.error("Error extrayendo %s: %s", row.video_id, exc)
|
||||
store.mark_error(row.video_id, str(exc))
|
||||
return "error"
|
||||
|
||||
if not data.segments:
|
||||
log.warning("Sin transcripcion para %s", row.video_id)
|
||||
store.mark_status(row.video_id, "no_subtitles")
|
||||
return "no_subtitles"
|
||||
|
||||
chapters = chapters_from_info(data.info)
|
||||
sections = align_chapters(data.segments, chapters)
|
||||
|
||||
info = data.info
|
||||
|
||||
# persist segments + rich metadata to DB (for search, stats, webapp)
|
||||
store.store_segments(row.video_id, data.segments)
|
||||
seg_json = json.dumps(
|
||||
[{"start": s.start, "end": s.end, "text": s.text} for s in data.segments],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
ch_json = json.dumps(
|
||||
[{"title": c.title, "start": c.start_time, "end": c.end_time} for c in chapters],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
store.update_video_metadata(
|
||||
row.video_id,
|
||||
view_count=info.get("view_count"),
|
||||
like_count=info.get("like_count"),
|
||||
tags=info.get("tags") or None,
|
||||
thumbnail=info.get("thumbnail"),
|
||||
description=(info.get("description") or "").strip() or None,
|
||||
chapters_json=ch_json,
|
||||
segments_json=seg_json,
|
||||
)
|
||||
|
||||
# ensure upload_date is populated (discovery sometimes lacks it)
|
||||
ud = info.get("upload_date") or row.upload_date
|
||||
if ud:
|
||||
store.set_upload_date(row.video_id, ud.replace("-", "") if "-" in str(ud) else str(ud))
|
||||
|
||||
context = {
|
||||
"video_id": row.video_id,
|
||||
"title": info.get("title") or row.title or row.video_id,
|
||||
"channel_name": info.get("channel") or channel_name,
|
||||
"channel_id": channel_id,
|
||||
"channel_url": channel_url,
|
||||
"upload_date": _normalize_date(info.get("upload_date") or row.upload_date),
|
||||
"duration": info.get("duration") or row.duration or 0,
|
||||
"url": row.url,
|
||||
"transcript_lang": data.subtitle.lang if data.subtitle else "",
|
||||
"transcript_src": data.subtitle.source if data.subtitle else "",
|
||||
"view_count": info.get("view_count"),
|
||||
"like_count": info.get("like_count"),
|
||||
"tags": info.get("tags") or [],
|
||||
"thumbnail": info.get("thumbnail") or "",
|
||||
"description": (info.get("description") or "").strip(),
|
||||
"sections": sections,
|
||||
}
|
||||
|
||||
stem = build_filename_stem(
|
||||
upload_date=context["upload_date"],
|
||||
title=context["title"],
|
||||
template=cfg.filename_template,
|
||||
)
|
||||
out_subdir = Path(cfg.output_dir_resolved) / _safe_dirname(channel_name)
|
||||
md_path = render_markdown(cfg.template_path_resolved, out_subdir, stem, context)
|
||||
|
||||
out_root = Path(cfg.output_dir_resolved).parent
|
||||
try:
|
||||
rel = md_path.relative_to(out_root) if md_path.is_relative_to(out_root) else md_path
|
||||
except ValueError:
|
||||
rel = md_path
|
||||
store.mark_done(
|
||||
row.video_id,
|
||||
str(rel),
|
||||
data.subtitle.lang if data.subtitle else None,
|
||||
data.subtitle.source if data.subtitle else None,
|
||||
data.has_chapters,
|
||||
)
|
||||
_emit(f"OK {row.video_id} -> {md_path.name}")
|
||||
return "done"
|
||||
|
||||
|
||||
def _normalize_date(d: str | None) -> str:
|
||||
if not d:
|
||||
return ""
|
||||
if len(d) == 8:
|
||||
return f"{d[:4]}-{d[4:6]}-{d[6:8]}"
|
||||
return d
|
||||
|
||||
|
||||
def _safe_dirname(name: str) -> str:
|
||||
safe = "".join(c for c in name if c not in r'\/:*?"<>|')
|
||||
return safe.strip().strip(".") or "unknown"
|
||||
|
||||
|
||||
def re_render_videos(store: Store, cfg: Config, channel_id: str | None = None) -> int:
|
||||
"""Regenerate .md for done videos that have stored segments. Returns count re-rendered."""
|
||||
import json
|
||||
from .chapters import align_chapters, Chapter
|
||||
from .parse import Segment
|
||||
from .render import build_filename_stem, render_markdown
|
||||
|
||||
videos = [v for v in store.get_all(channel_id) if v.status == "done" and v.segments_json]
|
||||
n = 0
|
||||
for v in videos:
|
||||
try:
|
||||
segs = [Segment(start=s["start"], end=s["end"], text=s["text"]) for s in json.loads(v.segments_json)]
|
||||
chapters = [
|
||||
Chapter(title=c["title"], start_time=c["start"], end_time=c.get("end", c["start"]))
|
||||
for c in json.loads(v.chapters_json or "[]")
|
||||
]
|
||||
sections = align_chapters(segs, chapters)
|
||||
ch = store.get_channel(v.channel_id) or {}
|
||||
tags = []
|
||||
if v.tags:
|
||||
try:
|
||||
tags = json.loads(v.tags)
|
||||
except Exception:
|
||||
tags = []
|
||||
context = {
|
||||
"video_id": v.video_id, "title": v.title or v.video_id,
|
||||
"channel_name": ch.get("name") or "", "channel_id": v.channel_id, "channel_url": "",
|
||||
"upload_date": v.upload_date or "", "duration": v.duration or 0, "url": v.url,
|
||||
"transcript_lang": v.transcript_lang or "", "transcript_src": v.transcript_src or "",
|
||||
"view_count": v.view_count, "like_count": v.like_count, "tags": tags,
|
||||
"thumbnail": v.thumbnail or "", "description": v.description or "", "sections": sections,
|
||||
}
|
||||
stem = build_filename_stem(v.upload_date, v.title or v.video_id, cfg.filename_template)
|
||||
out_subdir = Path(cfg.output_dir_resolved) / _safe_dirname(ch.get("name") or "unknown")
|
||||
render_markdown(cfg.template_path_resolved, out_subdir, stem, context)
|
||||
n += 1
|
||||
except Exception as exc:
|
||||
log.warning("re-render failed for %s: %s", v.video_id, exc)
|
||||
return n
|
||||
Reference in New Issue
Block a user