feat: local content-mining platform for YouTube creator scraping
Scraper de canales de YouTube hacia notas Markdown para base de conocimiento (Obsidian-ready), con plataforma web local. Engine + CLI (Workstream A): - Modular pipeline: discover/extract/parse/chapters/render/store + ratelimit - SQLite store con migración idempotente: FTS5 (transcript search), columnas de metadata enriquecida, tablas cookies_meta y scrape_jobs - Módulos: segments, cookies (Netscape vault), export (json/csv/srt/html), analysis (word freq/timeline/wordcloud), monitor (watch loop), pipeline - CLI Click group: search, export, audio, channels, watch, analyze, re-render - Fix del bug de scoping de cookies en cli.py Webapp local (Workstream B): - FastAPI backend: dashboard, channels, videos facetado, transcript, search FTS, analysis, scrape jobs con SSE, cookies drag-and-drop, exports, folders (abrir en OS), tools (re-render, formato) - SPA no-build (Alpine.js + Tailwind + Chart.js por CDN): 9 vistas, tema dark command-center con acento rojo→rosa, cookie vault drag-drop, consola de scrapeo con progreso live vía SSE Launcher + subagentes (Workstream C): - start-server.bat / stop-server.bat con auto port-scan + browser open - .opencode/agent/webapp-builder.md + .opencode/goals/webapp-build.md Tests: 38 pytest verdes. Sin funcionalidad de IA (enfoque data-mining).
This commit is contained in:
urieljareth
@@ -0,0 +1,371 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File
|
||||
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from .. import analysis as analysis_mod
|
||||
from .. import cookies as cookies_mod
|
||||
from .. import export as export_mod
|
||||
from ..config import Config
|
||||
from ..store import Store
|
||||
|
||||
|
||||
def build_router(store: Store, cfg: Config, jobs) -> APIRouter:
|
||||
r = APIRouter(prefix="/api")
|
||||
|
||||
# -------------------------------------------------- dashboard
|
||||
@r.get("/dashboard")
|
||||
def dashboard():
|
||||
return store.dashboard()
|
||||
|
||||
# -------------------------------------------------- channels
|
||||
@r.get("/channels")
|
||||
def channels_list():
|
||||
return {"items": store.list_channels()}
|
||||
|
||||
@r.post("/channels")
|
||||
async def add_channel(payload: dict):
|
||||
from ..discover import discover_channel
|
||||
url = (payload or {}).get("url")
|
||||
if not url:
|
||||
raise HTTPException(400, "url required")
|
||||
cid, name, refs = discover_channel(url, sleep_subrequests=cfg.yt_dlp.sleep_subrequests)
|
||||
store.upsert_channel(cid, _handle(url), name, len(refs))
|
||||
store.upsert_videos(refs)
|
||||
return {"channel_id": cid, "name": name, "video_count": len(refs)}
|
||||
|
||||
@r.delete("/channels/{channel_id}")
|
||||
def del_channel(channel_id: str):
|
||||
store.delete_channel(channel_id)
|
||||
return {"deleted": channel_id}
|
||||
|
||||
# -------------------------------------------------- videos
|
||||
@r.get("/videos")
|
||||
def videos(
|
||||
channel: str | None = None, status: str | None = None,
|
||||
date_from: str | None = Query(None, alias="from"),
|
||||
date_to: str | None = Query(None, alias="to"),
|
||||
min_dur: int | None = None, q: str | None = None,
|
||||
sort: str = "upload_date_desc", page: int = 1, size: int = 50,
|
||||
):
|
||||
rows, total = store.query_videos(channel, status, date_from, date_to, min_dur, q, sort, page, size)
|
||||
return {"items": [_video_dict(v) for v in rows], "total": total, "page": page, "size": size}
|
||||
|
||||
@r.get("/videos/{video_id}")
|
||||
def video_detail(video_id: str):
|
||||
v = store.get_video(video_id)
|
||||
if not v:
|
||||
raise HTTPException(404, "video not found")
|
||||
return _video_dict(v)
|
||||
|
||||
@r.get("/videos/{video_id}/transcript")
|
||||
def video_transcript(video_id: str):
|
||||
segs = store.get_segments(video_id)
|
||||
return {"items": [
|
||||
{"idx": s.idx, "start_sec": s.start_sec, "end_sec": s.end_sec, "text": s.text}
|
||||
for s in segs
|
||||
]}
|
||||
|
||||
# -------------------------------------------------- search
|
||||
@r.get("/search")
|
||||
def search(q: str, channel: str | None = None, limit: int = 30):
|
||||
hits = store.search_segments(q, channel_id=channel, limit=limit)
|
||||
return {"items": [
|
||||
{"video_id": h.video_id, "title": h.title, "channel_id": h.channel_id,
|
||||
"start_sec": h.start_sec, "end_sec": h.end_sec, "snippet": h.snippet}
|
||||
for h in hits
|
||||
]}
|
||||
|
||||
# -------------------------------------------------- analysis
|
||||
@r.get("/analysis")
|
||||
def analysis(channel: str | None = None, type: str = "topwords", term: str | None = None, n: int = 50):
|
||||
if type == "topwords":
|
||||
tw = analysis_mod.top_words(store, n, channel)
|
||||
return {"items": [{"word": w, "count": c} for w, c in tw]}
|
||||
if type == "timeline":
|
||||
tl = analysis_mod.term_timeline(store, term or "", channel)
|
||||
return {"term": term, "items": [{"month": m, "count": c} for m, c in tl]}
|
||||
if type == "wordcloud":
|
||||
freq = analysis_mod.word_frequency(store, channel, top=300)
|
||||
return {"items": [{"word": w, "count": c} for w, c in freq.items()]}
|
||||
raise HTTPException(400, "unknown analysis type")
|
||||
|
||||
# -------------------------------------------------- scrape jobs
|
||||
@r.post("/scrape")
|
||||
async def start_scrape(payload: dict):
|
||||
channel_id = (payload or {}).get("channel_id")
|
||||
opts = (payload or {}).get("opts", {})
|
||||
job_id = jobs.enqueue(channel_id, opts)
|
||||
return {"job_id": job_id}
|
||||
|
||||
@r.get("/scrape")
|
||||
def list_jobs(limit: int = 30):
|
||||
return {"items": [_job_dict(j) for j in store.list_jobs(limit)]}
|
||||
|
||||
@r.get("/scrape/{job_id}")
|
||||
def get_job(job_id: str):
|
||||
j = store.get_job(job_id)
|
||||
if not j:
|
||||
raise HTTPException(404, "job not found")
|
||||
return _job_dict(j)
|
||||
|
||||
@r.get("/scrape/{job_id}/stream")
|
||||
async def stream_job(job_id: str, request: Request):
|
||||
import asyncio
|
||||
async def gen():
|
||||
cursor = 0
|
||||
while True:
|
||||
if await request.is_disconnected():
|
||||
return
|
||||
for ev in jobs.events_since(job_id, cursor):
|
||||
cursor += 1
|
||||
yield {"event": ev["event"], "data": json.dumps(ev["data"])}
|
||||
if ev["event"] in ("done", "error", "cancelled"):
|
||||
return
|
||||
terminal = jobs.is_terminal(job_id)
|
||||
if terminal and not jobs.events_since(job_id, cursor):
|
||||
yield {"event": terminal, "data": json.dumps({"status": terminal})}
|
||||
return
|
||||
await asyncio.sleep(0.25)
|
||||
return EventSourceResponse(gen())
|
||||
|
||||
@r.delete("/scrape/{job_id}")
|
||||
def cancel_job(job_id: str):
|
||||
ok = jobs.cancel(job_id)
|
||||
return {"cancelled": ok}
|
||||
|
||||
# -------------------------------------------------- cookies
|
||||
@r.get("/cookies")
|
||||
def cookies_list():
|
||||
items = []
|
||||
for c in cookies_mod.list_vault(store):
|
||||
items.append({
|
||||
"id": c.id, "filename": c.filename, "label": c.label,
|
||||
"added_at": c.added_at, "expires_at": c.expires_at,
|
||||
"is_active": c.is_active, "has_session": c.has_session,
|
||||
"cookie_count": c.cookie_count, "expired": cookies_mod.is_expired(c),
|
||||
})
|
||||
return {"items": items}
|
||||
|
||||
@r.post("/cookies/upload")
|
||||
async def cookies_upload(request: Request, files: list[UploadFile] = File(default=[])):
|
||||
# accept multipart files OR a JSON body {text, label}
|
||||
content_type = request.headers.get("content-type", "")
|
||||
imported = []
|
||||
if "application/json" in content_type:
|
||||
payload = await request.json()
|
||||
text = payload.get("text", "")
|
||||
label = payload.get("label") or "pasted"
|
||||
if not text:
|
||||
raise HTTPException(400, "no text")
|
||||
cid = cookies_mod.import_text(store, text, label)
|
||||
imported.append(cid)
|
||||
else:
|
||||
if not files:
|
||||
raise HTTPException(400, "no files")
|
||||
for f in files:
|
||||
raw = await f.read()
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
try:
|
||||
cid = cookies_mod.import_text(store, text, label=Path(f.filename or "upload").stem)
|
||||
imported.append(cid)
|
||||
except ValueError:
|
||||
continue
|
||||
if not imported:
|
||||
raise HTTPException(400, "no valid Netscape cookie files")
|
||||
# activate the first if nothing active
|
||||
if not store.get_active_cookie():
|
||||
cookies_mod.set_active(store, imported[0])
|
||||
return {"imported": imported}
|
||||
|
||||
@r.post("/cookies/{cookie_id}/activate")
|
||||
def cookies_activate(cookie_id: str):
|
||||
cookies_mod.set_active(store, cookie_id)
|
||||
return {"active": cookie_id}
|
||||
|
||||
@r.delete("/cookies/{cookie_id}")
|
||||
def cookies_delete(cookie_id: str):
|
||||
cookies_mod.delete(store, cookie_id)
|
||||
return {"deleted": cookie_id}
|
||||
|
||||
@r.post("/cookies/{cookie_id}/test")
|
||||
def cookies_test(cookie_id: str):
|
||||
c = store.get_cookie(cookie_id)
|
||||
if not c:
|
||||
raise HTTPException(404, "cookie not found")
|
||||
path = cookies_mod.resolve_active_path(store) if c.is_active else str(cookies_mod.cookies_dir() / c.filename)
|
||||
if not Path(path).exists():
|
||||
return {"ok": False, "message": "cookie file missing"}
|
||||
# lightweight validation via parse
|
||||
ok, info = cookies_mod.parse_netscape_file(path)
|
||||
expired = cookies_mod.is_expired(c)
|
||||
return {"ok": ok and not expired, "has_session": info["has_session"], "expired": expired,
|
||||
"expires_at": c.expires_at, "cookie_count": info["count"]}
|
||||
|
||||
# -------------------------------------------------- export
|
||||
@r.get("/export")
|
||||
def export(format: str = "json", channel: str | None = None):
|
||||
import tempfile, os
|
||||
tmp = Path(tempfile.gettempdir()) / "yt_scraper_export"
|
||||
tmp.mkdir(parents=True, exist_ok=True)
|
||||
label = channel or "all"
|
||||
try:
|
||||
if format == "json":
|
||||
p = export_mod.export_json(store, channel, tmp / f"{label}.json")
|
||||
elif format == "csv":
|
||||
p = export_mod.export_csv(store, channel, tmp / f"{label}.csv")
|
||||
elif format == "html":
|
||||
p = export_mod.export_html(store, channel, tmp / f"{label}.html")
|
||||
elif format == "srt":
|
||||
# stream a zip-less approach: return count + note (SRT is multi-file)
|
||||
files = export_mod.export_srt(store, channel, tmp / "srt")
|
||||
return {"files": [str(f.name) for f in files], "dir": str(tmp / "srt")}
|
||||
else:
|
||||
raise HTTPException(400, "unknown format")
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, str(exc))
|
||||
return FileResponse(str(p), filename=p.name)
|
||||
|
||||
# -------------------------------------------------- folders (open in OS)
|
||||
@r.get("/folders")
|
||||
def folders():
|
||||
data_root = Path(cfg.output_dir_resolved).parent
|
||||
return {"items": {
|
||||
"markdown": _folder_info(Path(cfg.output_dir_resolved)),
|
||||
"exports": _folder_info(data_root / "exports"),
|
||||
"audio": _folder_info(data_root / "audio"),
|
||||
"analysis": _folder_info(data_root / "analysis"),
|
||||
"database": _folder_info(Path(cfg.database_path_resolved).parent),
|
||||
}}
|
||||
|
||||
@r.post("/folders/open")
|
||||
async def open_folder(payload: dict):
|
||||
kind = (payload or {}).get("kind")
|
||||
channel_id = (payload or {}).get("channel_id")
|
||||
data_root = Path(cfg.output_dir_resolved).parent
|
||||
if kind == "markdown":
|
||||
target = Path(cfg.output_dir_resolved)
|
||||
if channel_id:
|
||||
ch = store.get_channel(channel_id) or {}
|
||||
if ch.get("name"):
|
||||
target = Path(cfg.output_dir_resolved) / _safe_dir(ch["name"])
|
||||
elif kind == "exports":
|
||||
target = data_root / "exports"
|
||||
elif kind == "audio":
|
||||
target = data_root / "audio"
|
||||
elif kind == "analysis":
|
||||
target = data_root / "analysis"
|
||||
elif kind == "database":
|
||||
target = Path(cfg.database_path_resolved).parent
|
||||
elif kind == "path" and (payload or {}).get("path"):
|
||||
req = Path(payload["path"]).resolve()
|
||||
proj = Path.cwd().resolve()
|
||||
try:
|
||||
req.relative_to(proj)
|
||||
except ValueError:
|
||||
raise HTTPException(403, "path outside project root")
|
||||
target = req
|
||||
else:
|
||||
raise HTTPException(400, "unknown kind")
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
_open_in_os(target)
|
||||
return {"opened": str(target)}
|
||||
|
||||
# -------------------------------------------------- tools (knowledge base)
|
||||
@r.post("/tools/re-render")
|
||||
def tools_rerender(payload: dict | None = None):
|
||||
from ..pipeline import re_render_videos
|
||||
from ..segments import backfill_from_markdown
|
||||
# ensure segments are populated first (idempotent)
|
||||
backfill_from_markdown(store, Path(cfg.output_dir_resolved))
|
||||
channel_id = (payload or {}).get("channel_id") if payload else None
|
||||
n = re_render_videos(store, cfg, channel_id)
|
||||
return {"re_rendered": n}
|
||||
|
||||
@r.get("/tools/format")
|
||||
def tools_format():
|
||||
# describe the established .md knowledge-base format
|
||||
return {
|
||||
"template": "templates/video.md.j2",
|
||||
"frontmatter": [
|
||||
"video_id", "title", "channel", "channel_id", "channel_url",
|
||||
"upload_date", "duration", "url", "transcript_lang", "transcript_src",
|
||||
"views", "likes", "tags", "thumbnail",
|
||||
],
|
||||
"body": "H1 title -> YouTube link -> (optional description) -> ## Transcripcion -> ### Chapter (MM:SS) -> **MM:SS** · segment lines",
|
||||
"output_dir": str(Path(cfg.output_dir_resolved)),
|
||||
}
|
||||
|
||||
return r
|
||||
|
||||
|
||||
def _video_dict(v) -> dict:
|
||||
import json as _json
|
||||
tags = []
|
||||
if v.tags:
|
||||
try:
|
||||
tags = _json.loads(v.tags)
|
||||
except Exception:
|
||||
tags = []
|
||||
return {
|
||||
"video_id": v.video_id, "channel_id": v.channel_id, "title": v.title, "url": v.url,
|
||||
"upload_date": v.upload_date, "duration": v.duration, "status": v.status,
|
||||
"transcript_lang": v.transcript_lang, "has_chapters": bool(v.has_chapters),
|
||||
"view_count": v.view_count, "like_count": v.like_count, "tags": tags,
|
||||
"thumbnail": v.thumbnail, "markdown_path": v.markdown_path,
|
||||
"error_msg": v.error_msg,
|
||||
}
|
||||
|
||||
|
||||
def _job_dict(j) -> dict:
|
||||
return {
|
||||
"id": j.id, "channel_id": j.channel_id, "opts": _loads(j.opts_json),
|
||||
"status": j.status, "started_at": j.started_at, "finished_at": j.finished_at,
|
||||
"total": j.total, "completed": j.completed, "last_error": j.last_error,
|
||||
}
|
||||
|
||||
|
||||
def _loads(s):
|
||||
if not s:
|
||||
return {}
|
||||
try:
|
||||
return json.loads(s)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _handle(url: str) -> str:
|
||||
if "@" in url:
|
||||
return "@" + url.split("@", 1)[1].split("/", 1)[0]
|
||||
return ""
|
||||
|
||||
|
||||
def _folder_info(path: Path) -> dict:
|
||||
path = Path(path)
|
||||
return {"path": str(path.resolve()), "exists": path.exists()}
|
||||
|
||||
|
||||
def _safe_dir(name: str) -> str:
|
||||
safe = "".join(c for c in (name or "") if c not in r'\/:*?"<>|')
|
||||
return (safe.strip().strip(".") or "unknown")
|
||||
|
||||
|
||||
def _open_in_os(path: Path) -> None:
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
p = str(path)
|
||||
try:
|
||||
if sys.platform.startswith("win"):
|
||||
os.startfile(p) # type: ignore[attr-defined]
|
||||
elif sys.platform == "darwin":
|
||||
subprocess.Popen(["open", p])
|
||||
else:
|
||||
subprocess.Popen(["xdg-open", p])
|
||||
except Exception as exc:
|
||||
raise HTTPException(500, f"cannot open folder: {exc}")
|
||||
Reference in New Issue
Block a user