from __future__ import annotations import json from pathlib import Path from typing import Any from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File from fastapi.responses import FileResponse, JSONResponse, StreamingResponse from sse_starlette.sse import EventSourceResponse from .. import analysis as analysis_mod from .. import cookies as cookies_mod from .. import export as export_mod from ..config import Config from ..store import Store def build_router(store: Store, cfg: Config, jobs) -> APIRouter: r = APIRouter(prefix="/api") # -------------------------------------------------- dashboard @r.get("/dashboard") def dashboard(): return store.dashboard() # -------------------------------------------------- channels @r.get("/channels") def channels_list(): return {"items": store.list_channels()} @r.post("/channels") async def add_channel(payload: dict): from ..discover import discover_channel url = (payload or {}).get("url") if not url: raise HTTPException(400, "url required") cid, name, refs = discover_channel(url, sleep_subrequests=cfg.yt_dlp.sleep_subrequests) store.upsert_channel(cid, _handle(url), name, len(refs)) store.upsert_videos(refs) return {"channel_id": cid, "name": name, "video_count": len(refs)} @r.delete("/channels/{channel_id}") def del_channel(channel_id: str): store.delete_channel(channel_id) return {"deleted": channel_id} # -------------------------------------------------- videos @r.get("/videos") def videos( channel: str | None = None, status: str | None = None, date_from: str | None = Query(None, alias="from"), date_to: str | None = Query(None, alias="to"), min_dur: int | None = None, q: str | None = None, sort: str = "upload_date_desc", page: int = 1, size: int = 50, ): rows, total = store.query_videos(channel, status, date_from, date_to, min_dur, q, sort, page, size) return {"items": [_video_dict(v) for v in rows], "total": total, "page": page, "size": size} @r.get("/videos/{video_id}") def video_detail(video_id: str): v = store.get_video(video_id) if not v: raise HTTPException(404, "video not found") return _video_dict(v) @r.get("/videos/{video_id}/transcript") def video_transcript(video_id: str): segs = store.get_segments(video_id) return {"items": [ {"idx": s.idx, "start_sec": s.start_sec, "end_sec": s.end_sec, "text": s.text} for s in segs ]} # -------------------------------------------------- search @r.get("/search") def search(q: str, channel: str | None = None, limit: int = 30): hits = store.search_segments(q, channel_id=channel, limit=limit) return {"items": [ {"video_id": h.video_id, "title": h.title, "channel_id": h.channel_id, "start_sec": h.start_sec, "end_sec": h.end_sec, "snippet": h.snippet} for h in hits ]} # -------------------------------------------------- analysis @r.get("/analysis") def analysis(channel: str | None = None, type: str = "topwords", term: str | None = None, n: int = 50): if type == "topwords": tw = analysis_mod.top_words(store, n, channel) return {"items": [{"word": w, "count": c} for w, c in tw]} if type == "timeline": tl = analysis_mod.term_timeline(store, term or "", channel) return {"term": term, "items": [{"month": m, "count": c} for m, c in tl]} if type == "wordcloud": freq = analysis_mod.word_frequency(store, channel, top=300) return {"items": [{"word": w, "count": c} for w, c in freq.items()]} raise HTTPException(400, "unknown analysis type") # -------------------------------------------------- scrape jobs @r.post("/scrape") async def start_scrape(payload: dict): channel_id = (payload or {}).get("channel_id") opts = (payload or {}).get("opts", {}) job_id = jobs.enqueue(channel_id, opts) return {"job_id": job_id} @r.get("/scrape") def list_jobs(limit: int = 30): return {"items": [_job_dict(j) for j in store.list_jobs(limit)]} @r.get("/scrape/{job_id}") def get_job(job_id: str): j = store.get_job(job_id) if not j: raise HTTPException(404, "job not found") return _job_dict(j) @r.get("/scrape/{job_id}/stream") async def stream_job(job_id: str, request: Request): import asyncio async def gen(): cursor = 0 while True: if await request.is_disconnected(): return for ev in jobs.events_since(job_id, cursor): cursor += 1 yield {"event": ev["event"], "data": json.dumps(ev["data"])} if ev["event"] in ("done", "error", "cancelled"): return terminal = jobs.is_terminal(job_id) if terminal and not jobs.events_since(job_id, cursor): yield {"event": terminal, "data": json.dumps({"status": terminal})} return await asyncio.sleep(0.25) return EventSourceResponse(gen()) @r.delete("/scrape/{job_id}") def cancel_job(job_id: str): ok = jobs.cancel(job_id) return {"cancelled": ok} # -------------------------------------------------- cookies @r.get("/cookies") def cookies_list(): items = [] for c in cookies_mod.list_vault(store): items.append({ "id": c.id, "filename": c.filename, "label": c.label, "added_at": c.added_at, "expires_at": c.expires_at, "is_active": c.is_active, "has_session": c.has_session, "cookie_count": c.cookie_count, "expired": cookies_mod.is_expired(c), }) return {"items": items} @r.post("/cookies/upload") async def cookies_upload(request: Request, files: list[UploadFile] = File(default=[])): # accept multipart files OR a JSON body {text, label} content_type = request.headers.get("content-type", "") imported = [] if "application/json" in content_type: payload = await request.json() text = payload.get("text", "") label = payload.get("label") or "pasted" if not text: raise HTTPException(400, "no text") cid = cookies_mod.import_text(store, text, label) imported.append(cid) else: if not files: raise HTTPException(400, "no files") for f in files: raw = await f.read() text = raw.decode("utf-8", errors="replace") try: cid = cookies_mod.import_text(store, text, label=Path(f.filename or "upload").stem) imported.append(cid) except ValueError: continue if not imported: raise HTTPException(400, "no valid Netscape cookie files") # activate the first if nothing active if not store.get_active_cookie(): cookies_mod.set_active(store, imported[0]) return {"imported": imported} @r.post("/cookies/{cookie_id}/activate") def cookies_activate(cookie_id: str): cookies_mod.set_active(store, cookie_id) return {"active": cookie_id} @r.delete("/cookies/{cookie_id}") def cookies_delete(cookie_id: str): cookies_mod.delete(store, cookie_id) return {"deleted": cookie_id} @r.post("/cookies/{cookie_id}/test") def cookies_test(cookie_id: str): c = store.get_cookie(cookie_id) if not c: raise HTTPException(404, "cookie not found") path = cookies_mod.resolve_active_path(store) if c.is_active else str(cookies_mod.cookies_dir() / c.filename) if not Path(path).exists(): return {"ok": False, "message": "cookie file missing"} # lightweight validation via parse ok, info = cookies_mod.parse_netscape_file(path) expired = cookies_mod.is_expired(c) return {"ok": ok and not expired, "has_session": info["has_session"], "expired": expired, "expires_at": c.expires_at, "cookie_count": info["count"]} # -------------------------------------------------- export @r.get("/export") def export(format: str = "json", channel: str | None = None): import tempfile, os tmp = Path(tempfile.gettempdir()) / "yt_scraper_export" tmp.mkdir(parents=True, exist_ok=True) label = channel or "all" try: if format == "json": p = export_mod.export_json(store, channel, tmp / f"{label}.json") elif format == "csv": p = export_mod.export_csv(store, channel, tmp / f"{label}.csv") elif format == "html": p = export_mod.export_html(store, channel, tmp / f"{label}.html") elif format == "srt": # stream a zip-less approach: return count + note (SRT is multi-file) files = export_mod.export_srt(store, channel, tmp / "srt") return {"files": [str(f.name) for f in files], "dir": str(tmp / "srt")} else: raise HTTPException(400, "unknown format") except Exception as exc: raise HTTPException(500, str(exc)) return FileResponse(str(p), filename=p.name) # -------------------------------------------------- folders (open in OS) @r.get("/folders") def folders(): data_root = Path(cfg.output_dir_resolved).parent return {"items": { "markdown": _folder_info(Path(cfg.output_dir_resolved)), "exports": _folder_info(data_root / "exports"), "audio": _folder_info(data_root / "audio"), "analysis": _folder_info(data_root / "analysis"), "database": _folder_info(Path(cfg.database_path_resolved).parent), }} @r.post("/folders/open") async def open_folder(payload: dict): kind = (payload or {}).get("kind") channel_id = (payload or {}).get("channel_id") data_root = Path(cfg.output_dir_resolved).parent if kind == "markdown": target = Path(cfg.output_dir_resolved) if channel_id: ch = store.get_channel(channel_id) or {} if ch.get("name"): target = Path(cfg.output_dir_resolved) / _safe_dir(ch["name"]) elif kind == "exports": target = data_root / "exports" elif kind == "audio": target = data_root / "audio" elif kind == "analysis": target = data_root / "analysis" elif kind == "database": target = Path(cfg.database_path_resolved).parent elif kind == "path" and (payload or {}).get("path"): req = Path(payload["path"]).resolve() proj = Path.cwd().resolve() try: req.relative_to(proj) except ValueError: raise HTTPException(403, "path outside project root") target = req else: raise HTTPException(400, "unknown kind") target.mkdir(parents=True, exist_ok=True) _open_in_os(target) return {"opened": str(target)} # -------------------------------------------------- tools (knowledge base) @r.post("/tools/re-render") def tools_rerender(payload: dict | None = None): from ..pipeline import re_render_videos from ..segments import backfill_from_markdown # ensure segments are populated first (idempotent) backfill_from_markdown(store, Path(cfg.output_dir_resolved)) channel_id = (payload or {}).get("channel_id") if payload else None n = re_render_videos(store, cfg, channel_id) return {"re_rendered": n} @r.get("/tools/format") def tools_format(): # describe the established .md knowledge-base format return { "template": "templates/video.md.j2", "frontmatter": [ "video_id", "title", "channel", "channel_id", "channel_url", "upload_date", "duration", "url", "transcript_lang", "transcript_src", "views", "likes", "tags", "thumbnail", ], "body": "H1 title -> YouTube link -> (optional description) -> ## Transcripcion -> ### Chapter (MM:SS) -> **MM:SS** ยท segment lines", "output_dir": str(Path(cfg.output_dir_resolved)), } # -------------------------------------------------- per-video / batch markdown @r.post("/scrape/video/{video_id}") def scrape_one_video(video_id: str, payload: dict | None = None): opts = (payload or {}).get("opts", {}) or {} opts["video_ids"] = [video_id] job_id = jobs.enqueue(None, opts) return {"job_id": job_id} @r.post("/scrape/batch") async def scrape_batch(payload: dict): video_ids = (payload or {}).get("video_ids") or [] if not video_ids: raise HTTPException(400, "video_ids required") opts = (payload or {}).get("opts", {}) or {} opts["video_ids"] = list(video_ids) job_id = jobs.enqueue(None, opts) return {"job_id": job_id, "count": len(video_ids)} @r.get("/videos/{video_id}/markdown") def video_markdown(video_id: str): v = store.get_video(video_id) if not v or not v.markdown_path: raise HTTPException(404, "markdown not generated yet") p = Path(cfg.output_dir_resolved).parent / v.markdown_path if not p.exists(): raise HTTPException(404, "markdown file missing on disk") return FileResponse(str(p), filename=p.name, media_type="text/markdown") # -------------------------------------------------- thumbnails (local cache) @r.post("/tools/thumbnails") async def download_thumbnails(payload: dict): import requests as _req video_ids = (payload or {}).get("video_ids") or [] channel_id = (payload or {}).get("channel_id") if channel_id and not video_ids: video_ids = [v.video_id for v in store.get_all(channel_id) if v.thumbnail] out_dir = Path(cfg.output_dir_resolved).parent / "thumbnails" out_dir.mkdir(parents=True, exist_ok=True) n = 0 for vid in video_ids: v = store.get_video(vid) if not v or not v.thumbnail: continue target = out_dir / f"{vid}.jpg" if target.exists(): n += 1 continue try: r2 = _req.get(v.thumbnail, timeout=15, headers={"User-Agent": "Mozilla/5.0"}) r2.raise_for_status() target.write_bytes(r2.content) n += 1 except Exception: continue return {"downloaded": n, "dir": str(out_dir)} @r.get("/thumbnails/{video_id}") def serve_thumbnail(video_id: str): from fastapi.responses import RedirectResponse local = Path(cfg.output_dir_resolved).parent / "thumbnails" / f"{video_id}.jpg" if local.exists(): return FileResponse(str(local), media_type="image/jpeg") v = store.get_video(video_id) if v and v.thumbnail: return RedirectResponse(url=v.thumbnail, status_code=302) raise HTTPException(404, "no thumbnail") # -------------------------------------------------- clip (transcript segment) @r.get("/clip/{video_id}") def clip_video(video_id: str, frm: str = Query("0:00", alias="from"), to: str = Query("", alias="to")): from ..segments import ts_to_seconds segs = store.get_segments(video_id) if not segs: raise HTTPException(404, "no transcript for this video") start = ts_to_seconds(frm) end = ts_to_seconds(to) if to else float("inf") picked = [s for s in segs if start <= s.start_sec < end] text = "\n".join(f"[{int(s.start_sec//60):02d}:{int(s.start_sec%60):02d}] {s.text}" for s in picked) v = store.get_video(video_id) return { "video_id": video_id, "title": v.title if v else "", "from": frm, "to": to or "end", "segment_count": len(picked), "text": text, } # -------------------------------------------------- stats (per channel / all) @r.get("/stats") def stats(channel: str | None = None): d = store.dashboard() if channel: d["channels"] = [c for c in d["channels"] if c["channel_id"] == channel] totals = { "videos": sum((c.get("video_count_db") or 0) for c in d["channels"]), "duration_sec": sum((c.get("total_duration") or 0) for c in d["channels"]), "views": sum((c.get("total_views") or 0) for c in d["channels"]), "likes": sum((c.get("total_likes") or 0) for c in d["channels"]), } return {"channels": d["channels"], "status_breakdown": d["status_breakdown"], "top_tags": d["top_tags"], "totals": totals} # -------------------------------------------------- audio (job) @r.post("/tools/audio") async def tools_audio(payload: dict): video_ids = (payload or {}).get("video_ids") or [] channel_id = (payload or {}).get("channel_id") if not video_ids and not channel_id: raise HTTPException(400, "video_ids or channel_id required") opts = {"mode": "audio", "channel_id": channel_id} if video_ids: opts["video_ids"] = list(video_ids) job_id = jobs.enqueue(channel_id, opts) return {"job_id": job_id} return r def _video_dict(v) -> dict: import json as _json tags = [] if v.tags: try: tags = _json.loads(v.tags) except Exception: tags = [] return { "video_id": v.video_id, "channel_id": v.channel_id, "title": v.title, "url": v.url, "upload_date": v.upload_date, "duration": v.duration, "status": v.status, "transcript_lang": v.transcript_lang, "has_chapters": bool(v.has_chapters), "view_count": v.view_count, "like_count": v.like_count, "tags": tags, "thumbnail": v.thumbnail, "markdown_path": v.markdown_path, "error_msg": v.error_msg, } def _job_dict(j) -> dict: return { "id": j.id, "channel_id": j.channel_id, "opts": _loads(j.opts_json), "status": j.status, "started_at": j.started_at, "finished_at": j.finished_at, "total": j.total, "completed": j.completed, "last_error": j.last_error, } def _loads(s): if not s: return {} try: return json.loads(s) except Exception: return {} def _handle(url: str) -> str: if "@" in url: return "@" + url.split("@", 1)[1].split("/", 1)[0] return "" def _folder_info(path: Path) -> dict: path = Path(path) return {"path": str(path.resolve()), "exists": path.exists()} def _safe_dir(name: str) -> str: safe = "".join(c for c in (name or "") if c not in r'\/:*?"<>|') return (safe.strip().strip(".") or "unknown") def _open_in_os(path: Path) -> None: import os import sys import subprocess p = str(path) try: if sys.platform.startswith("win"): os.startfile(p) # type: ignore[attr-defined] elif sys.platform == "darwin": subprocess.Popen(["open", p]) else: subprocess.Popen(["xdg-open", p]) except Exception as exc: raise HTTPException(500, f"cannot open folder: {exc}")