urieljareth/yt-channel-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0682d2d80649a8bcb4417a9f66f75235b83daed1
yt-channel-scraper/src/yt_scraper/webapp/api.py
T
urieljareth 0682d2d806 feat: batch .md download, multi-select, thumbnails, all OPPORTUNITIES tools 
Videos view: multi-select (checkboxes) + select-all/deselect-all + bulk action
bar (Download .md batch, Download thumbnails, Download audio). Per-video buttons:
.md download (done) or Process to .md (pending), clip.

Channels: per-channel pending count + 'Download pending .md' (batches pending
video_ids). Each channel keeps Videos/.md folder/Remove actions.

Backend (jobs.py + api.py):
- POST /api/scrape/batch {video_ids} + POST /api/scrape/video/{id} (on-demand
  .md for any video incl. pending) as SSE-tracked jobs
- GET /api/videos/{id}/markdown (file download)
- POST /api/tools/thumbnails + GET /api/thumbnails/{id} (local cache, offline)
- GET /api/clip/{id}?from=&to= (transcript segment)
- GET /api/stats (aggregate totals)
- POST /api/tools/audio (mp3 job)

Tools view rebuilt as full Knowledge Base grid surfacing every OPPORTUNITIES.md
feature as electable buttons (md download, re-render, export, search, analysis,
thumbnails, audio, clip, stats, cookies, channels, watch, open folders).

Floating job widget (SSE) for any running job. Thumbnails served via
/api/thumbnails/{id} everywhere (works offline once cached).

Launcher: robust Open-Browser helper (3 fallback methods) auto-opens
http://127.0.0.1:<port> on start + on idempotent re-open. data/ gitignored.
2026-07-26 23:39:05 -06:00

487 lines
20 KiB
Python
Raw Blame History

from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
from sse_starlette.sse import EventSourceResponse
from .. import analysis as analysis_mod
from .. import cookies as cookies_mod
from .. import export as export_mod
from ..config import Config
from ..store import Store
def build_router(store: Store, cfg: Config, jobs) -> APIRouter:
r = APIRouter(prefix="/api")
# -------------------------------------------------- dashboard
@r.get("/dashboard")
def dashboard():
return store.dashboard()
# -------------------------------------------------- channels
@r.get("/channels")
def channels_list():
return {"items": store.list_channels()}
@r.post("/channels")
async def add_channel(payload: dict):
from ..discover import discover_channel
url = (payload or {}).get("url")
if not url:
raise HTTPException(400, "url required")
cid, name, refs = discover_channel(url, sleep_subrequests=cfg.yt_dlp.sleep_subrequests)
store.upsert_channel(cid, _handle(url), name, len(refs))
store.upsert_videos(refs)
return {"channel_id": cid, "name": name, "video_count": len(refs)}
@r.delete("/channels/{channel_id}")
def del_channel(channel_id: str):
store.delete_channel(channel_id)
return {"deleted": channel_id}
# -------------------------------------------------- videos
@r.get("/videos")
def videos(
channel: str | None = None, status: str | None = None,
date_from: str | None = Query(None, alias="from"),
date_to: str | None = Query(None, alias="to"),
min_dur: int | None = None, q: str | None = None,
sort: str = "upload_date_desc", page: int = 1, size: int = 50,
):
rows, total = store.query_videos(channel, status, date_from, date_to, min_dur, q, sort, page, size)
return {"items": [_video_dict(v) for v in rows], "total": total, "page": page, "size": size}
@r.get("/videos/{video_id}")
def video_detail(video_id: str):
v = store.get_video(video_id)
if not v:
raise HTTPException(404, "video not found")
return _video_dict(v)
@r.get("/videos/{video_id}/transcript")
def video_transcript(video_id: str):
segs = store.get_segments(video_id)
return {"items": [
{"idx": s.idx, "start_sec": s.start_sec, "end_sec": s.end_sec, "text": s.text}
for s in segs
]}
# -------------------------------------------------- search
@r.get("/search")
def search(q: str, channel: str | None = None, limit: int = 30):
hits = store.search_segments(q, channel_id=channel, limit=limit)
return {"items": [
{"video_id": h.video_id, "title": h.title, "channel_id": h.channel_id,
"start_sec": h.start_sec, "end_sec": h.end_sec, "snippet": h.snippet}
for h in hits
]}
# -------------------------------------------------- analysis
@r.get("/analysis")
def analysis(channel: str | None = None, type: str = "topwords", term: str | None = None, n: int = 50):
if type == "topwords":
tw = analysis_mod.top_words(store, n, channel)
return {"items": [{"word": w, "count": c} for w, c in tw]}
if type == "timeline":
tl = analysis_mod.term_timeline(store, term or "", channel)
return {"term": term, "items": [{"month": m, "count": c} for m, c in tl]}
if type == "wordcloud":
freq = analysis_mod.word_frequency(store, channel, top=300)
return {"items": [{"word": w, "count": c} for w, c in freq.items()]}
raise HTTPException(400, "unknown analysis type")
# -------------------------------------------------- scrape jobs
@r.post("/scrape")
async def start_scrape(payload: dict):
channel_id = (payload or {}).get("channel_id")
opts = (payload or {}).get("opts", {})
job_id = jobs.enqueue(channel_id, opts)
return {"job_id": job_id}
@r.get("/scrape")
def list_jobs(limit: int = 30):
return {"items": [_job_dict(j) for j in store.list_jobs(limit)]}
@r.get("/scrape/{job_id}")
def get_job(job_id: str):
j = store.get_job(job_id)
if not j:
raise HTTPException(404, "job not found")
return _job_dict(j)
@r.get("/scrape/{job_id}/stream")
async def stream_job(job_id: str, request: Request):
import asyncio
async def gen():
cursor = 0
while True:
if await request.is_disconnected():
return
for ev in jobs.events_since(job_id, cursor):
cursor += 1
yield {"event": ev["event"], "data": json.dumps(ev["data"])}
if ev["event"] in ("done", "error", "cancelled"):
return
terminal = jobs.is_terminal(job_id)
if terminal and not jobs.events_since(job_id, cursor):
yield {"event": terminal, "data": json.dumps({"status": terminal})}
return
await asyncio.sleep(0.25)
return EventSourceResponse(gen())
@r.delete("/scrape/{job_id}")
def cancel_job(job_id: str):
ok = jobs.cancel(job_id)
return {"cancelled": ok}
# -------------------------------------------------- cookies
@r.get("/cookies")
def cookies_list():
items = []
for c in cookies_mod.list_vault(store):
items.append({
"id": c.id, "filename": c.filename, "label": c.label,
"added_at": c.added_at, "expires_at": c.expires_at,
"is_active": c.is_active, "has_session": c.has_session,
"cookie_count": c.cookie_count, "expired": cookies_mod.is_expired(c),
})
return {"items": items}
@r.post("/cookies/upload")
async def cookies_upload(request: Request, files: list[UploadFile] = File(default=[])):
# accept multipart files OR a JSON body {text, label}
content_type = request.headers.get("content-type", "")
imported = []
if "application/json" in content_type:
payload = await request.json()
text = payload.get("text", "")
label = payload.get("label") or "pasted"
if not text:
raise HTTPException(400, "no text")
cid = cookies_mod.import_text(store, text, label)
imported.append(cid)
else:
if not files:
raise HTTPException(400, "no files")
for f in files:
raw = await f.read()
text = raw.decode("utf-8", errors="replace")
try:
cid = cookies_mod.import_text(store, text, label=Path(f.filename or "upload").stem)
imported.append(cid)
except ValueError:
continue
if not imported:
raise HTTPException(400, "no valid Netscape cookie files")
# activate the first if nothing active
if not store.get_active_cookie():
cookies_mod.set_active(store, imported[0])
return {"imported": imported}
@r.post("/cookies/{cookie_id}/activate")
def cookies_activate(cookie_id: str):
cookies_mod.set_active(store, cookie_id)
return {"active": cookie_id}
@r.delete("/cookies/{cookie_id}")
def cookies_delete(cookie_id: str):
cookies_mod.delete(store, cookie_id)
return {"deleted": cookie_id}
@r.post("/cookies/{cookie_id}/test")
def cookies_test(cookie_id: str):
c = store.get_cookie(cookie_id)
if not c:
raise HTTPException(404, "cookie not found")
path = cookies_mod.resolve_active_path(store) if c.is_active else str(cookies_mod.cookies_dir() / c.filename)
if not Path(path).exists():
return {"ok": False, "message": "cookie file missing"}
# lightweight validation via parse
ok, info = cookies_mod.parse_netscape_file(path)
expired = cookies_mod.is_expired(c)
return {"ok": ok and not expired, "has_session": info["has_session"], "expired": expired,
"expires_at": c.expires_at, "cookie_count": info["count"]}
# -------------------------------------------------- export
@r.get("/export")
def export(format: str = "json", channel: str | None = None):
import tempfile, os
tmp = Path(tempfile.gettempdir()) / "yt_scraper_export"
tmp.mkdir(parents=True, exist_ok=True)
label = channel or "all"
try:
if format == "json":
p = export_mod.export_json(store, channel, tmp / f"{label}.json")
elif format == "csv":
p = export_mod.export_csv(store, channel, tmp / f"{label}.csv")
elif format == "html":
p = export_mod.export_html(store, channel, tmp / f"{label}.html")
elif format == "srt":
# stream a zip-less approach: return count + note (SRT is multi-file)
files = export_mod.export_srt(store, channel, tmp / "srt")
return {"files": [str(f.name) for f in files], "dir": str(tmp / "srt")}
else:
raise HTTPException(400, "unknown format")
except Exception as exc:
raise HTTPException(500, str(exc))
return FileResponse(str(p), filename=p.name)
# -------------------------------------------------- folders (open in OS)
@r.get("/folders")
def folders():
data_root = Path(cfg.output_dir_resolved).parent
return {"items": {
"markdown": _folder_info(Path(cfg.output_dir_resolved)),
"exports": _folder_info(data_root / "exports"),
"audio": _folder_info(data_root / "audio"),
"analysis": _folder_info(data_root / "analysis"),
"database": _folder_info(Path(cfg.database_path_resolved).parent),
}}
@r.post("/folders/open")
async def open_folder(payload: dict):
kind = (payload or {}).get("kind")
channel_id = (payload or {}).get("channel_id")
data_root = Path(cfg.output_dir_resolved).parent
if kind == "markdown":
target = Path(cfg.output_dir_resolved)
if channel_id:
ch = store.get_channel(channel_id) or {}
if ch.get("name"):
target = Path(cfg.output_dir_resolved) / _safe_dir(ch["name"])
elif kind == "exports":
target = data_root / "exports"
elif kind == "audio":
target = data_root / "audio"
elif kind == "analysis":
target = data_root / "analysis"
elif kind == "database":
target = Path(cfg.database_path_resolved).parent
elif kind == "path" and (payload or {}).get("path"):
req = Path(payload["path"]).resolve()
proj = Path.cwd().resolve()
try:
req.relative_to(proj)
except ValueError:
raise HTTPException(403, "path outside project root")
target = req
else:
raise HTTPException(400, "unknown kind")
target.mkdir(parents=True, exist_ok=True)
_open_in_os(target)
return {"opened": str(target)}
# -------------------------------------------------- tools (knowledge base)
@r.post("/tools/re-render")
def tools_rerender(payload: dict | None = None):
from ..pipeline import re_render_videos
from ..segments import backfill_from_markdown
# ensure segments are populated first (idempotent)
backfill_from_markdown(store, Path(cfg.output_dir_resolved))
channel_id = (payload or {}).get("channel_id") if payload else None
n = re_render_videos(store, cfg, channel_id)
return {"re_rendered": n}
@r.get("/tools/format")
def tools_format():
# describe the established .md knowledge-base format
return {
"template": "templates/video.md.j2",
"frontmatter": [
"video_id", "title", "channel", "channel_id", "channel_url",
"upload_date", "duration", "url", "transcript_lang", "transcript_src",
"views", "likes", "tags", "thumbnail",
],
"body": "H1 title -> YouTube link -> (optional description) -> ## Transcripcion -> ### Chapter (MM:SS) -> **MM:SS** · segment lines",
"output_dir": str(Path(cfg.output_dir_resolved)),
}
# -------------------------------------------------- per-video / batch markdown
@r.post("/scrape/video/{video_id}")
def scrape_one_video(video_id: str, payload: dict | None = None):
opts = (payload or {}).get("opts", {}) or {}
opts["video_ids"] = [video_id]
job_id = jobs.enqueue(None, opts)
return {"job_id": job_id}
@r.post("/scrape/batch")
async def scrape_batch(payload: dict):
video_ids = (payload or {}).get("video_ids") or []
if not video_ids:
raise HTTPException(400, "video_ids required")
opts = (payload or {}).get("opts", {}) or {}
opts["video_ids"] = list(video_ids)
job_id = jobs.enqueue(None, opts)
return {"job_id": job_id, "count": len(video_ids)}
@r.get("/videos/{video_id}/markdown")
def video_markdown(video_id: str):
v = store.get_video(video_id)
if not v or not v.markdown_path:
raise HTTPException(404, "markdown not generated yet")
p = Path(cfg.output_dir_resolved).parent / v.markdown_path
if not p.exists():
raise HTTPException(404, "markdown file missing on disk")
return FileResponse(str(p), filename=p.name, media_type="text/markdown")
# -------------------------------------------------- thumbnails (local cache)
@r.post("/tools/thumbnails")
async def download_thumbnails(payload: dict):
import requests as _req
video_ids = (payload or {}).get("video_ids") or []
channel_id = (payload or {}).get("channel_id")
if channel_id and not video_ids:
video_ids = [v.video_id for v in store.get_all(channel_id) if v.thumbnail]
out_dir = Path(cfg.output_dir_resolved).parent / "thumbnails"
out_dir.mkdir(parents=True, exist_ok=True)
n = 0
for vid in video_ids:
v = store.get_video(vid)
if not v or not v.thumbnail:
continue
target = out_dir / f"{vid}.jpg"
if target.exists():
n += 1
continue
try:
r2 = _req.get(v.thumbnail, timeout=15, headers={"User-Agent": "Mozilla/5.0"})
r2.raise_for_status()
target.write_bytes(r2.content)
n += 1
except Exception:
continue
return {"downloaded": n, "dir": str(out_dir)}
@r.get("/thumbnails/{video_id}")
def serve_thumbnail(video_id: str):
from fastapi.responses import RedirectResponse
local = Path(cfg.output_dir_resolved).parent / "thumbnails" / f"{video_id}.jpg"
if local.exists():
return FileResponse(str(local), media_type="image/jpeg")
v = store.get_video(video_id)
if v and v.thumbnail:
return RedirectResponse(url=v.thumbnail, status_code=302)
raise HTTPException(404, "no thumbnail")
# -------------------------------------------------- clip (transcript segment)
@r.get("/clip/{video_id}")
def clip_video(video_id: str, frm: str = Query("0:00", alias="from"), to: str = Query("", alias="to")):
from ..segments import ts_to_seconds
segs = store.get_segments(video_id)
if not segs:
raise HTTPException(404, "no transcript for this video")
start = ts_to_seconds(frm)
end = ts_to_seconds(to) if to else float("inf")
picked = [s for s in segs if start <= s.start_sec < end]
text = "\n".join(f"[{int(s.start_sec//60):02d}:{int(s.start_sec%60):02d}] {s.text}" for s in picked)
v = store.get_video(video_id)
return {
"video_id": video_id,
"title": v.title if v else "",
"from": frm, "to": to or "end",
"segment_count": len(picked),
"text": text,
}
# -------------------------------------------------- stats (per channel / all)
@r.get("/stats")
def stats(channel: str | None = None):
d = store.dashboard()
if channel:
d["channels"] = [c for c in d["channels"] if c["channel_id"] == channel]
totals = {
"videos": sum((c.get("video_count_db") or 0) for c in d["channels"]),
"duration_sec": sum((c.get("total_duration") or 0) for c in d["channels"]),
"views": sum((c.get("total_views") or 0) for c in d["channels"]),
"likes": sum((c.get("total_likes") or 0) for c in d["channels"]),
}
return {"channels": d["channels"], "status_breakdown": d["status_breakdown"],
"top_tags": d["top_tags"], "totals": totals}
# -------------------------------------------------- audio (job)
@r.post("/tools/audio")
async def tools_audio(payload: dict):
video_ids = (payload or {}).get("video_ids") or []
channel_id = (payload or {}).get("channel_id")
if not video_ids and not channel_id:
raise HTTPException(400, "video_ids or channel_id required")
opts = {"mode": "audio", "channel_id": channel_id}
if video_ids:
opts["video_ids"] = list(video_ids)
job_id = jobs.enqueue(channel_id, opts)
return {"job_id": job_id}
return r
def _video_dict(v) -> dict:
import json as _json
tags = []
if v.tags:
try:
tags = _json.loads(v.tags)
except Exception:
tags = []
return {
"video_id": v.video_id, "channel_id": v.channel_id, "title": v.title, "url": v.url,
"upload_date": v.upload_date, "duration": v.duration, "status": v.status,
"transcript_lang": v.transcript_lang, "has_chapters": bool(v.has_chapters),
"view_count": v.view_count, "like_count": v.like_count, "tags": tags,
"thumbnail": v.thumbnail, "markdown_path": v.markdown_path,
"error_msg": v.error_msg,
}
def _job_dict(j) -> dict:
return {
"id": j.id, "channel_id": j.channel_id, "opts": _loads(j.opts_json),
"status": j.status, "started_at": j.started_at, "finished_at": j.finished_at,
"total": j.total, "completed": j.completed, "last_error": j.last_error,
}
def _loads(s):
if not s:
return {}
try:
return json.loads(s)
except Exception:
return {}
def _handle(url: str) -> str:
if "@" in url:
return "@" + url.split("@", 1)[1].split("/", 1)[0]
return ""
def _folder_info(path: Path) -> dict:
path = Path(path)
return {"path": str(path.resolve()), "exists": path.exists()}
def _safe_dir(name: str) -> str:
safe = "".join(c for c in (name or "") if c not in r'\/:*?"<>|')
return (safe.strip().strip(".") or "unknown")
def _open_in_os(path: Path) -> None:
import os
import sys
import subprocess
p = str(path)
try:
if sys.platform.startswith("win"):
os.startfile(p) # type: ignore[attr-defined]
elif sys.platform == "darwin":
subprocess.Popen(["open", p])
else:
subprocess.Popen(["xdg-open", p])
except Exception as exc:
raise HTTPException(500, f"cannot open folder: {exc}")
Reference in New Issue View Git Blame Copy Permalink