urieljareth
06497299ab
- Video detail (done videos): added 'Download audio' button; removed the
manual 'Thumbnail' button (thumbnails auto-download now, so it was redundant).
- New 'GET /api/videos/{id}/audio' endpoint (also serves HEAD for probing),
serving data/audio/<video_id>.mp3. Audio job outtmpl switched to %(id)s so
files are addressable per video.
- Integrated <audio> player appears in the detail view once the MP3 is present
(probed via HEAD on openVideo; polled after a download job until ready).
- Synced transcript follower: as audio plays, the matching segment is
highlighted (seg-active) and auto-scrolled into view like a karaoke/lyrics
tracker. Clicking any segment timestamp or chapter seeks the audio to that
point (falls back to scroll when no audio). Playback indicator pulses while
playing.
286 lines
12 KiB
Python
286 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from collections import deque
|
|
from typing import Any
|
|
|
|
from ..config import Config, load_config
|
|
from ..cookies import resolve_active_path
|
|
from ..discover import discover_channel
|
|
from ..pipeline import process_video
|
|
from ..ratelimit import polite_sleep
|
|
from ..store import Store
|
|
|
|
|
|
class JobManager:
|
|
"""Single-worker scrape job runner with an in-memory event log per job (SSE-polled)."""
|
|
|
|
def __init__(self, store: Store, cfg: Config):
|
|
self.store = store
|
|
self.cfg = cfg
|
|
self._lock = threading.Lock()
|
|
self._events: dict[str, list[dict]] = {}
|
|
self._cancel: set[str] = set()
|
|
self._queue: deque[str] = deque()
|
|
self._thread = threading.Thread(target=self._worker, daemon=True, name="scrape-worker")
|
|
self._running = False
|
|
|
|
def start(self) -> None:
|
|
if self._running:
|
|
return
|
|
self._running = True
|
|
self._thread.start()
|
|
|
|
def enqueue(self, channel_id: str | None, opts: dict[str, Any]) -> str:
|
|
job_id = uuid.uuid4().hex[:12]
|
|
self.store.create_job(job_id, channel_id, opts)
|
|
with self._lock:
|
|
self._events[job_id] = []
|
|
self._emit(job_id, "queued", {"job_id": job_id, "channel_id": channel_id, "opts": opts})
|
|
self._queue.append(job_id)
|
|
return job_id
|
|
|
|
def cancel(self, job_id: str) -> bool:
|
|
job = self.store.get_job(job_id)
|
|
if not job or job.status not in ("queued", "running"):
|
|
return False
|
|
self._cancel.add(job_id)
|
|
self._emit(job_id, "log", {"msg": "cancel requested"})
|
|
return True
|
|
|
|
def events_since(self, job_id: str, cursor: int) -> list[dict]:
|
|
with self._lock:
|
|
evs = self._events.get(job_id, [])
|
|
return evs[cursor:]
|
|
|
|
def is_terminal(self, job_id: str) -> str | None:
|
|
job = self.store.get_job(job_id)
|
|
if not job:
|
|
return None
|
|
if job.status in ("done", "error", "cancelled"):
|
|
return job.status
|
|
return None
|
|
|
|
# ---------------------------------------------------------- internals
|
|
|
|
def _emit(self, job_id: str, event: str, data: dict) -> None:
|
|
with self._lock:
|
|
self._events.setdefault(job_id, []).append({"event": event, "data": data})
|
|
|
|
def _worker(self) -> None:
|
|
while True:
|
|
try:
|
|
job_id = self._queue.popleft()
|
|
except IndexError:
|
|
time.sleep(0.2)
|
|
continue
|
|
try:
|
|
self._run_job(job_id)
|
|
except Exception as exc:
|
|
self.store.update_job(job_id, status="error", last_error=str(exc), finished=True)
|
|
self._emit(job_id, "error", {"message": str(exc)})
|
|
|
|
def _run_job(self, job_id: str) -> None:
|
|
job = self.store.get_job(job_id)
|
|
if not job:
|
|
return
|
|
opts = json.loads(job.opts_json) if job.opts_json else {}
|
|
if opts.get("video_ids"):
|
|
self._run_batch(job_id, opts)
|
|
return
|
|
if opts.get("mode") == "audio":
|
|
self._run_audio(job_id, opts)
|
|
return
|
|
self._run_channel(job_id, opts)
|
|
|
|
def _resolve_channel_for_video(self, video_id: str) -> tuple[str, str, str]:
|
|
row = self.store.get_video(video_id)
|
|
if not row:
|
|
return ("unknown", "", f"https://www.youtube.com/watch?v={video_id}")
|
|
ch = self.store.get_channel(row.channel_id) or {}
|
|
name = ch.get("name") or "unknown"
|
|
handle = (ch.get("handle") or "").lstrip("@")
|
|
url = f"https://www.youtube.com/@{handle}/videos" if handle else row.url
|
|
return (name, row.channel_id, url)
|
|
|
|
def _run_batch(self, job_id: str, opts: dict[str, Any]) -> None:
|
|
from ..pipeline import process_video as _process_video, cache_thumbnail
|
|
video_ids = opts.get("video_ids") or []
|
|
cookie_path = opts.get("cookies_file") or resolve_active_path(self.store)
|
|
thumb_dir = Path(self.cfg.output_dir_resolved).parent / "thumbnails"
|
|
total = len(video_ids)
|
|
self.store.update_job(job_id, status="running", total=total, completed=0)
|
|
self._emit(job_id, "progress", {"completed": 0, "total": total})
|
|
self._emit(job_id, "log", {"msg": f"processing {total} videos (.md + thumbnails)"})
|
|
cfg = _clone_config(self.cfg)
|
|
if opts.get("languages"):
|
|
cfg.languages = opts["languages"]
|
|
completed = 0
|
|
for vid in video_ids:
|
|
if job_id in self._cancel:
|
|
self.store.update_job(job_id, status="cancelled", completed=completed, finished=True)
|
|
self._emit(job_id, "cancelled", {"completed": completed, "total": total})
|
|
return
|
|
row = self.store.get_video(vid)
|
|
if not row:
|
|
self._emit(job_id, "log", {"msg": f"skip unknown {vid}"})
|
|
completed += 1
|
|
self.store.update_job(job_id, completed=completed)
|
|
continue
|
|
# UX optimization: skip re-extracting videos whose .md already exists — just ensure thumbnail cached.
|
|
md_path = Path(cfg.output_dir_resolved).parent / row.markdown_path if row.markdown_path else None
|
|
if row.status == "done" and row.markdown_path and md_path and md_path.exists():
|
|
cache_thumbnail(self.store, vid, thumb_dir)
|
|
status = "done"
|
|
self._emit(job_id, "log", {"msg": f"cached {vid} (md already present)"})
|
|
else:
|
|
channel_name, channel_id, channel_url = self._resolve_channel_for_video(vid)
|
|
status = _process_video(
|
|
row, cfg, self.store, channel_name, channel_id, channel_url,
|
|
cookies_file=cookie_path,
|
|
on_log=lambda m: self._emit(job_id, "log", {"msg": m}),
|
|
)
|
|
cache_thumbnail(self.store, vid, thumb_dir)
|
|
completed += 1
|
|
self.store.update_job(job_id, completed=completed)
|
|
self._emit(job_id, "progress", {"completed": completed, "total": total, "video_id": vid, "status": status})
|
|
self.store.update_job(job_id, status="done", completed=completed, finished=True)
|
|
self._emit(job_id, "done", {"completed": completed, "total": total})
|
|
|
|
def _run_audio(self, job_id: str, opts: dict[str, Any]) -> None:
|
|
import yt_dlp
|
|
video_ids = opts.get("video_ids") or []
|
|
channel_id = opts.get("channel_id")
|
|
cookie_path = opts.get("cookies_file") or resolve_active_path(self.store)
|
|
if video_ids:
|
|
videos = [self.store.get_video(v) for v in video_ids if self.store.get_video(v)]
|
|
else:
|
|
videos = [v for v in self.store.get_all(channel_id) if v.status == "done"]
|
|
out_dir = Path(self.cfg.output_dir_resolved).parent / "audio"
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
total = len(videos)
|
|
self.store.update_job(job_id, status="running", total=total, completed=0)
|
|
self._emit(job_id, "log", {"msg": f"downloading {total} audio tracks -> {out_dir}"})
|
|
ydl_opts = {
|
|
"format": "bestaudio/best",
|
|
"outtmpl": str(out_dir / "%(id)s.%(ext)s"),
|
|
"postprocessors": [{"key": "FFmpegExtractAudio", "preferredcodec": "mp3", "preferredquality": "128"}],
|
|
"quiet": True, "no_warnings": True, "noprogress": True,
|
|
}
|
|
if cookie_path:
|
|
ydl_opts["cookiefile"] = cookie_path
|
|
completed = 0
|
|
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
|
for v in videos:
|
|
if job_id in self._cancel:
|
|
self.store.update_job(job_id, status="cancelled", completed=completed, finished=True)
|
|
self._emit(job_id, "cancelled", {"completed": completed, "total": total})
|
|
return
|
|
try:
|
|
ydl.download([v.url])
|
|
self._emit(job_id, "log", {"msg": f"OK {v.video_id}"})
|
|
except Exception as exc:
|
|
self._emit(job_id, "log", {"msg": f"FAIL {v.video_id}: {exc}"})
|
|
completed += 1
|
|
self.store.update_job(job_id, completed=completed)
|
|
self._emit(job_id, "progress", {"completed": completed, "total": total, "video_id": v.video_id})
|
|
self.store.update_job(job_id, status="done", completed=completed, finished=True)
|
|
self._emit(job_id, "done", {"completed": completed, "total": total})
|
|
|
|
def _run_channel(self, job_id: str, opts: dict[str, Any]) -> None:
|
|
job = self.store.get_job(job_id)
|
|
if not job:
|
|
return
|
|
channel_id = job.channel_id
|
|
cfg = _clone_config(self.cfg)
|
|
channel_url = _resolve_channel_url(self.store, cfg, channel_id)
|
|
if not channel_url:
|
|
self.store.update_job(job_id, status="error", last_error="no channel url", finished=True)
|
|
self._emit(job_id, "error", {"message": "no channel url"})
|
|
return
|
|
cfg.channel_url = channel_url
|
|
|
|
self.store.update_job(job_id, status="running")
|
|
self._emit(job_id, "log", {"msg": f"discovering {channel_url}"})
|
|
try:
|
|
_channel_id, channel_name, refs = discover_channel(channel_url, sleep_subrequests=cfg.yt_dlp.sleep_subrequests)
|
|
except Exception as exc:
|
|
self.store.update_job(job_id, status="error", last_error=str(exc), finished=True)
|
|
self._emit(job_id, "error", {"message": f"discovery failed: {exc}"})
|
|
return
|
|
|
|
limit = opts.get("limit")
|
|
since = opts.get("since")
|
|
languages = opts.get("languages")
|
|
include_shorts = opts.get("include_shorts", cfg.include_shorts)
|
|
no_live = opts.get("no_live", not cfg.include_live)
|
|
cookie_override = opts.get("cookies_file")
|
|
|
|
if languages:
|
|
cfg.languages = languages
|
|
if since:
|
|
refs = [r for r in refs if (r.upload_date or "") >= since.replace("-", "")]
|
|
if not include_shorts:
|
|
refs = [r for r in refs if "/shorts/" not in (r.url or "")]
|
|
if no_live:
|
|
refs = [r for r in refs if not (r.url or "").startswith("https://www.youtube.com/live/")]
|
|
if limit:
|
|
refs = refs[: int(limit)]
|
|
|
|
self.store.upsert_channel(_channel_id, _handle(channel_url), channel_name, len(refs))
|
|
self.store.upsert_videos(refs)
|
|
|
|
pending = [r for r in self.store.get_pending(_channel_id) if r.video_id in {x.video_id for x in refs}]
|
|
total = len(pending)
|
|
self.store.update_job(job_id, total=total)
|
|
self._emit(job_id, "progress", {"completed": 0, "total": total})
|
|
self._emit(job_id, "log", {"msg": f"{total} videos to process"})
|
|
|
|
cookie_path = cookie_override or resolve_active_path(self.store)
|
|
completed = 0
|
|
for row in pending:
|
|
if job_id in self._cancel:
|
|
self.store.update_job(job_id, status="cancelled", completed=completed, finished=True)
|
|
self._emit(job_id, "cancelled", {"completed": completed, "total": total})
|
|
return
|
|
status = process_video(
|
|
row, cfg, self.store, channel_name, _channel_id, channel_url,
|
|
cookies_file=cookie_path,
|
|
on_log=lambda m: self._emit(job_id, "log", {"msg": m}),
|
|
)
|
|
completed += 1
|
|
self.store.update_job(job_id, completed=completed)
|
|
self._emit(job_id, "progress", {"completed": completed, "total": total, "video_id": row.video_id, "status": status})
|
|
if completed < total:
|
|
polite_sleep(cfg.delay.min_seconds, cfg.delay.max_seconds)
|
|
|
|
self.store.update_job(job_id, status="done", completed=completed, finished=True)
|
|
self._emit(job_id, "done", {"completed": completed, "total": total})
|
|
|
|
|
|
def _clone_config(cfg: Config) -> Config:
|
|
import copy
|
|
return copy.deepcopy(cfg)
|
|
|
|
|
|
def _resolve_channel_url(store: Store, cfg: Config, channel_id: str | None) -> str | None:
|
|
if not channel_id:
|
|
return cfg.channel_url or None
|
|
ch = store.get_channel(channel_id)
|
|
if not ch:
|
|
return None
|
|
handle = (ch.get("handle") or "").lstrip("@")
|
|
if handle:
|
|
return f"https://www.youtube.com/@{handle}/videos"
|
|
return f"https://www.youtube.com/channel/{channel_id}"
|
|
|
|
|
|
def _handle(url: str) -> str:
|
|
if "@" in url:
|
|
return "@" + url.split("@", 1)[1].split("/", 1)[0]
|
|
return ""
|