urieljareth/yt-channel-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: batch .md download, multi-select, thumbnails, all OPPORTUNITIES tools

Browse Source 
Videos view: multi-select (checkboxes) + select-all/deselect-all + bulk action
bar (Download .md batch, Download thumbnails, Download audio). Per-video buttons:
.md download (done) or Process to .md (pending), clip.

Channels: per-channel pending count + 'Download pending .md' (batches pending
video_ids). Each channel keeps Videos/.md folder/Remove actions.

Backend (jobs.py + api.py):
- POST /api/scrape/batch {video_ids} + POST /api/scrape/video/{id} (on-demand
  .md for any video incl. pending) as SSE-tracked jobs
- GET /api/videos/{id}/markdown (file download)
- POST /api/tools/thumbnails + GET /api/thumbnails/{id} (local cache, offline)
- GET /api/clip/{id}?from=&to= (transcript segment)
- GET /api/stats (aggregate totals)
- POST /api/tools/audio (mp3 job)

Tools view rebuilt as full Knowledge Base grid surfacing every OPPORTUNITIES.md
feature as electable buttons (md download, re-render, export, search, analysis,
thumbnails, audio, clip, stats, cookies, channels, watch, open folders).

Floating job widget (SSE) for any running job. Thumbnails served via
/api/thumbnails/{id} everywhere (works offline once cached).

Launcher: robust Open-Browser helper (3 fallback methods) auto-opens
http://127.0.0.1:<port> on start + on idempotent re-open. data/ gitignored.
This commit is contained in:
urieljareth
2026-07-26 23:39:05 -06:00
parent 621bbc5f5c
commit 0682d2d806
7 changed files with 852 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/yt_scraper/webapp/jobs.py
+96
View File
@@ -89,6 +89,102 @@ class JobManager:
if not job:
return
opts = json.loads(job.opts_json) if job.opts_json else {}
if opts.get("video_ids"):
self._run_batch(job_id, opts)
return
if opts.get("mode") == "audio":
self._run_audio(job_id, opts)
return
self._run_channel(job_id, opts)
def _resolve_channel_for_video(self, video_id: str) -> tuple[str, str, str]:
row = self.store.get_video(video_id)
if not row:
return ("unknown", "", f"https://www.youtube.com/watch?v={video_id}")
ch = self.store.get_channel(row.channel_id) or {}
name = ch.get("name") or "unknown"
handle = (ch.get("handle") or "").lstrip("@")
url = f"https://www.youtube.com/@{handle}/videos" if handle else row.url
return (name, row.channel_id, url)
def _run_batch(self, job_id: str, opts: dict[str, Any]) -> None:
video_ids = opts.get("video_ids") or []
cookie_path = opts.get("cookies_file") or resolve_active_path(self.store)
total = len(video_ids)
self.store.update_job(job_id, status="running", total=total, completed=0)
self._emit(job_id, "progress", {"completed": 0, "total": total})
self._emit(job_id, "log", {"msg": f"processing {total} videos"})
cfg = _clone_config(self.cfg)
if opts.get("languages"):
cfg.languages = opts["languages"]
completed = 0
for vid in video_ids:
if job_id in self._cancel:
self.store.update_job(job_id, status="cancelled", completed=completed, finished=True)
self._emit(job_id, "cancelled", {"completed": completed, "total": total})
return
row = self.store.get_video(vid)
if not row:
self._emit(job_id, "log", {"msg": f"skip unknown {vid}"})
completed += 1
self.store.update_job(job_id, completed=completed)
continue
channel_name, channel_id, channel_url = self._resolve_channel_for_video(vid)
status = process_video(
row, cfg, self.store, channel_name, channel_id, channel_url,
cookies_file=cookie_path,
on_log=lambda m: self._emit(job_id, "log", {"msg": m}),
)
completed += 1
self.store.update_job(job_id, completed=completed)
self._emit(job_id, "progress", {"completed": completed, "total": total, "video_id": vid, "status": status})
self.store.update_job(job_id, status="done", completed=completed, finished=True)
self._emit(job_id, "done", {"completed": completed, "total": total})
def _run_audio(self, job_id: str, opts: dict[str, Any]) -> None:
import yt_dlp
video_ids = opts.get("video_ids") or []
channel_id = opts.get("channel_id")
cookie_path = opts.get("cookies_file") or resolve_active_path(self.store)
if video_ids:
videos = [self.store.get_video(v) for v in video_ids if self.store.get_video(v)]
else:
videos = [v for v in self.store.get_all(channel_id) if v.status == "done"]
out_dir = Path(self.cfg.output_dir_resolved).parent / "audio"
out_dir.mkdir(parents=True, exist_ok=True)
total = len(videos)
self.store.update_job(job_id, status="running", total=total, completed=0)
self._emit(job_id, "log", {"msg": f"downloading {total} audio tracks -> {out_dir}"})
ydl_opts = {
"format": "bestaudio/best",
"outtmpl": str(out_dir / "%(title)s.%(ext)s"),
"postprocessors": [{"key": "FFmpegExtractAudio", "preferredcodec": "mp3", "preferredquality": "128"}],
"quiet": True, "no_warnings": True, "noprogress": True,
}
if cookie_path:
ydl_opts["cookiefile"] = cookie_path
completed = 0
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
for v in videos:
if job_id in self._cancel:
self.store.update_job(job_id, status="cancelled", completed=completed, finished=True)
self._emit(job_id, "cancelled", {"completed": completed, "total": total})
return
try:
ydl.download([v.url])
self._emit(job_id, "log", {"msg": f"OK {v.video_id}"})
except Exception as exc:
self._emit(job_id, "log", {"msg": f"FAIL {v.video_id}: {exc}"})
completed += 1
self.store.update_job(job_id, completed=completed)
self._emit(job_id, "progress", {"completed": completed, "total": total, "video_id": v.video_id})
self.store.update_job(job_id, status="done", completed=completed, finished=True)
self._emit(job_id, "done", {"completed": completed, "total": total})
def _run_channel(self, job_id: str, opts: dict[str, Any]) -> None:
job = self.store.get_job(job_id)
if not job:
return
channel_id = job.channel_id
cfg = _clone_config(self.cfg)
channel_url = _resolve_channel_url(self.store, cfg, channel_id)