diff --git a/src/yt_scraper/pipeline.py b/src/yt_scraper/pipeline.py index c960d0a..1cec049 100644 --- a/src/yt_scraper/pipeline.py +++ b/src/yt_scraper/pipeline.py @@ -140,6 +140,32 @@ def _safe_dirname(name: str) -> str: return safe.strip().strip(".") or "unknown" +def thumbnail_url_for(video_row) -> str: + """Thumbnail URL for a video: stored URL, else the canonical YouTube one derived from its id.""" + if video_row and getattr(video_row, "thumbnail", None): + return video_row.thumbnail + vid = getattr(video_row, "video_id", "") if video_row else "" + return f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg" + + +def cache_thumbnail(store, video_id: str, thumbnails_dir) -> bool: + """Download + cache a video's thumbnail to /.jpg. Returns True on success/existing.""" + import requests as _requests + out = Path(thumbnails_dir) / f"{video_id}.jpg" + if out.exists(): + return True + v = store.get_video(video_id) + url = thumbnail_url_for(v) if v else f"https://i.ytimg.com/vi/{video_id}/hqdefault.jpg" + try: + r = _requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"}) + r.raise_for_status() + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(r.content) + return True + except Exception: + return False + + def re_render_videos(store: Store, cfg: Config, channel_id: str | None = None) -> int: """Regenerate .md for done videos that have stored segments. Returns count re-rendered.""" import json diff --git a/src/yt_scraper/webapp/api.py b/src/yt_scraper/webapp/api.py index bc82ee4..89ba3dd 100644 --- a/src/yt_scraper/webapp/api.py +++ b/src/yt_scraper/webapp/api.py @@ -332,41 +332,29 @@ def build_router(store: Store, cfg: Config, jobs) -> APIRouter: # -------------------------------------------------- thumbnails (local cache) @r.post("/tools/thumbnails") async def download_thumbnails(payload: dict): - import requests as _req + from ..pipeline import cache_thumbnail video_ids = (payload or {}).get("video_ids") or [] channel_id = (payload or {}).get("channel_id") if channel_id and not video_ids: - video_ids = [v.video_id for v in store.get_all(channel_id) if v.thumbnail] + video_ids = [v.video_id for v in store.get_all(channel_id)] out_dir = Path(cfg.output_dir_resolved).parent / "thumbnails" out_dir.mkdir(parents=True, exist_ok=True) n = 0 for vid in video_ids: - v = store.get_video(vid) - if not v or not v.thumbnail: - continue - target = out_dir / f"{vid}.jpg" - if target.exists(): + if cache_thumbnail(store, vid, out_dir): n += 1 - continue - try: - r2 = _req.get(v.thumbnail, timeout=15, headers={"User-Agent": "Mozilla/5.0"}) - r2.raise_for_status() - target.write_bytes(r2.content) - n += 1 - except Exception: - continue return {"downloaded": n, "dir": str(out_dir)} @r.get("/thumbnails/{video_id}") def serve_thumbnail(video_id: str): from fastapi.responses import RedirectResponse + from ..pipeline import thumbnail_url_for local = Path(cfg.output_dir_resolved).parent / "thumbnails" / f"{video_id}.jpg" if local.exists(): return FileResponse(str(local), media_type="image/jpeg") v = store.get_video(video_id) - if v and v.thumbnail: - return RedirectResponse(url=v.thumbnail, status_code=302) - raise HTTPException(404, "no thumbnail") + url = thumbnail_url_for(v) if v else f"https://i.ytimg.com/vi/{video_id}/hqdefault.jpg" + return RedirectResponse(url=url, status_code=302) # -------------------------------------------------- clip (transcript segment) @r.get("/clip/{video_id}") diff --git a/src/yt_scraper/webapp/jobs.py b/src/yt_scraper/webapp/jobs.py index 879b3fe..c2b2b47 100644 --- a/src/yt_scraper/webapp/jobs.py +++ b/src/yt_scraper/webapp/jobs.py @@ -108,12 +108,14 @@ class JobManager: return (name, row.channel_id, url) def _run_batch(self, job_id: str, opts: dict[str, Any]) -> None: + from ..pipeline import process_video as _process_video, cache_thumbnail video_ids = opts.get("video_ids") or [] cookie_path = opts.get("cookies_file") or resolve_active_path(self.store) + thumb_dir = Path(self.cfg.output_dir_resolved).parent / "thumbnails" total = len(video_ids) self.store.update_job(job_id, status="running", total=total, completed=0) self._emit(job_id, "progress", {"completed": 0, "total": total}) - self._emit(job_id, "log", {"msg": f"processing {total} videos"}) + self._emit(job_id, "log", {"msg": f"processing {total} videos (.md + thumbnails)"}) cfg = _clone_config(self.cfg) if opts.get("languages"): cfg.languages = opts["languages"] @@ -129,12 +131,20 @@ class JobManager: completed += 1 self.store.update_job(job_id, completed=completed) continue - channel_name, channel_id, channel_url = self._resolve_channel_for_video(vid) - status = process_video( - row, cfg, self.store, channel_name, channel_id, channel_url, - cookies_file=cookie_path, - on_log=lambda m: self._emit(job_id, "log", {"msg": m}), - ) + # UX optimization: skip re-extracting videos whose .md already exists — just ensure thumbnail cached. + md_path = Path(cfg.output_dir_resolved).parent / row.markdown_path if row.markdown_path else None + if row.status == "done" and row.markdown_path and md_path and md_path.exists(): + cache_thumbnail(self.store, vid, thumb_dir) + status = "done" + self._emit(job_id, "log", {"msg": f"cached {vid} (md already present)"}) + else: + channel_name, channel_id, channel_url = self._resolve_channel_for_video(vid) + status = _process_video( + row, cfg, self.store, channel_name, channel_id, channel_url, + cookies_file=cookie_path, + on_log=lambda m: self._emit(job_id, "log", {"msg": m}), + ) + cache_thumbnail(self.store, vid, thumb_dir) completed += 1 self.store.update_job(job_id, completed=completed) self._emit(job_id, "progress", {"completed": completed, "total": total, "video_id": vid, "status": status}) diff --git a/src/yt_scraper/webapp/static/app.js b/src/yt_scraper/webapp/static/app.js index 17626e0..ea9af91 100644 --- a/src/yt_scraper/webapp/static/app.js +++ b/src/yt_scraper/webapp/static/app.js @@ -184,6 +184,12 @@ this.forms.channelUrl = ""; await this.loadChannels(); this.toast("Added " + (d.name || "channel") + " (" + (d.video_count || 0) + " videos)"); + // Auto-cache thumbnails in the background so the catalog looks populated immediately + if (d.channel_id) { + this.api("/api/tools/thumbnails", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ channel_id: d.channel_id }) }) + .then((r) => { if (r && r.downloaded) this.toast((r.downloaded) + " thumbnails cached"); }) + .catch(() => {}); + } } catch (e) { this.forms.channelError = e.message; } finally { this.loading.addChannel = false; } }, @@ -302,7 +308,7 @@ if (!ids.length) { this.toast("No videos selected", "error"); return; } try { const d = await this.api("/api/scrape/batch", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ video_ids: ids }) }); - this.toast("Batch .md job started: " + (d.count || ids.length) + " videos"); + this.toast("Downloading .md + thumbnails for " + (d.count || ids.length) + " videos"); this.subscribeJob(d.job_id, "batch"); if (!videoIds) this.selectNone(); } catch (e) { this.toast("Batch failed: " + e.message, "error"); } diff --git a/src/yt_scraper/webapp/static/index.html b/src/yt_scraper/webapp/static/index.html index 083cdda..14671af 100644 --- a/src/yt_scraper/webapp/static/index.html +++ b/src/yt_scraper/webapp/static/index.html @@ -213,13 +213,9 @@