urieljareth
0866c92650
- Bulk 'Download .md' now also caches thumbnails (one action does both); removed the redundant standalone Thumbnails button from the bulk action bar. - Scrape UX optimization: batch job skips videos whose .md already exists (just caches their thumbnail) instead of re-extracting. - Auto-thumbnails: on addChannel, thumbnails cache in the background so a newly-attached channel's catalog looks populated immediately, even before any .md is generated. - Thumbnail fallback: pending videos without a stored thumbnail URL use the canonical https://i.ytimg.com/vi/<id>/hqdefault.jpg, so every video shows a thumbnail (and /api/thumbnails/{id} never 404s — redirects if uncached). - New helpers in pipeline.py: thumbnail_url_for() + cache_thumbnail().
208 lines
7.6 KiB
Python
208 lines
7.6 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
|
|
from .chapters import align_chapters, chapters_from_info
|
|
from .config import Config
|
|
from .extract import extract_video
|
|
from .render import build_filename_stem, render_markdown
|
|
from .store import Store, VideoRow
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def process_video(
|
|
row: VideoRow,
|
|
cfg: Config,
|
|
store: Store,
|
|
channel_name: str,
|
|
channel_id: str,
|
|
channel_url: str,
|
|
cookies_file: str | None = None,
|
|
cookies_from_browser: str | None = None,
|
|
on_log: Callable[[str], None] | None = None,
|
|
) -> str:
|
|
"""Extract + parse + store + render a single video. Returns final status string."""
|
|
|
|
def _emit(msg: str) -> None:
|
|
log.info(msg)
|
|
if on_log:
|
|
on_log(msg)
|
|
|
|
try:
|
|
data = extract_video(
|
|
row.url,
|
|
cfg.languages,
|
|
retries=cfg.yt_dlp.retries,
|
|
sleep_subrequests=cfg.yt_dlp.sleep_subrequests,
|
|
prefer_manual=cfg.prefer_manual,
|
|
cookies_file=cookies_file,
|
|
cookies_from_browser=cookies_from_browser,
|
|
)
|
|
except Exception as exc:
|
|
log.error("Error extrayendo %s: %s", row.video_id, exc)
|
|
store.mark_error(row.video_id, str(exc))
|
|
return "error"
|
|
|
|
if not data.segments:
|
|
log.warning("Sin transcripcion para %s", row.video_id)
|
|
store.mark_status(row.video_id, "no_subtitles")
|
|
return "no_subtitles"
|
|
|
|
chapters = chapters_from_info(data.info)
|
|
sections = align_chapters(data.segments, chapters)
|
|
|
|
info = data.info
|
|
|
|
# persist segments + rich metadata to DB (for search, stats, webapp)
|
|
store.store_segments(row.video_id, data.segments)
|
|
seg_json = json.dumps(
|
|
[{"start": s.start, "end": s.end, "text": s.text} for s in data.segments],
|
|
ensure_ascii=False,
|
|
)
|
|
ch_json = json.dumps(
|
|
[{"title": c.title, "start": c.start_time, "end": c.end_time} for c in chapters],
|
|
ensure_ascii=False,
|
|
)
|
|
store.update_video_metadata(
|
|
row.video_id,
|
|
view_count=info.get("view_count"),
|
|
like_count=info.get("like_count"),
|
|
tags=info.get("tags") or None,
|
|
thumbnail=info.get("thumbnail"),
|
|
description=(info.get("description") or "").strip() or None,
|
|
chapters_json=ch_json,
|
|
segments_json=seg_json,
|
|
)
|
|
|
|
# ensure upload_date is populated (discovery sometimes lacks it)
|
|
ud = info.get("upload_date") or row.upload_date
|
|
if ud:
|
|
store.set_upload_date(row.video_id, ud.replace("-", "") if "-" in str(ud) else str(ud))
|
|
|
|
context = {
|
|
"video_id": row.video_id,
|
|
"title": info.get("title") or row.title or row.video_id,
|
|
"channel_name": info.get("channel") or channel_name,
|
|
"channel_id": channel_id,
|
|
"channel_url": channel_url,
|
|
"upload_date": _normalize_date(info.get("upload_date") or row.upload_date),
|
|
"duration": info.get("duration") or row.duration or 0,
|
|
"url": row.url,
|
|
"transcript_lang": data.subtitle.lang if data.subtitle else "",
|
|
"transcript_src": data.subtitle.source if data.subtitle else "",
|
|
"view_count": info.get("view_count"),
|
|
"like_count": info.get("like_count"),
|
|
"tags": info.get("tags") or [],
|
|
"thumbnail": info.get("thumbnail") or "",
|
|
"description": (info.get("description") or "").strip(),
|
|
"sections": sections,
|
|
}
|
|
|
|
stem = build_filename_stem(
|
|
upload_date=context["upload_date"],
|
|
title=context["title"],
|
|
template=cfg.filename_template,
|
|
)
|
|
out_subdir = Path(cfg.output_dir_resolved) / _safe_dirname(channel_name)
|
|
md_path = render_markdown(cfg.template_path_resolved, out_subdir, stem, context)
|
|
|
|
out_root = Path(cfg.output_dir_resolved).parent
|
|
try:
|
|
rel = md_path.relative_to(out_root) if md_path.is_relative_to(out_root) else md_path
|
|
except ValueError:
|
|
rel = md_path
|
|
store.mark_done(
|
|
row.video_id,
|
|
str(rel),
|
|
data.subtitle.lang if data.subtitle else None,
|
|
data.subtitle.source if data.subtitle else None,
|
|
data.has_chapters,
|
|
)
|
|
_emit(f"OK {row.video_id} -> {md_path.name}")
|
|
return "done"
|
|
|
|
|
|
def _normalize_date(d: str | None) -> str:
|
|
if not d:
|
|
return ""
|
|
if len(d) == 8:
|
|
return f"{d[:4]}-{d[4:6]}-{d[6:8]}"
|
|
return d
|
|
|
|
|
|
def _safe_dirname(name: str) -> str:
|
|
safe = "".join(c for c in name if c not in r'\/:*?"<>|')
|
|
return safe.strip().strip(".") or "unknown"
|
|
|
|
|
|
def thumbnail_url_for(video_row) -> str:
|
|
"""Thumbnail URL for a video: stored URL, else the canonical YouTube one derived from its id."""
|
|
if video_row and getattr(video_row, "thumbnail", None):
|
|
return video_row.thumbnail
|
|
vid = getattr(video_row, "video_id", "") if video_row else ""
|
|
return f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"
|
|
|
|
|
|
def cache_thumbnail(store, video_id: str, thumbnails_dir) -> bool:
|
|
"""Download + cache a video's thumbnail to <thumbnails_dir>/<video_id>.jpg. Returns True on success/existing."""
|
|
import requests as _requests
|
|
out = Path(thumbnails_dir) / f"{video_id}.jpg"
|
|
if out.exists():
|
|
return True
|
|
v = store.get_video(video_id)
|
|
url = thumbnail_url_for(v) if v else f"https://i.ytimg.com/vi/{video_id}/hqdefault.jpg"
|
|
try:
|
|
r = _requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"})
|
|
r.raise_for_status()
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_bytes(r.content)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def re_render_videos(store: Store, cfg: Config, channel_id: str | None = None) -> int:
|
|
"""Regenerate .md for done videos that have stored segments. Returns count re-rendered."""
|
|
import json
|
|
from .chapters import align_chapters, Chapter
|
|
from .parse import Segment
|
|
from .render import build_filename_stem, render_markdown
|
|
|
|
videos = [v for v in store.get_all(channel_id) if v.status == "done" and v.segments_json]
|
|
n = 0
|
|
for v in videos:
|
|
try:
|
|
segs = [Segment(start=s["start"], end=s["end"], text=s["text"]) for s in json.loads(v.segments_json)]
|
|
chapters = [
|
|
Chapter(title=c["title"], start_time=c["start"], end_time=c.get("end", c["start"]))
|
|
for c in json.loads(v.chapters_json or "[]")
|
|
]
|
|
sections = align_chapters(segs, chapters)
|
|
ch = store.get_channel(v.channel_id) or {}
|
|
tags = []
|
|
if v.tags:
|
|
try:
|
|
tags = json.loads(v.tags)
|
|
except Exception:
|
|
tags = []
|
|
context = {
|
|
"video_id": v.video_id, "title": v.title or v.video_id,
|
|
"channel_name": ch.get("name") or "", "channel_id": v.channel_id, "channel_url": "",
|
|
"upload_date": v.upload_date or "", "duration": v.duration or 0, "url": v.url,
|
|
"transcript_lang": v.transcript_lang or "", "transcript_src": v.transcript_src or "",
|
|
"view_count": v.view_count, "like_count": v.like_count, "tags": tags,
|
|
"thumbnail": v.thumbnail or "", "description": v.description or "", "sections": sections,
|
|
}
|
|
stem = build_filename_stem(v.upload_date, v.title or v.video_id, cfg.filename_template)
|
|
out_subdir = Path(cfg.output_dir_resolved) / _safe_dirname(ch.get("name") or "unknown")
|
|
render_markdown(cfg.template_path_resolved, out_subdir, stem, context)
|
|
n += 1
|
|
except Exception as exc:
|
|
log.warning("re-render failed for %s: %s", v.video_id, exc)
|
|
return n
|