357 lines
14 KiB
Python
357 lines
14 KiB
Python
"""Recovery paths: retryable statuses, recorded skip reasons, disk<->DB reconcile.
|
|
|
|
The motivating incident: 511 videos were stored as `no_subtitles` because the
|
|
language policy was manual-only while the channel publishes only auto-generated
|
|
captions. `reset_errors` could not reach them, and nothing recorded why they
|
|
were skipped, so the failure was both invisible and irreversible.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from yt_scraper.extract import describe_missing_subtitle
|
|
from yt_scraper.segments import reconcile_markdown
|
|
from yt_scraper.store import Store, VideoRef
|
|
|
|
|
|
def _store(tmp_path) -> Store:
|
|
store = Store(tmp_path / "state.db")
|
|
store.upsert_channel("UC1", "@alpha", "Alpha", 0)
|
|
return store
|
|
|
|
|
|
# ---------------------------------------------------------------- reset
|
|
|
|
def test_reset_reaches_no_subtitles_not_just_error(tmp_path):
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([
|
|
VideoRef("a", "UC1", "A", "https://y/watch?v=a"),
|
|
VideoRef("b", "UC1", "B", "https://y/watch?v=b"),
|
|
VideoRef("c", "UC1", "C", "https://y/watch?v=c"),
|
|
])
|
|
store.mark_status("a", "no_subtitles", "policy rejected auto captions")
|
|
store.mark_error("b", "rate limited")
|
|
store.mark_done("c", "markdown/c.md", "es", "auto", False)
|
|
|
|
n = store.reset_videos("UC1", ("error", "no_subtitles"))
|
|
|
|
assert n == 2
|
|
assert store.get_video("a").status == "pending"
|
|
assert store.get_video("b").status == "pending"
|
|
assert store.get_video("c").status == "done", "finished work must not be re-queued"
|
|
|
|
|
|
def test_reset_never_touches_done_even_if_asked(tmp_path):
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("c", "UC1", "C", "https://y/watch?v=c")])
|
|
store.mark_done("c", "markdown/c.md", "es", "auto", False)
|
|
|
|
assert store.reset_videos("UC1", ("done",)) == 0
|
|
assert store.get_video("c").status == "done"
|
|
|
|
|
|
def test_reset_errors_still_only_resets_errors(tmp_path):
|
|
"""The narrower legacy helper keeps its old meaning."""
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([
|
|
VideoRef("a", "UC1", "A", "https://y/watch?v=a"),
|
|
VideoRef("b", "UC1", "B", "https://y/watch?v=b"),
|
|
])
|
|
store.mark_status("a", "no_subtitles")
|
|
store.mark_error("b", "boom")
|
|
|
|
assert store.reset_errors("UC1") == 1
|
|
assert store.get_video("a").status == "no_subtitles"
|
|
assert store.get_video("b").status == "pending"
|
|
|
|
|
|
def test_permanent_failures_are_excluded_from_retries(tmp_path):
|
|
"""Members-only videos cannot be fixed by retrying; re-running them only
|
|
spends requests the recoverable videos need."""
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([
|
|
VideoRef(v, "UC1", v, f"https://y/watch?v={v}") for v in ("members", "throttled", "priv")
|
|
])
|
|
store.mark_error("members", "ERROR: [youtube] x: Join this channel to get access to members-only content")
|
|
store.mark_error("throttled", "ERROR: Video unavailable. The current session has been rate-limited by YouTube")
|
|
store.mark_error("priv", "ERROR: Private video. Sign in if you've been granted access")
|
|
|
|
counts = store.retryable_counts("UC1")
|
|
assert counts["error"] == 1, "only the throttled one is worth retrying"
|
|
assert counts["permanent"] == 2
|
|
|
|
assert store.reset_videos("UC1", ("error",)) == 1
|
|
assert store.get_video("throttled").status == "pending"
|
|
assert store.get_video("members").status == "error"
|
|
assert store.get_video("priv").status == "error"
|
|
|
|
|
|
def test_permanent_failures_can_be_reset_when_explicitly_asked(tmp_path):
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("members", "UC1", "M", "https://y/watch?v=members")])
|
|
store.mark_error("members", "ERROR: members-only content")
|
|
|
|
assert store.reset_videos("UC1", ("error",)) == 0
|
|
assert store.reset_videos("UC1", ("error",), include_permanent=True) == 1
|
|
assert store.get_video("members").status == "pending"
|
|
|
|
|
|
def test_rate_limit_wording_is_never_treated_as_permanent(tmp_path):
|
|
"""The throttling message is the one that must stay retryable."""
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("t", "UC1", "T", "https://y/watch?v=t")])
|
|
store.mark_error(
|
|
"t",
|
|
"ERROR: [youtube] t: Video unavailable. This content isn't available, try again later. "
|
|
"The current session has been rate-limited by YouTube for up to an hour.",
|
|
)
|
|
|
|
assert store.retryable_counts("UC1")["permanent"] == 0
|
|
assert store.reset_videos("UC1", ("error",)) == 1
|
|
|
|
|
|
def test_retryable_counts_reports_both_statuses(tmp_path):
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([
|
|
VideoRef(v, "UC1", v, f"https://y/watch?v={v}") for v in ("a", "b", "c")
|
|
])
|
|
store.mark_status("a", "no_subtitles")
|
|
store.mark_status("b", "no_subtitles")
|
|
store.mark_error("c", "boom")
|
|
|
|
assert store.retryable_counts("UC1") == {"error": 1, "no_subtitles": 2, "permanent": 0}
|
|
|
|
|
|
# ---------------------------------------------------------------- skip reasons
|
|
|
|
def test_mark_status_records_the_reason(tmp_path):
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("a", "UC1", "A", "https://y/watch?v=a")])
|
|
|
|
store.mark_status("a", "no_subtitles", "no track matched the language policy")
|
|
|
|
assert "language policy" in store.get_video("a").error_msg
|
|
|
|
|
|
def test_describe_distinguishes_no_captions_from_policy_rejection():
|
|
none_at_all = describe_missing_subtitle({"subtitles": {}, "automatic_captions": {}}, {"es": "manual"})
|
|
assert "no caption tracks published" in none_at_all
|
|
|
|
auto_only = describe_missing_subtitle(
|
|
{"subtitles": {}, "automatic_captions": {"es": [{"url": "u", "ext": "json3"}]}},
|
|
{"es": "manual"},
|
|
)
|
|
assert "ONLY auto-generated" in auto_only
|
|
assert "'any' or 'auto'" in auto_only
|
|
|
|
|
|
def test_describe_does_not_blame_config_when_mode_already_allows_auto():
|
|
msg = describe_missing_subtitle(
|
|
{"subtitles": {}, "automatic_captions": {"de": [{"url": "u", "ext": "json3"}]}},
|
|
{"es": "any"},
|
|
)
|
|
assert "ONLY auto-generated" not in msg
|
|
assert "no track matched the language policy" in msg
|
|
|
|
|
|
# ---------------------------------------------------------------- reconcile
|
|
|
|
_MD = """---
|
|
video_id: "{vid}"
|
|
title: "T"
|
|
upload_date: "2026-01-02"
|
|
---
|
|
|
|
## Transcript
|
|
|
|
**00:00** · hola mundo
|
|
"""
|
|
|
|
|
|
def test_reconcile_marks_done_when_the_md_is_already_on_disk(tmp_path):
|
|
"""The exact symptom the user reported: a .md exists but the row still
|
|
shows a failure, and nothing reconciles it without a server restart."""
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("vid1", "UC1", "T", "https://y/watch?v=vid1")])
|
|
store.mark_status("vid1", "no_subtitles", "stale failure")
|
|
|
|
md_root = tmp_path / "markdown" / "Alpha"
|
|
md_root.mkdir(parents=True)
|
|
(md_root / "2026-01-02_t.md").write_text(_MD.format(vid="vid1"), encoding="utf-8")
|
|
|
|
result = reconcile_markdown(store, tmp_path / "markdown")
|
|
|
|
row = store.get_video("vid1")
|
|
assert row.status == "done"
|
|
assert row.markdown_path == "markdown/Alpha/2026-01-02_t.md"
|
|
assert result["repaired_done"] == 1
|
|
|
|
|
|
def test_reconcile_requeues_rows_whose_md_vanished_only_with_prune(tmp_path):
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([
|
|
VideoRef("gone", "UC1", "G", "https://y/watch?v=gone"),
|
|
VideoRef("vid1", "UC1", "T", "https://y/watch?v=vid1"),
|
|
])
|
|
store.mark_done("gone", "markdown/Alpha/nope.md", "es", "auto", False)
|
|
md_root = tmp_path / "markdown" / "Alpha"
|
|
md_root.mkdir(parents=True)
|
|
(md_root / "2026-01-02_t.md").write_text(_MD.format(vid="vid1"), encoding="utf-8")
|
|
|
|
# default is non-destructive
|
|
assert reconcile_markdown(store, tmp_path / "markdown")["missing_md"] == 0
|
|
assert store.get_video("gone").status == "done"
|
|
|
|
result = reconcile_markdown(store, tmp_path / "markdown", prune=True)
|
|
assert result["missing_md"] == 1
|
|
assert store.get_video("gone").status == "pending"
|
|
|
|
|
|
def test_prune_refuses_to_demote_everything_when_the_root_is_empty(tmp_path):
|
|
"""Pointed at a wrong or not-yet-populated markdown root, prune must be a
|
|
no-op rather than wiping every finished video in the database."""
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("c", "UC1", "C", "https://y/watch?v=c")])
|
|
store.mark_done("c", "markdown/Alpha/c.md", "es", "auto", False)
|
|
(tmp_path / "markdown").mkdir()
|
|
|
|
result = reconcile_markdown(store, tmp_path / "markdown", prune=True)
|
|
|
|
assert result["missing_md"] == 0
|
|
assert store.get_video("c").status == "done"
|
|
|
|
|
|
def test_reconcile_is_idempotent(tmp_path):
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("vid1", "UC1", "T", "https://y/watch?v=vid1")])
|
|
md_root = tmp_path / "markdown" / "Alpha"
|
|
md_root.mkdir(parents=True)
|
|
(md_root / "2026-01-02_t.md").write_text(_MD.format(vid="vid1"), encoding="utf-8")
|
|
|
|
first = reconcile_markdown(store, tmp_path / "markdown")
|
|
second = reconcile_markdown(store, tmp_path / "markdown")
|
|
|
|
assert first["repaired_done"] == 1
|
|
assert second["repaired_done"] == 0
|
|
assert second["missing_md"] == 0
|
|
assert store.get_video("vid1").status == "done"
|
|
|
|
|
|
def test_a_bad_encoding_does_not_abort_the_whole_scan(tmp_path):
|
|
"""UnicodeDecodeError is a ValueError, not an OSError. Letting it escape
|
|
aborted the loop, so one bad file silently hid every later one."""
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([
|
|
VideoRef("v1", "UC1", "A", "https://y/watch?v=v1"),
|
|
VideoRef("v3", "UC1", "C", "https://y/watch?v=v3"),
|
|
])
|
|
md_root = tmp_path / "markdown" / "Alpha"
|
|
md_root.mkdir(parents=True)
|
|
(md_root / "a.md").write_text(_MD.format(vid="v1"), encoding="utf-8")
|
|
(md_root / "b.md").write_bytes(b"---\nvideo_id: \xe9\xe9\xe9\n---\n")
|
|
(md_root / "c.md").write_text(_MD.format(vid="v3"), encoding="utf-8")
|
|
|
|
result = reconcile_markdown(store, tmp_path / "markdown")
|
|
|
|
assert store.get_video("v3").status == "done", "the file after the bad one must still import"
|
|
assert result["repaired_done"] == 2
|
|
|
|
|
|
# ---------------------------------------------------------------- re-render
|
|
|
|
def test_re_render_updates_markdown_path_and_removes_the_old_file(tmp_path):
|
|
"""re-render used a raw compact upload_date while process_video uses the
|
|
hyphenated form, so it wrote a SECOND .md and never told the DB — leaving
|
|
the app serving the older file. Measured on real data: 94 files, 61 rows."""
|
|
from yt_scraper.config import Config
|
|
from yt_scraper.pipeline import re_render_videos
|
|
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("v1", "UC1", "Mi Video", "https://y/watch?v=v1", "20240519", 60)])
|
|
store.update_video_metadata(
|
|
"v1", view_count=1, like_count=1, tags=None, thumbnail=None, description=None,
|
|
chapters_json="[]",
|
|
segments_json='[{"start": 0.0, "end": 2.0, "text": "hola"}]',
|
|
)
|
|
md_root = tmp_path / "markdown"
|
|
old_dir = md_root / "Alpha"
|
|
old_dir.mkdir(parents=True)
|
|
(old_dir / "2024-05-19_mi-video.md").write_text("stale", encoding="utf-8")
|
|
store.mark_done("v1", "markdown/Alpha/2024-05-19_mi-video.md", "es", "auto", False)
|
|
|
|
cfg = Config(
|
|
database_path=str(tmp_path / "state.db"),
|
|
output_dir=str(md_root),
|
|
template_path=str(Path("templates/video.md.j2").resolve()),
|
|
)
|
|
assert re_render_videos(store, cfg) == 1
|
|
|
|
files = sorted(p.name for p in md_root.rglob("*.md"))
|
|
assert len(files) == 1, f"re-render must not leave an orphan beside it: {files}"
|
|
row = store.get_video("v1")
|
|
assert row.markdown_path.replace("\\", "/").endswith(files[0])
|
|
assert (md_root.parent / row.markdown_path).exists()
|
|
assert "stale" not in (md_root.parent / row.markdown_path).read_text(encoding="utf-8")
|
|
|
|
|
|
def test_re_render_is_idempotent(tmp_path):
|
|
from yt_scraper.config import Config
|
|
from yt_scraper.pipeline import re_render_videos
|
|
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("v1", "UC1", "Mi Video", "https://y/watch?v=v1", "20240519", 60)])
|
|
store.update_video_metadata(
|
|
"v1", view_count=None, like_count=None, tags=None, thumbnail=None, description=None,
|
|
chapters_json="[]", segments_json='[{"start": 0.0, "end": 2.0, "text": "hola"}]',
|
|
)
|
|
store.mark_done("v1", "markdown/Alpha/whatever.md", "es", "auto", False)
|
|
cfg = Config(
|
|
database_path=str(tmp_path / "state.db"),
|
|
output_dir=str(tmp_path / "markdown"),
|
|
template_path=str(Path("templates/video.md.j2").resolve()),
|
|
)
|
|
|
|
re_render_videos(store, cfg)
|
|
first = store.get_video("v1").markdown_path
|
|
re_render_videos(store, cfg)
|
|
|
|
assert store.get_video("v1").markdown_path == first
|
|
assert len(list((tmp_path / "markdown").rglob("*.md"))) == 1
|
|
|
|
|
|
def test_reconcile_reports_stale_duplicates_and_deletes_them_only_with_prune(tmp_path):
|
|
"""The 33 leftover files the old re-render wrote under a second filename:
|
|
the DB points at one, the other is dead weight."""
|
|
store = _store(tmp_path)
|
|
store.upsert_videos([VideoRef("vid1", "UC1", "T", "https://y/watch?v=vid1")])
|
|
md_root = tmp_path / "markdown" / "Alpha"
|
|
md_root.mkdir(parents=True)
|
|
canonical = md_root / "2026-01-02_t.md"
|
|
duplicate = md_root / "20260102_t.md"
|
|
canonical.write_text(_MD.format(vid="vid1"), encoding="utf-8")
|
|
duplicate.write_text(_MD.format(vid="vid1"), encoding="utf-8")
|
|
store.mark_done("vid1", "markdown/Alpha/2026-01-02_t.md", "es", "auto", False)
|
|
|
|
result = reconcile_markdown(store, tmp_path / "markdown")
|
|
assert result["stale_dupe"] == 1
|
|
assert duplicate.exists(), "reporting only by default"
|
|
assert store.get_video("vid1").markdown_path == "markdown/Alpha/2026-01-02_t.md"
|
|
|
|
result = reconcile_markdown(store, tmp_path / "markdown", prune=True)
|
|
assert result["stale_dupe"] == 1
|
|
assert not duplicate.exists()
|
|
assert canonical.exists(), "the file the DB points at must survive"
|
|
assert store.get_video("vid1").status == "done"
|
|
|
|
|
|
def test_reconcile_counts_orphan_markdown(tmp_path):
|
|
store = _store(tmp_path)
|
|
md_root = tmp_path / "markdown" / "Alpha"
|
|
md_root.mkdir(parents=True)
|
|
(md_root / "ghost.md").write_text(_MD.format(vid="not-in-db"), encoding="utf-8")
|
|
|
|
result = reconcile_markdown(store, tmp_path / "markdown")
|
|
|
|
assert result["orphan_md"] == 1
|
|
assert result["repaired_done"] == 0
|