feat: local content-mining platform for YouTube creator scraping

Scraper de canales de YouTube hacia notas Markdown para base de conocimiento
(Obsidian-ready), con plataforma web local.

Engine + CLI (Workstream A):
- Modular pipeline: discover/extract/parse/chapters/render/store + ratelimit
- SQLite store con migración idempotente: FTS5 (transcript search), columnas
  de metadata enriquecida, tablas cookies_meta y scrape_jobs
- Módulos: segments, cookies (Netscape vault), export (json/csv/srt/html),
  analysis (word freq/timeline/wordcloud), monitor (watch loop), pipeline
- CLI Click group: search, export, audio, channels, watch, analyze, re-render
- Fix del bug de scoping de cookies en cli.py

Webapp local (Workstream B):
- FastAPI backend: dashboard, channels, videos facetado, transcript, search
  FTS, analysis, scrape jobs con SSE, cookies drag-and-drop, exports,
  folders (abrir en OS), tools (re-render, formato)
- SPA no-build (Alpine.js + Tailwind + Chart.js por CDN): 9 vistas, tema
  dark command-center con acento rojo→rosa, cookie vault drag-drop,
  consola de scrapeo con progreso live vía SSE

Launcher + subagentes (Workstream C):
- start-server.bat / stop-server.bat con auto port-scan + browser open
- .opencode/agent/webapp-builder.md + .opencode/goals/webapp-build.md

Tests: 38 pytest verdes. Sin funcionalidad de IA (enfoque data-mining).

tests/fixtures/sample.json3

@@ -0,0 +1,34 @@
+{
+  "events": [
+    {
+      "tStartMs": 0,
+      "dDurationMs": 2000,
+      "segs": [
+        {"utf8": "Hola "},
+        {"utf8": "mundo"}
+      ]
+    },
+    {
+      "tStartMs": 2000,
+      "dDurationMs": 3000,
+      "segs": [
+        {"utf8": "Esta es "},
+        {"utf8": "la segunda frase"}
+      ]
+    },
+    {
+      "tStartMs": 5000,
+      "dDurationMs": 0,
+      "segs": [
+        {"utf8": "\n"}
+      ]
+    },
+    {
+      "tStartMs": 6000,
+      "dDurationMs": 2000,
+      "segs": [
+        {"utf8": "Tercera frase"}
+      ]
+    }
+  ]
+}

tests/fixtures/sample.vtt

@@ -0,0 +1,10 @@
+WEBVTT
+
+00:00:00.000 --> 00:00:02.500
+Hola desde VTT
+
+00:00:02.500 --> 00:00:05.000
+Segundo bloque <c.colorE1E1E1>con tag</c>
+
+00:00:06.000 --> 00:00:09.000
+Tercer bloque final

tests/test_chapters.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from yt_scraper.chapters import align_chapters, chapters_from_info, Chapter, Section
+from yt_scraper.parse import Segment
+
+
+def test_align_no_chapters():
+    segs = [Segment(0.0, 1.0, "a"), Segment(1.0, 2.0, "b")]
+    sections = align_chapters(segs, [])
+    assert len(sections) == 1
+    assert sections[0].title == "Transcripcion"
+    assert len(sections[0].segments) == 2
+
+
+def test_align_with_chapters():
+    segs = [
+        Segment(0.0, 1.0, "intro"),
+        Segment(5.0, 6.0, "mid"),
+        Segment(20.0, 21.0, "end"),
+    ]
+    chapters = [
+        Chapter(title="Intro", start_time=0.0, end_time=10.0),
+        Chapter(title="Desarrollo", start_time=10.0, end_time=30.0),
+    ]
+    sections = align_chapters(segs, chapters)
+    assert len(sections) == 2
+    assert sections[0].title == "Intro"
+    assert len(sections[0].segments) == 2
+    assert sections[1].title == "Desarrollo"
+    assert len(sections[1].segments) == 1
+    assert sections[1].segments[0].text == "end"
+
+
+def test_align_pre_chapter_segments():
+    segs = [Segment(0.0, 1.0, "antes")]
+    chapters = [Chapter(title="Cap 1", start_time=5.0, end_time=10.0)]
+    sections = align_chapters(segs, chapters)
+    assert len(sections) == 2
+    assert sections[0].title == "Inicio"
+    assert sections[0].segments[0].text == "antes"
+
+
+def test_chapters_from_info():
+    info = {
+        "chapters": [
+            {"title": "Cap 1", "start_time": 0, "end_time": 60},
+            {"title": "Cap 2", "start_time": 60, "end_time": 120},
+        ]
+    }
+    chapters = chapters_from_info(info)
+    assert len(chapters) == 2
+    assert chapters[0].title == "Cap 1"
+    assert chapters[1].start_time == 60.0
+
+
+def test_chapters_from_info_empty():
+    assert chapters_from_info({}) == []
+    assert chapters_from_info({"chapters": []}) == []

tests/test_features.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import csv
+import io
+import json
+from pathlib import Path
+
+import pytest
+
+from yt_scraper.store import Store, VideoRef
+from yt_scraper.parse import Segment
+from yt_scraper import segments, export, analysis
+from yt_scraper.render import build_filename_stem
+
+
+SAMPLE_MD = """---
+video_id: abc123
+title: "Vaporwave and nostalgia"
+channel: Alpha
+upload_date: 20240115
+duration: 600
+views: 1234
+likes: 56
+tags: ["vaporwave", "nostalgia"]
+thumbnail: http://img.test/abc.jpg
+---
+
+# Vaporwave and nostalgia
+
+> [Ver en YouTube](https://www.youtube.com/watch?v=abc123)
+
+## Transcripcion
+
+### Intro (00:00)
+
+**00:15** · Bienvenidos al video sobre vaporwave.
+**00:32** · El vaporwave es una estetica muy interesante.
+
+### Historia (01:15)
+
+**01:15** · El vaporwave nacio en los 2010s.
+**02:40** · dragon ball es un anime clasico.
+"""
+
+
+@pytest.fixture()
+def store(tmp_path):
+    s = Store(tmp_path / "t.db")
+    s.upsert_channel("UC1", "@alpha", "Alpha", 1)
+    s.upsert_videos([VideoRef("abc123", "UC1", "Vaporwave and nostalgia", "https://y/watch?v=abc123", "20240115", 600)])
+    s.mark_done("abc123", "data/markdown/Alpha/x.md", "es", "auto", True)
+    return s
+
+
+def test_parse_markdown_metadata():
+    parsed = segments.parse_markdown(SAMPLE_MD)
+    assert parsed.metadata["video_id"] == "abc123"
+    assert parsed.metadata["title"] == "Vaporwave and nostalgia"
+    assert parsed.metadata["channel"] == "Alpha"
+    assert len(parsed.segments) == 4
+    assert abs(parsed.segments[0].start - 15.0) < 0.01
+    assert "vaporwave" in parsed.segments[0].text.lower()
+    chapters = parsed.chapters
+    assert chapters[0]["title"] == "Intro"
+    assert chapters[1]["title"] == "Historia"
+
+
+def test_backfill_from_markdown(store, tmp_path):
+    md = tmp_path / "md" / "Alpha" / "vapor.md"
+    md.parent.mkdir(parents=True)
+    md.write_text(SAMPLE_MD, encoding="utf-8")
+    n = segments.backfill_from_markdown(store, tmp_path / "md")
+    assert n == 1
+    v = store.get_video("abc123")
+    assert v.view_count == 1234
+    assert v.like_count == 56
+    assert json.loads(v.tags) == ["vaporwave", "nostalgia"]
+    assert v.thumbnail == "http://img.test/abc.jpg"
+    assert v.segments_json is not None
+    segs = store.get_segments("abc123")
+    assert len(segs) == 4
+    # search now works
+    hits = store.search_segments("vaporwave")
+    assert len(hits) >= 1
+    # idempotent
+    n2 = segments.backfill_from_markdown(store, tmp_path / "md")
+    assert n2 == 0
+
+
+def test_ts_helpers():
+    assert segments.ts_to_seconds("01:30") == 90.0
+    assert segments.ts_to_seconds("1:02:03") == 3723.0
+
+
+def test_export_json_csv(store, tmp_path):
+    store.store_segments("abc123", [Segment(0.0, 2.0, "hello vaporwave")])
+    store.update_video_metadata("abc123", view_count=1234, tags=["vaporwave"])
+    j = export.export_json(store, "UC1", tmp_path / "out.json")
+    data = json.loads(j.read_text(encoding="utf-8"))
+    assert len(data["videos"]) == 1
+    assert data["videos"][0]["video_id"] == "abc123"
+
+    c = export.export_csv(store, "UC1", tmp_path / "out.csv")
+    with open(c, encoding="utf-8") as f:
+        reader = csv.DictReader(f)
+        rows = list(reader)
+    assert rows[0]["video_id"] == "abc123"
+    assert rows[0]["view_count"] == "1234"
+
+
+def test_export_srt(store, tmp_path):
+    segs = [Segment(1.5, 3.5, "first line"), Segment(5.0, 7.0, "second line")]
+    store.store_segments("abc123", segs)
+    out = export.export_srt_video(store, "abc123", tmp_path / "x.srt")
+    assert out is not None
+    text = out.read_text(encoding="utf-8")
+    assert "1\n" in text
+    assert "00:00:01,500 --> 00:00:03,500" in text
+    assert "first line" in text
+    assert "second line" in text
+
+
+def test_export_html(store, tmp_path):
+    store.update_video_metadata("abc123", thumbnail="http://t.test/x.jpg", tags=["vaporwave"])
+    out = export.export_html(store, "UC1", tmp_path / "g.html")
+    html = out.read_text(encoding="utf-8")
+    assert "abc123" in html
+    assert "Gallery" in html
+
+
+def test_word_frequency(store):
+    segs = [
+        Segment(0, 1, "vaporwave vaporwave nostalgia nostalgia nostalgia"),
+        Segment(2, 3, "dragon dragon anime"),
+    ]
+    store.store_segments("abc123", segs)
+    freq = analysis.word_frequency(store, "UC1", top=5)
+    assert freq["nostalgia"] == 3
+    assert freq["vaporwave"] == 2
+    assert freq["dragon"] == 2
+    # stopwords filtered
+    assert "y" not in freq
+    assert "the" not in freq
+
+
+def test_term_timeline(store):
+    segs = [Segment(0, 1, "vaporwave mentioned here")]
+    store.store_segments("abc123", segs)
+    tl = analysis.term_timeline(store, "vaporwave", "UC1")
+    assert len(tl) == 1
+    assert tl[0][0] == "2024-01"
+
+
+def test_build_filename_stem():
+    # callers pass already-normalized dates; stem uses them verbatim
+    stem = build_filename_stem("2024-01-01", "Hola Mundo!", "{upload_date}_{slug}")
+    assert stem == "2024-01-01_hola-mundo"

tests/test_parse.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from yt_scraper.parse import Segment, parse_json3, parse_vtt, merge_adjacent
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def test_parse_json3_basic():
+    raw = (FIXTURES / "sample.json3").read_text(encoding="utf-8")
+    segs = parse_json3(raw)
+    assert len(segs) >= 2
+    assert segs[0].start == 0.0
+    assert "Hola" in segs[0].text
+    assert "segunda" in segs[0].text
+    assert segs[-1].start == 6.0
+    assert "Tercera" in segs[-1].text
+
+
+def test_parse_json3_skips_empty_events():
+    raw = (FIXTURES / "sample.json3").read_text(encoding="utf-8")
+    segs = parse_json3(raw)
+    texts = [s.text for s in segs]
+    assert all(t.strip() for t in texts)
+
+
+def test_parse_vtt_basic():
+    raw = (FIXTURES / "sample.vtt").read_text(encoding="utf-8")
+    segs = parse_vtt(raw)
+    assert len(segs) == 2
+    assert segs[0].start == 0.0
+    assert "Hola" in segs[0].text
+    assert "tag" in segs[0].text
+    assert segs[1].start == 6.0
+    assert "Tercer" in segs[1].text
+
+
+def test_parse_vtt_strips_tags():
+    raw = (FIXTURES / "sample.vtt").read_text(encoding="utf-8")
+    segs = parse_vtt(raw)
+    assert "<c." not in segs[1].text
+
+
+def test_merge_adjacent():
+    segs = [
+        Segment(start=0.0, end=1.0, text="a"),
+        Segment(start=1.0, end=2.0, text="b"),
+        Segment(start=5.0, end=6.0, text="c"),
+    ]
+    merged = merge_adjacent(segs, max_gap=0.0)
+    assert len(merged) == 2
+    assert "a b" in merged[0].text
+    assert merged[1].text == "c"
+
+
+def test_parse_json3_invalid_json():
+    assert parse_json3("not json") == []
+    assert parse_json3('{"events": []}') == []

tests/test_render.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from yt_scraper.render import build_filename_stem, format_timestamp, quote_yaml
+from yt_scraper.chapters import Section
+from yt_scraper.parse import Segment
+
+TEMPLATES = Path(__file__).parent.parent / "templates"
+
+
+def test_format_timestamp_minutes():
+    assert format_timestamp(0.0) == "00:00"
+    assert format_timestamp(65.0) == "01:05"
+    assert format_timestamp(3599.0) == "59:59"
+
+
+def test_format_timestamp_hours():
+    assert format_timestamp(3661.0) == "01:01:01"
+
+
+def test_quote_yaml_plain():
+    assert quote_yaml("hola") == "hola"
+
+
+def test_quote_yaml_special():
+    result = quote_yaml('hello "world"')
+    assert result.startswith('"')
+    assert result.endswith('"')
+    assert '\\"' in result
+
+
+def test_quote_yaml_none():
+    assert quote_yaml(None) == '""'
+
+
+def test_build_filename_stem():
+    stem = build_filename_stem("20240115", "Mi Video Genial!")
+    assert stem.startswith("20240115")
+    assert "mi-video-genial" in stem.lower()
+
+
+def test_build_filename_stem_no_date():
+    stem = build_filename_stem(None, "Titulo")
+    assert "unknown" in stem
+    assert "titulo" in stem.lower()
+
+
+def test_render_markdown(tmp_path):
+    template_file = tmp_path / "test.j2"
+    template_file.write_text(
+        "---\ntitle: {{ title | quote_yaml }}\n---\n# {{ title }}\n\n"
+        "{% for s in sections %}## {{ s.title }}\n{% for seg in s.segments %}**{{ seg.start | format_timestamp }}** {{ seg.text }}\n{% endfor %}{% endfor %}",
+        encoding="utf-8",
+    )
+    out_dir = tmp_path / "out"
+    sections = [
+        Section(
+            title="Intro",
+            start=0.0,
+            segments=[Segment(start=0.0, end=1.0, text="Hola"), Segment(start=5.0, end=6.0, text="Mundo")],
+        )
+    ]
+    from yt_scraper.render import render_markdown
+
+    md_path = render_markdown(
+        template_file,
+        out_dir,
+        "20240115_test",
+        {"title": "Test Video", "sections": sections},
+    )
+    assert md_path.exists()
+    content = md_path.read_text(encoding="utf-8")
+    assert "# Test Video" in content
+    assert "## Intro" in content
+    assert "**00:00** Hola" in content
+    assert "**00:05** Mundo" in content

tests/test_store_platform.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from yt_scraper.store import Store, VideoRef
+from yt_scraper.parse import Segment
+
+
+@pytest.fixture()
+def store(tmp_path):
+    return Store(tmp_path / "test.db")
+
+
+@pytest.fixture()
+def seeded_store(store):
+    store.upsert_channel("UC1", "@alpha", "Alpha", 2)
+    store.upsert_channel("UC2", "@beta", "Beta", 1)
+    refs = [
+        VideoRef("v1", "UC1", "Hello world vaporwave", "https://y/watch?v=v1", "20240101", 120),
+        VideoRef("v2", "UC1", "Dragon ball nostalgia", "https://y/watch?v=v2", "20240201", 200),
+        VideoRef("v3", "UC2", "Vaporwave aesthetics", "https://y/watch?v=v3", "20240301", 300),
+    ]
+    store.upsert_videos(refs)
+    return store
+
+
+def test_migration_idempotent(tmp_path):
+    db = tmp_path / "a.db"
+    s1 = Store(db)
+    s2 = Store(db)  # second instantiation should not error
+    # new columns exist
+    cols = {r["name"] for r in s1._connect().execute("PRAGMA table_info(videos)")}
+    for c in ("view_count", "tags", "segments_json", "chapters_json", "thumbnail"):
+        assert c in cols
+    # new tables exist
+    names = {r["name"] for r in s1._connect().execute("SELECT name FROM sqlite_master WHERE type='table'")}
+    for t in ("transcript_segments", "cookies_meta", "scrape_jobs"):
+        assert t in names
+    # fts table
+    fts = {r["name"] for r in s1._connect().execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%fts%'")}
+    assert "transcript_fts" in fts
+
+
+def test_store_and_search_segments(seeded_store):
+    store = seeded_store
+    segs = [
+        Segment(0.0, 2.0, "vaporwave is an aesthetic"),
+        Segment(3.0, 5.0, "dragon ball is a classic anime"),
+    ]
+    store.store_segments("v1", segs)
+    store.store_segments("v3", [Segment(0.0, 2.0, "retro vaporwave vibes")])
+
+    hits = store.search_segments("vaporwave")
+    assert len(hits) == 2
+    vids = {h.video_id for h in hits}
+    assert vids == {"v1", "v3"}
+
+    # channel-scoped search
+    hits_c = store.search_segments("vaporwave", channel_id="UC2")
+    assert len(hits_c) == 1
+    assert hits_c[0].video_id == "v3"
+
+    # get_segments ordered
+    seg_rows = store.get_segments("v1")
+    assert len(seg_rows) == 2
+    assert seg_rows[0].idx == 0
+
+
+def test_store_segments_overwrites(seeded_store):
+    store = seeded_store
+    store.store_segments("v1", [Segment(0.0, 1.0, "old")])
+    store.store_segments("v1", [Segment(0.0, 1.0, "new"), Segment(2.0, 3.0, "second")])
+    rows = store.get_segments("v1")
+    assert len(rows) == 2
+    assert rows[0].text == "new"
+
+
+def test_update_video_metadata(seeded_store):
+    store = seeded_store
+    store.update_video_metadata("v1", view_count=1000, like_count=50, tags=["music", "retro"], thumbnail="http://t.png")
+    v = store.get_video("v1")
+    assert v.view_count == 1000
+    assert v.like_count == 50
+    assert json.loads(v.tags) == ["music", "retro"]
+    assert v.thumbnail == "http://t.png"
+
+
+def test_query_videos_filters(seeded_store):
+    store = seeded_store
+    rows, total = store.query_videos(channel_id="UC1")
+    assert total == 2
+    rows2, total2 = store.query_videos(min_duration=250)
+    assert total2 == 1
+    assert rows2[0].video_id == "v3"
+    rows3, total3 = store.query_videos(q="dragon")
+    assert total3 == 1
+    assert rows3[0].video_id == "v2"
+
+
+def test_cookie_vault(store):
+    from yt_scraper import cookies
+    sample = (
+        "# Netscape HTTP Cookie File\n"
+        ".youtube.com\tTRUE\t/\tTRUE\t1800287621\tSID\tabc\n"
+        ".youtube.com\tTRUE\t/\tTRUE\t1800287621\tSAPISID\tdef\n"
+        ".youtube.com\tTRUE\t/\tFALSE\t1800287621\tPREF\tf1=2\n"
+    )
+    cid = cookies.import_text(store, sample, label="test", cookie_dir=str(Path(store.db_path).parent / "ck"))
+    vault = cookies.list_vault(store)
+    assert len(vault) == 1
+    assert vault[0].has_session is True
+    assert vault[0].cookie_count == 3
+    # set active + resolve
+    cookies.set_active(store, cid)
+    path = cookies.resolve_active_path(store, cookie_dir=str(Path(store.db_path).parent / "ck"))
+    assert path and Path(path).exists()
+    # delete
+    cookies.delete(store, cid, cookie_dir=str(Path(store.db_path).parent / "ck"))
+    assert cookies.list_vault(store) == []
+
+
+def test_cookie_invalid_rejected(store):
+    from yt_scraper import cookies
+    with pytest.raises(ValueError):
+        cookies.import_text(store, "not a cookie file", label="bad", cookie_dir=str(Path(store.db_path).parent / "ck"))
+
+
+def test_cookie_only_one_active(store):
+    from yt_scraper import cookies
+    cdir = str(Path(store.db_path).parent / "ck2")
+    sample = "# Netscape\n.youtube.com\tTRUE\t/\tTRUE\t1800287621\tSID\tx\n"
+    c1 = cookies.import_text(store, sample, label="a", cookie_dir=cdir)
+    c2 = cookies.import_text(store, sample, label="b", cookie_dir=cdir)
+    cookies.set_active(store, c1)
+    cookies.set_active(store, c2)
+    actives = [c for c in cookies.list_vault(store) if c.is_active]
+    assert len(actives) == 1
+    assert actives[0].id == c2
+
+
+def test_jobs_crud(store):
+    store.create_job("job1", "UC1", {"limit": 5})
+    store.update_job("job1", status="running", total=5)
+    store.update_job("job1", completed=3)
+    j = store.get_job("job1")
+    assert j.status == "running"
+    assert j.total == 5
+    assert j.completed == 3
+    store.update_job("job1", status="done", completed=5, finished=True)
+    j2 = store.get_job("job1")
+    assert j2.status == "done"
+    assert j2.finished_at is not None
+
+
+def test_dashboard_aggregates(seeded_store):
+    store = seeded_store
+    store.store_segments("v1", [Segment(0.0, 2.0, "vaporwave hello")])
+    store.update_video_metadata("v1", view_count=500, tags=["music", "vaporwave"])
+    store.update_video_metadata("v2", tags=["music", "anime"])
+    d = store.dashboard()
+    assert len(d["channels"]) == 2
+    ch1 = [c for c in d["channels"] if c["channel_id"] == "UC1"][0]
+    assert ch1["video_count_db"] == 2
+    assert ch1["total_views"] == 500
+    tags = {t["tag"]: t["count"] for t in d["top_tags"]}
+    assert tags.get("music") == 2