urieljareth
621bbc5f5c
Scraper de canales de YouTube hacia notas Markdown para base de conocimiento (Obsidian-ready), con plataforma web local. Engine + CLI (Workstream A): - Modular pipeline: discover/extract/parse/chapters/render/store + ratelimit - SQLite store con migración idempotente: FTS5 (transcript search), columnas de metadata enriquecida, tablas cookies_meta y scrape_jobs - Módulos: segments, cookies (Netscape vault), export (json/csv/srt/html), analysis (word freq/timeline/wordcloud), monitor (watch loop), pipeline - CLI Click group: search, export, audio, channels, watch, analyze, re-render - Fix del bug de scoping de cookies en cli.py Webapp local (Workstream B): - FastAPI backend: dashboard, channels, videos facetado, transcript, search FTS, analysis, scrape jobs con SSE, cookies drag-and-drop, exports, folders (abrir en OS), tools (re-render, formato) - SPA no-build (Alpine.js + Tailwind + Chart.js por CDN): 9 vistas, tema dark command-center con acento rojo→rosa, cookie vault drag-drop, consola de scrapeo con progreso live vía SSE Launcher + subagentes (Workstream C): - start-server.bat / stop-server.bat con auto port-scan + browser open - .opencode/agent/webapp-builder.md + .opencode/goals/webapp-build.md Tests: 38 pytest verdes. Sin funcionalidad de IA (enfoque data-mining).
78 lines
2.2 KiB
Python
78 lines
2.2 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from yt_scraper.render import build_filename_stem, format_timestamp, quote_yaml
|
|
from yt_scraper.chapters import Section
|
|
from yt_scraper.parse import Segment
|
|
|
|
TEMPLATES = Path(__file__).parent.parent / "templates"
|
|
|
|
|
|
def test_format_timestamp_minutes():
|
|
assert format_timestamp(0.0) == "00:00"
|
|
assert format_timestamp(65.0) == "01:05"
|
|
assert format_timestamp(3599.0) == "59:59"
|
|
|
|
|
|
def test_format_timestamp_hours():
|
|
assert format_timestamp(3661.0) == "01:01:01"
|
|
|
|
|
|
def test_quote_yaml_plain():
|
|
assert quote_yaml("hola") == "hola"
|
|
|
|
|
|
def test_quote_yaml_special():
|
|
result = quote_yaml('hello "world"')
|
|
assert result.startswith('"')
|
|
assert result.endswith('"')
|
|
assert '\\"' in result
|
|
|
|
|
|
def test_quote_yaml_none():
|
|
assert quote_yaml(None) == '""'
|
|
|
|
|
|
def test_build_filename_stem():
|
|
stem = build_filename_stem("20240115", "Mi Video Genial!")
|
|
assert stem.startswith("20240115")
|
|
assert "mi-video-genial" in stem.lower()
|
|
|
|
|
|
def test_build_filename_stem_no_date():
|
|
stem = build_filename_stem(None, "Titulo")
|
|
assert "unknown" in stem
|
|
assert "titulo" in stem.lower()
|
|
|
|
|
|
def test_render_markdown(tmp_path):
|
|
template_file = tmp_path / "test.j2"
|
|
template_file.write_text(
|
|
"---\ntitle: {{ title | quote_yaml }}\n---\n# {{ title }}\n\n"
|
|
"{% for s in sections %}## {{ s.title }}\n{% for seg in s.segments %}**{{ seg.start | format_timestamp }}** {{ seg.text }}\n{% endfor %}{% endfor %}",
|
|
encoding="utf-8",
|
|
)
|
|
out_dir = tmp_path / "out"
|
|
sections = [
|
|
Section(
|
|
title="Intro",
|
|
start=0.0,
|
|
segments=[Segment(start=0.0, end=1.0, text="Hola"), Segment(start=5.0, end=6.0, text="Mundo")],
|
|
)
|
|
]
|
|
from yt_scraper.render import render_markdown
|
|
|
|
md_path = render_markdown(
|
|
template_file,
|
|
out_dir,
|
|
"20240115_test",
|
|
{"title": "Test Video", "sections": sections},
|
|
)
|
|
assert md_path.exists()
|
|
content = md_path.read_text(encoding="utf-8")
|
|
assert "# Test Video" in content
|
|
assert "## Intro" in content
|
|
assert "**00:00** Hola" in content
|
|
assert "**00:05** Mundo" in content
|