feat: local content-mining platform for YouTube creator scraping
Scraper de canales de YouTube hacia notas Markdown para base de conocimiento (Obsidian-ready), con plataforma web local. Engine + CLI (Workstream A): - Modular pipeline: discover/extract/parse/chapters/render/store + ratelimit - SQLite store con migración idempotente: FTS5 (transcript search), columnas de metadata enriquecida, tablas cookies_meta y scrape_jobs - Módulos: segments, cookies (Netscape vault), export (json/csv/srt/html), analysis (word freq/timeline/wordcloud), monitor (watch loop), pipeline - CLI Click group: search, export, audio, channels, watch, analyze, re-render - Fix del bug de scoping de cookies en cli.py Webapp local (Workstream B): - FastAPI backend: dashboard, channels, videos facetado, transcript, search FTS, analysis, scrape jobs con SSE, cookies drag-and-drop, exports, folders (abrir en OS), tools (re-render, formato) - SPA no-build (Alpine.js + Tailwind + Chart.js por CDN): 9 vistas, tema dark command-center con acento rojo→rosa, cookie vault drag-drop, consola de scrapeo con progreso live vía SSE Launcher + subagentes (Workstream C): - start-server.bat / stop-server.bat con auto port-scan + browser open - .opencode/agent/webapp-builder.md + .opencode/goals/webapp-build.md Tests: 38 pytest verdes. Sin funcionalidad de IA (enfoque data-mining).
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||
|
||||
from .chapters import Section
|
||||
|
||||
|
||||
def format_timestamp(seconds: float) -> str:
|
||||
total = int(seconds)
|
||||
hours, remainder = divmod(total, 3600)
|
||||
minutes, secs = divmod(remainder, 60)
|
||||
if hours > 0:
|
||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
||||
return f"{minutes:02d}:{secs:02d}"
|
||||
|
||||
|
||||
def quote_yaml(value: str | None) -> str:
|
||||
if value is None:
|
||||
return '""'
|
||||
needs_quote = any(c in value for c in [':', '#', '{', '}', '[', ']', ',', '&', '*', '!', '|', '>', "'", '"', '%', '@', '`', '\n'])
|
||||
if needs_quote:
|
||||
escaped = value.replace('"', '\\"')
|
||||
return f'"{escaped}"'
|
||||
return value
|
||||
|
||||
|
||||
def to_json(value) -> str:
|
||||
import json
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _make_env(template_dir: Path) -> Environment:
|
||||
env = Environment(
|
||||
loader=FileSystemLoader(str(template_dir)),
|
||||
autoescape=select_autoescape(disabled_extensions=("j2", "txt")),
|
||||
trim_blocks=True,
|
||||
lstrip_blocks=True,
|
||||
)
|
||||
env.filters["format_timestamp"] = format_timestamp
|
||||
env.filters["quote_yaml"] = quote_yaml
|
||||
env.filters["to_json"] = to_json
|
||||
return env
|
||||
|
||||
|
||||
def render_markdown(
|
||||
template_path: str | Path,
|
||||
output_dir: str | Path,
|
||||
filename_stem: str,
|
||||
context: dict,
|
||||
) -> Path:
|
||||
tpl_path = Path(template_path).resolve()
|
||||
env = _make_env(tpl_path.parent)
|
||||
template = env.get_template(tpl_path.name)
|
||||
rendered = template.render(**context)
|
||||
|
||||
out_dir = Path(output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_file = out_dir / f"{filename_stem}.md"
|
||||
out_file.write_text(rendered, encoding="utf-8")
|
||||
return out_file
|
||||
|
||||
|
||||
def build_filename_stem(upload_date: str | None, title: str, template: str = "{upload_date}_{slug}") -> str:
|
||||
from slugify import slugify
|
||||
date_part = upload_date or "unknown-date"
|
||||
slug = slugify(title, max_length=60) or "untitled"
|
||||
stem = template.format(upload_date=date_part, slug=slug, title=title)
|
||||
safe = "".join(c for c in stem if c not in r'\/:*?"<>|')
|
||||
return safe.strip().strip(".")
|
||||
Reference in New Issue
Block a user