diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..05aeea8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.git +.venv +__pycache__ +*.pyc +.pytest_cache +.mypy_cache +.ruff_cache +.env +.env.* +data +frontend/.next +frontend/node_modules +frontend/.env.local +frontend/.env.*.local +node_modules +npm-debug.log* +Dockerfile* +docker-compose*.yml +docker-compose*.yaml diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d7c72b2 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +APP_ENV=development +APP_DATA_DIR=./data +DATABASE_PATH=./data/app.db +MISTRAL_API_KEY= +DEEPGRAM_API_KEY= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f4f6dd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +.env +__pycache__/ +*.py[cod] +.pytest_cache/ +.venv/ +venv/ +node_modules/ +.next/ +.runtime/ +data/ +tmp/ +*.db +*.sqlite +frontend/.env.local diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..93db987 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,32 @@ +# AGENTS.md + +## Repo Shape +- The primary app is now `backend/` (Python/FastAPI) plus `frontend/` (Next.js/TypeScript/Tailwind); `estacion documentos.html` is a browser-only prototype/reference for behavior. +- Runtime data is stored under `data/` and is gitignored; do not commit uploaded class files, generated Markdown, SQLite DBs, temporary audio, or API keys. +- The included `Griffin(2020).Comportamiento_organizacionalSOLOCAPITULO7.13Ed.pdf` is a local sample/source PDF for import behavior. + +## Commands +- Local setup: `python -m venv .venv`, `.venv\Scripts\activate`, `pip install -r requirements.txt`, copy `.env.example` to `.env`. +- Run API: `uvicorn backend.app.main:app --reload` from the repo root. +- Run frontend: `cd frontend`, `npm install`, `npm run dev -- -p 3000` with `NEXT_PUBLIC_API_URL=http://127.0.0.1:8000`. +- Windows non-technical flow: double-click `abrir_estacion.bat`; it chooses free ports, starts backend/frontend, and opens the browser. Use `cerrar_estacion.bat` to stop only this project. +- Docker/Coolify check: `docker compose up --build`. +- Basic syntax check: `python -m compileall backend`. + +## Platform Behavior +- Knowledge is organized as `Materia -> Semana -> fuentes raw -> Markdown procesado -> exports` under `data/subjects//week-XX/`. +- `KnowledgeManager` owns filesystem layout and SQLite metadata; processors should not write final Markdown directly. +- `IngestionService` selects processors by `source_type` and wraps output with Markdown frontmatter for LLM ingestion. +- Text and Markdown are read directly; `.docx` uses `python-docx`; PDFs use PyMuPDF text extraction unless `use_ocr=true` is passed. PDF uploads can also pass `page_ranges` like `1-5,8,10-12`; ranges apply to both standard extraction and Mistral OCR. +- OCR uses Mistral `https://api.mistral.ai/v1/chat/completions` with model `pixtral-12b-2409`. +- Audio uses Deepgram `https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=es`. +- Video processing requires `ffmpeg`; audio is extracted to `data/tmp/`, transcribed with Deepgram, then deleted. +- The frontend calls the backend through `NEXT_PUBLIC_API_URL`; when ports change, update `frontend/.env.local` or use the adaptive `.bat`. + +## Secrets And Deployment +- Use `.env` locally and Coolify environment variables in production; never hardcode or commit `MISTRAL_API_KEY` or `DEEPGRAM_API_KEY`. +- For Coolify, mount persistent storage for `/app/data` or all subjects, outputs, and SQLite state will be lost on redeploy. + +## Prototype Notes +- Preserve the Spanish-language UX and Markdown-first output conventions from `estacion documentos.html` when building the new UI. +- The prototype sanitizes preview HTML with DOMPurify after `marked.parse`; keep equivalent sanitization in any future web UI. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ce03fec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY backend ./backend + +EXPOSE 8000 +CMD ["uvicorn", "backend.app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Griffin(2020).Comportamiento_organizacionalSOLOCAPITULO7.13Ed.pdf b/Griffin(2020).Comportamiento_organizacionalSOLOCAPITULO7.13Ed.pdf new file mode 100644 index 0000000..2b88862 Binary files /dev/null and b/Griffin(2020).Comportamiento_organizacionalSOLOCAPITULO7.13Ed.pdf differ diff --git a/README.md b/README.md new file mode 100644 index 0000000..0c1ec69 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# Knowledge Station + +Plataforma para organizar conocimiento escolar por materia y semana, procesar documentos/audio/video y generar archivos Markdown optimizados para ingesta en LLM. + +El archivo `estacion documentos.html` queda como prototipo funcional de referencia. La plataforma principal usa `backend/` (FastAPI) y `frontend/` (Next.js). + +## Capacidades Iniciales + +- Materias y semanas persistidas en SQLite. +- Estructura de archivos por materia/semana en `data/subjects/`. +- Ingesta de `.txt`, `.md`, `.docx`, `.pdf`, audio y video. +- PDF con texto mediante PyMuPDF. +- OCR opcional de PDF con Mistral. +- Audio con Deepgram en español. +- Video con extracción temporal de audio vía `ffmpeg` y transcripción con Deepgram. +- Exportación semanal consolidada a Markdown. +- UI dedicada con materias, semanas, subida de archivos, polling de trabajos y editor Markdown con preview. + +## Desarrollo Local + +```bash +python -m venv .venv +.venv\Scripts\activate +pip install -r requirements.txt +copy .env.example .env +uvicorn backend.app.main:app --reload +``` + +En otra terminal: + +```bash +cd frontend +npm install +set NEXT_PUBLIC_API_URL=http://127.0.0.1:8000 +npm run dev -- -p 3000 +``` + +La plataforma queda en `http://localhost:3000`, la API en `http://localhost:8000` y la documentación interactiva en `http://localhost:8000/docs`. + +Para usuarios no tecnicos en Windows, usa `abrir_estacion.bat`; detecta puertos libres, prepara backend/frontend y abre el navegador. Usa `cerrar_estacion.bat` para detener todo. + +## Docker / Coolify + +```bash +docker compose up --build +``` + +Para Coolify, configura backend y frontend como servicios separados. Monta un volumen persistente para `/app/data` en el backend y define `NEXT_PUBLIC_API_URL` en el frontend apuntando a la URL publica del backend. + +## Variables + +- `APP_DATA_DIR`: carpeta de datos persistentes. +- `DATABASE_PATH`: ruta SQLite. +- `MISTRAL_API_KEY`: OCR para PDFs escaneados. +- `DEEPGRAM_API_KEY`: transcripción de audio/video. + +No subas claves reales al repositorio. + +## Flujo API Basico + +Crear materia: + +```bash +curl -X POST http://localhost:8000/subjects -H "Content-Type: application/json" -d '{"name":"Comportamiento Organizacional"}' +``` + +Crear semana: + +```bash +curl -X POST http://localhost:8000/subjects/1/weeks -H "Content-Type: application/json" -d '{"number":1,"title":"Grupos y equipos"}' +``` + +Subir archivo: + +```bash +curl -X POST -F "file=@clase-01.mp4" http://localhost:8000/subjects/1/weeks/1/files +``` + +Consultar trabajo: + +```bash +curl http://localhost:8000/jobs/1 +``` + +Exportar semana: + +```bash +curl -L http://localhost:8000/subjects/1/weeks/1/export -o semana-01.md +``` diff --git a/abrir_estacion.bat b/abrir_estacion.bat new file mode 100644 index 0000000..201d759 --- /dev/null +++ b/abrir_estacion.bat @@ -0,0 +1,18 @@ +@echo off +setlocal +title Knowledge Station - Abrir + +set "ROOT=%~dp0" +set "ROOT=%ROOT:~0,-1%" +powershell -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\scripts\start-platform.ps1" -Root "%ROOT%" +if errorlevel 1 ( + echo. + echo No se pudo abrir la plataforma. Revisa el mensaje anterior. + pause + exit /b 1 +) + +echo. +echo Puedes dejar esta ventana cerrada. Para apagar la plataforma usa cerrar_estacion.bat. +pause +exit /b 0 diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..5ba5f01 --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,16 @@ +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + app_env: str = "development" + app_data_dir: Path = Path("./data") + database_path: Path = Path("./data/app.db") + mistral_api_key: str = "" + deepgram_api_key: str = "" + + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + +settings = Settings() diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..86ab314 --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi import BackgroundTasks, FastAPI, File, HTTPException, Query, UploadFile +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, PlainTextResponse + +from backend.app.config import settings +from backend.app.managers.knowledge_manager import DuplicateSourceError, KnowledgeManager +from backend.app.models.schemas import MarkdownUpdate, SubjectCreate, WeekCreate, WeekUpdate +from backend.app.services.ingestion_service import IngestionService + +app = FastAPI(title="Knowledge Station", version="0.1.0") +app.add_middleware( + CORSMiddleware, + allow_origins=["http://127.0.0.1:3000", "http://localhost:3000"], + allow_origin_regex=r"http://(127\.0\.0\.1|localhost):\d+", + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) +manager = KnowledgeManager() +ingestion = IngestionService(manager) + + +@app.get("/health") +def health() -> dict[str, str]: + return {"status": "ok"} + + +@app.get("/settings/status") +def settings_status() -> dict: + return { + "mistral_configured": bool(settings.mistral_api_key), + "deepgram_configured": bool(settings.deepgram_api_key), + "data_dir": str(settings.app_data_dir), + "database_path": str(settings.database_path), + } + + +@app.post("/subjects") +def create_subject(payload: SubjectCreate) -> dict: + try: + return manager.create_subject(payload.name) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@app.get("/subjects") +def list_subjects() -> list[dict]: + return manager.list_subjects() + + +@app.get("/subjects/{subject_id}") +def get_subject(subject_id: int) -> dict: + try: + return manager.get_subject(subject_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.delete("/subjects/{subject_id}") +def delete_subject(subject_id: int) -> dict: + try: + manager.delete_subject(subject_id) + return {"status": "deleted"} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.post("/subjects/{subject_id}/weeks") +def create_week(subject_id: int, payload: WeekCreate) -> dict: + try: + return manager.create_week(subject_id, payload.number, payload.title) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/subjects/{subject_id}/weeks") +def list_weeks(subject_id: int) -> list[dict]: + try: + manager.get_subject(subject_id) + return manager.list_weeks(subject_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/subjects/{subject_id}/weeks/{week_number}") +def get_week(subject_id: int, week_number: int) -> dict: + try: + return manager.get_week(subject_id, week_number) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.put("/subjects/{subject_id}/weeks/{week_number}") +def update_week(subject_id: int, week_number: int, payload: WeekUpdate) -> dict: + try: + return manager.update_week(subject_id, week_number, payload.number, payload.title) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@app.delete("/subjects/{subject_id}/weeks/{week_number}") +def delete_week(subject_id: int, week_number: int) -> dict: + try: + manager.delete_week(subject_id, week_number) + return {"status": "deleted"} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.post("/subjects/{subject_id}/weeks/{week_number}/files") +def upload_file( + subject_id: int, + week_number: int, + background_tasks: BackgroundTasks, + file: UploadFile = File(...), + use_ocr: bool = Query(False, description="Usar Mistral OCR para PDFs"), + page_ranges: str | None = Query(None, description="Rangos de paginas PDF, por ejemplo: 1-5,8,10-12"), +) -> dict: + try: + source = manager.save_upload(subject_id, week_number, file) + job = manager.create_job(source["id"], "Archivo recibido") + background_tasks.add_task(process_job, job["id"], source["id"], use_ocr, page_ranges) + return {"source": source, "job": job} + except DuplicateSourceError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/jobs/{job_id}") +def get_job(job_id: int) -> dict: + try: + return manager.get_job(job_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.post("/jobs/{job_id}/retry") +def retry_job( + job_id: int, + background_tasks: BackgroundTasks, + use_ocr: bool = Query(False), + page_ranges: str | None = Query(None), +) -> dict: + try: + job = manager.get_job(job_id) + if not job.get("source_id"): + raise HTTPException(status_code=400, detail="El trabajo no tiene fuente asociada") + manager.update_job(job_id, "queued", "Reintento en cola") + background_tasks.add_task(process_job, job_id, job["source_id"], use_ocr, page_ranges) + return manager.get_job(job_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/subjects/{subject_id}/weeks/{week_number}/jobs") +def list_week_jobs(subject_id: int, week_number: int) -> list[dict]: + try: + return manager.list_jobs_for_week(subject_id, week_number) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/subjects/{subject_id}/weeks/{week_number}/sources") +def list_sources(subject_id: int, week_number: int) -> list[dict]: + try: + return manager.list_sources(subject_id, week_number) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.delete("/sources/{source_id}") +def delete_source(source_id: int) -> dict: + try: + manager.delete_source(source_id) + return {"status": "deleted"} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.post("/sources/{source_id}/reprocess") +def reprocess_source( + source_id: int, + background_tasks: BackgroundTasks, + use_ocr: bool = Query(False), + page_ranges: str | None = Query(None), +) -> dict: + try: + source = manager.get_source(source_id) + if manager.source_has_active_job(source_id): + raise HTTPException(status_code=409, detail="La fuente ya tiene un trabajo en cola o en proceso") + job = manager.create_job(source["id"], "Reprocesamiento en cola") + background_tasks.add_task(process_job, job["id"], source["id"], use_ocr, page_ranges) + return {"source": source, "job": job} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/subjects/{subject_id}/weeks/{week_number}/documents") +def list_documents(subject_id: int, week_number: int) -> list[dict]: + try: + return manager.list_documents(subject_id, week_number) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/documents/{document_id}") +def get_document(document_id: int) -> dict: + try: + return manager.get_document(document_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/documents/{document_id}/markdown", response_class=PlainTextResponse) +def get_document_markdown(document_id: int) -> str: + try: + document = manager.get_document(document_id) + return Path(document["markdown_path"]).read_text(encoding="utf-8") + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.put("/documents/{document_id}/markdown") +def update_document_markdown(document_id: int, payload: MarkdownUpdate) -> dict: + try: + return manager.update_document_markdown(document_id, payload.content) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/documents/{document_id}/download") +def download_document(document_id: int) -> FileResponse: + try: + document = manager.get_document(document_id) + path = Path(document["markdown_path"]) + return FileResponse(path, filename=path.name, media_type="text/markdown") + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.delete("/documents/{document_id}") +def delete_document(document_id: int) -> dict: + try: + manager.delete_document(document_id) + return {"status": "deleted"} + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/subjects/{subject_id}/weeks/{week_number}/export") +def export_week(subject_id: int, week_number: int) -> FileResponse: + try: + path = manager.export_week_markdown(subject_id, week_number) + return FileResponse(path, filename=path.name, media_type="text/markdown") + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +@app.get("/subjects/{subject_id}/weeks/{week_number}/export.zip") +def export_week_zip(subject_id: int, week_number: int) -> FileResponse: + try: + path = manager.export_week_markdown_zip(subject_id, week_number) + return FileResponse(path, filename=path.name, media_type="application/zip") + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + +def process_job(job_id: int, source_id: int, use_ocr: bool, page_ranges: str | None = None) -> None: + manager.update_job(job_id, "processing", "Procesando archivo") + manager.update_source_status(source_id, "processing") + try: + document = ingestion.process_source(source_id, use_ocr=use_ocr, page_ranges=page_ranges) + manager.update_job(job_id, "completed", "Documento procesado", document_id=document["id"]) + except Exception as exc: + manager.update_source_status(source_id, "failed") + manager.update_job(job_id, "failed", str(exc)) diff --git a/backend/app/managers/__init__.py b/backend/app/managers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/managers/knowledge_manager.py b/backend/app/managers/knowledge_manager.py new file mode 100644 index 0000000..5813122 --- /dev/null +++ b/backend/app/managers/knowledge_manager.py @@ -0,0 +1,548 @@ +from __future__ import annotations + +import shutil +import sqlite3 +import re +import hashlib +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +from fastapi import UploadFile + +from backend.app.config import settings +from backend.app.utils.slug import slugify + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class DuplicateSourceError(ValueError): + def __init__(self, existing_source: dict[str, Any]): + self.existing_source = existing_source + super().__init__(f"El archivo ya existe en esta semana: {existing_source['original_name']}") + + +class KnowledgeManager: + def __init__(self, data_dir: Path | None = None, database_path: Path | None = None): + self.data_dir = data_dir or settings.app_data_dir + self.database_path = database_path or settings.database_path + self.subjects_dir = self.data_dir / "subjects" + self.tmp_dir = self.data_dir / "tmp" + self.data_dir.mkdir(parents=True, exist_ok=True) + self.subjects_dir.mkdir(parents=True, exist_ok=True) + self.tmp_dir.mkdir(parents=True, exist_ok=True) + self.database_path.parent.mkdir(parents=True, exist_ok=True) + self._init_db() + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.database_path) + conn.row_factory = sqlite3.Row + return conn + + def _init_db(self) -> None: + with self._connect() as conn: + conn.executescript( + """ + create table if not exists subjects ( + id integer primary key autoincrement, + name text not null, + slug text not null unique, + created_at text not null + ); + + create table if not exists weeks ( + id integer primary key autoincrement, + subject_id integer not null, + number integer not null, + title text, + created_at text not null, + unique(subject_id, number), + foreign key(subject_id) references subjects(id) + ); + + create table if not exists sources ( + id integer primary key autoincrement, + week_id integer not null, + original_name text not null, + stored_path text not null, + source_type text not null, + status text not null, + file_size integer, + sha256 text, + mime_type text, + processor text, + page_ranges text, + processed_at text, + created_at text not null, + foreign key(week_id) references weeks(id) + ); + + create table if not exists documents ( + id integer primary key autoincrement, + source_id integer not null, + title text not null, + markdown_path text not null, + created_at text not null, + foreign key(source_id) references sources(id) + ); + + create table if not exists jobs ( + id integer primary key autoincrement, + status text not null, + message text not null, + source_id integer, + document_id integer, + created_at text not null, + updated_at text not null, + foreign key(source_id) references sources(id), + foreign key(document_id) references documents(id) + ); + """ + ) + self._ensure_columns( + conn, + "sources", + { + "file_size": "integer", + "sha256": "text", + "mime_type": "text", + "processor": "text", + "page_ranges": "text", + "processed_at": "text", + }, + ) + self._backfill_source_file_metadata(conn) + + def _ensure_columns(self, conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None: + existing = {row["name"] for row in conn.execute(f"pragma table_info({table})").fetchall()} + for name, definition in columns.items(): + if name not in existing: + conn.execute(f"alter table {table} add column {name} {definition}") + + def _backfill_source_file_metadata(self, conn: sqlite3.Connection) -> None: + rows = conn.execute("select id, stored_path from sources where sha256 is null or file_size is null").fetchall() + for row in rows: + path = Path(row["stored_path"]) + if not path.exists(): + continue + sha256, file_size = self._file_hash_and_size(path) + conn.execute("update sources set sha256 = ?, file_size = ? where id = ?", (sha256, file_size, row["id"])) + + def create_subject(self, name: str) -> dict[str, Any]: + base_slug = slugify(name) + slug = base_slug + suffix = 2 + with self._connect() as conn: + while conn.execute("select 1 from subjects where slug = ?", (slug,)).fetchone(): + slug = f"{base_slug}-{suffix}" + suffix += 1 + now = utc_now() + cursor = conn.execute( + "insert into subjects (name, slug, created_at) values (?, ?, ?)", + (name, slug, now), + ) + (self.subjects_dir / slug).mkdir(parents=True, exist_ok=True) + return self.get_subject(cursor.lastrowid) + + def get_subject(self, subject_id: int) -> dict[str, Any]: + with self._connect() as conn: + row = conn.execute("select * from subjects where id = ?", (subject_id,)).fetchone() + if not row: + raise ValueError("Materia no encontrada") + return dict(row) + + def delete_subject(self, subject_id: int) -> None: + subject = self.get_subject(subject_id) + with self._connect() as conn: + week_rows = conn.execute("select id from weeks where subject_id = ?", (subject_id,)).fetchall() + week_ids = [row["id"] for row in week_rows] + for week_id in week_ids: + source_rows = conn.execute("select id from sources where week_id = ?", (week_id,)).fetchall() + source_ids = [row["id"] for row in source_rows] + for source_id in source_ids: + conn.execute("delete from documents where source_id = ?", (source_id,)) + conn.execute("delete from jobs where source_id = ?", (source_id,)) + conn.execute("delete from sources where week_id = ?", (week_id,)) + conn.execute("delete from weeks where subject_id = ?", (subject_id,)) + conn.execute("delete from subjects where id = ?", (subject_id,)) + shutil.rmtree(self.subjects_dir / subject["slug"], ignore_errors=True) + + def list_subjects(self) -> list[dict[str, Any]]: + with self._connect() as conn: + rows = conn.execute("select * from subjects order by created_at desc").fetchall() + return [dict(row) for row in rows] + + def create_week(self, subject_id: int, number: int, title: str | None = None) -> dict[str, Any]: + subject = self.get_subject(subject_id) + now = utc_now() + with self._connect() as conn: + conn.execute( + "insert or ignore into weeks (subject_id, number, title, created_at) values (?, ?, ?, ?)", + (subject_id, number, title, now), + ) + if title: + conn.execute("update weeks set title = ? where subject_id = ? and number = ?", (title, subject_id, number)) + row = conn.execute( + "select * from weeks where subject_id = ? and number = ?", + (subject_id, number), + ).fetchone() + self.week_dir(subject["slug"], number).mkdir(parents=True, exist_ok=True) + for child in ("raw", "processed", "exports"): + (self.week_dir(subject["slug"], number) / child).mkdir(parents=True, exist_ok=True) + return dict(row) + + def list_weeks(self, subject_id: int) -> list[dict[str, Any]]: + with self._connect() as conn: + rows = conn.execute("select * from weeks where subject_id = ? order by number", (subject_id,)).fetchall() + return [dict(row) for row in rows] + + def delete_week(self, subject_id: int, number: int) -> None: + subject = self.get_subject(subject_id) + week = self.get_week(subject_id, number) + with self._connect() as conn: + source_rows = conn.execute("select id from sources where week_id = ?", (week["id"],)).fetchall() + for row in source_rows: + conn.execute("delete from documents where source_id = ?", (row["id"],)) + conn.execute("delete from jobs where source_id = ?", (row["id"],)) + conn.execute("delete from sources where week_id = ?", (week["id"],)) + conn.execute("delete from weeks where id = ?", (week["id"],)) + shutil.rmtree(self.week_dir(subject["slug"], number), ignore_errors=True) + + def get_week(self, subject_id: int, number: int) -> dict[str, Any]: + with self._connect() as conn: + row = conn.execute( + "select * from weeks where subject_id = ? and number = ?", + (subject_id, number), + ).fetchone() + if not row: + raise ValueError("Semana no encontrada") + return dict(row) + + def update_week(self, subject_id: int, current_number: int, new_number: int, title: str | None = None) -> dict[str, Any]: + subject = self.get_subject(subject_id) + week = self.get_week(subject_id, current_number) + if new_number != current_number: + with self._connect() as conn: + existing = conn.execute( + "select 1 from weeks where subject_id = ? and number = ?", + (subject_id, new_number), + ).fetchone() + if existing: + raise ValueError("Ya existe una semana con ese numero") + + old_dir = self.week_dir(subject["slug"], current_number) + new_dir = self.week_dir(subject["slug"], new_number) + if old_dir.exists() and new_dir.exists(): + raise ValueError("Ya existe una carpeta para ese numero de semana") + if old_dir.exists(): + old_dir.rename(new_dir) + else: + for child in ("raw", "processed", "exports"): + (new_dir / child).mkdir(parents=True, exist_ok=True) + + with self._connect() as conn: + conn.execute( + "update weeks set number = ?, title = ? where id = ?", + (new_number, title, week["id"]), + ) + return self.get_week(subject_id, new_number) + + def week_dir(self, subject_slug: str, week_number: int) -> Path: + return self.subjects_dir / subject_slug / f"week-{week_number:02d}" + + def save_upload(self, subject_id: int, week_number: int, upload: UploadFile) -> dict[str, Any]: + subject = self.get_subject(subject_id) + week = self.get_week(subject_id, week_number) + filename = Path(upload.filename or "archivo").name + raw_dir = self.week_dir(subject["slug"], week_number) / "raw" + raw_dir.mkdir(parents=True, exist_ok=True) + stored_path = self._unique_path(raw_dir / filename) + digest = hashlib.sha256() + file_size = 0 + with stored_path.open("wb") as out_file: + while chunk := upload.file.read(1024 * 1024): + file_size += len(chunk) + digest.update(chunk) + out_file.write(chunk) + sha256 = digest.hexdigest() + source_type = self.detect_source_type(stored_path) + now = utc_now() + with self._connect() as conn: + duplicate = conn.execute( + "select * from sources where week_id = ? and sha256 = ? limit 1", + (week["id"], sha256), + ).fetchone() + if duplicate: + stored_path.unlink(missing_ok=True) + raise DuplicateSourceError(dict(duplicate)) + cursor = conn.execute( + """ + insert into sources (week_id, original_name, stored_path, source_type, status, file_size, sha256, mime_type, created_at) + values (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (week["id"], filename, str(stored_path), source_type, "stored", file_size, sha256, upload.content_type, now), + ) + return self.get_source(cursor.lastrowid) + + def _file_hash_and_size(self, path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + file_size = 0 + with path.open("rb") as file: + while chunk := file.read(1024 * 1024): + file_size += len(chunk) + digest.update(chunk) + return digest.hexdigest(), file_size + + def create_job(self, source_id: int, message: str = "En cola") -> dict[str, Any]: + now = utc_now() + with self._connect() as conn: + cursor = conn.execute( + "insert into jobs (status, message, source_id, created_at, updated_at) values (?, ?, ?, ?, ?)", + ("queued", message, source_id, now, now), + ) + return self.get_job(cursor.lastrowid) + + def update_job(self, job_id: int, status: str, message: str, document_id: int | None = None) -> None: + with self._connect() as conn: + conn.execute( + "update jobs set status = ?, message = ?, document_id = coalesce(?, document_id), updated_at = ? where id = ?", + (status, message, document_id, utc_now(), job_id), + ) + + def update_source_status(self, source_id: int, status: str) -> None: + with self._connect() as conn: + conn.execute("update sources set status = ? where id = ?", (status, source_id)) + + def source_has_active_job(self, source_id: int) -> bool: + with self._connect() as conn: + row = conn.execute( + "select 1 from jobs where source_id = ? and status in ('queued', 'processing') limit 1", + (source_id,), + ).fetchone() + return bool(row) + + def get_job(self, job_id: int) -> dict[str, Any]: + with self._connect() as conn: + row = conn.execute("select * from jobs where id = ?", (job_id,)).fetchone() + if not row: + raise ValueError("Trabajo no encontrado") + return dict(row) + + def list_jobs_for_week(self, subject_id: int, week_number: int) -> list[dict[str, Any]]: + week = self.get_week(subject_id, week_number) + with self._connect() as conn: + rows = conn.execute( + """ + select jobs.* from jobs + join sources on sources.id = jobs.source_id + where sources.week_id = ? + order by jobs.created_at desc + """, + (week["id"],), + ).fetchall() + return [dict(row) for row in rows] + + def get_source(self, source_id: int) -> dict[str, Any]: + with self._connect() as conn: + row = conn.execute("select * from sources where id = ?", (source_id,)).fetchone() + if not row: + raise ValueError("Fuente no encontrada") + return dict(row) + + def list_sources(self, subject_id: int, week_number: int) -> list[dict[str, Any]]: + week = self.get_week(subject_id, week_number) + with self._connect() as conn: + rows = conn.execute( + """ + select sources.*, documents.id as document_id, documents.title as document_title, + documents.markdown_path as document_markdown_path, documents.created_at as document_created_at + from sources + left join documents on documents.source_id = sources.id + where sources.week_id = ? + order by sources.created_at desc + """, + (week["id"],), + ).fetchall() + return [dict(row) for row in rows] + + def delete_source(self, source_id: int) -> None: + source = self.get_source(source_id) + with self._connect() as conn: + doc_rows = conn.execute("select markdown_path from documents where source_id = ?", (source_id,)).fetchall() + conn.execute("delete from documents where source_id = ?", (source_id,)) + conn.execute("delete from jobs where source_id = ?", (source_id,)) + conn.execute("delete from sources where id = ?", (source_id,)) + Path(source["stored_path"]).unlink(missing_ok=True) + for row in doc_rows: + Path(row["markdown_path"]).unlink(missing_ok=True) + + def create_document(self, source_id: int, title: str, markdown: str, processor: str | None = None, page_ranges: str | None = None) -> dict[str, Any]: + source = self.get_source(source_id) + week_context = self.get_week_context(source["week_id"]) + processed_dir = self.week_dir(week_context["subject_slug"], week_context["week_number"]) / "processed" + processed_dir.mkdir(parents=True, exist_ok=True) + markdown_path = self._unique_path(processed_dir / f"{slugify(title)}.md") + markdown_path.write_text(self._sanitize_markdown(markdown), encoding="utf-8") + now = utc_now() + with self._connect() as conn: + old_docs = conn.execute("select markdown_path from documents where source_id = ?", (source_id,)).fetchall() + conn.execute("delete from jobs where document_id in (select id from documents where source_id = ?)", (source_id,)) + conn.execute("delete from documents where source_id = ?", (source_id,)) + cursor = conn.execute( + "insert into documents (source_id, title, markdown_path, created_at) values (?, ?, ?, ?)", + (source_id, title, str(markdown_path), now), + ) + conn.execute( + "update sources set status = ?, processor = ?, page_ranges = ?, processed_at = ? where id = ?", + ("processed", processor, page_ranges, now, source_id), + ) + for row in old_docs: + Path(row["markdown_path"]).unlink(missing_ok=True) + return self.get_document(cursor.lastrowid) + + def get_document(self, document_id: int) -> dict[str, Any]: + with self._connect() as conn: + row = conn.execute("select * from documents where id = ?", (document_id,)).fetchone() + if not row: + raise ValueError("Documento no encontrado") + return dict(row) + + def update_document_markdown(self, document_id: int, content: str) -> dict[str, Any]: + document = self.get_document(document_id) + Path(document["markdown_path"]).write_text(self._sanitize_markdown(content), encoding="utf-8") + return self.get_document(document_id) + + def delete_document(self, document_id: int) -> None: + document = self.get_document(document_id) + Path(document["markdown_path"]).unlink(missing_ok=True) + with self._connect() as conn: + conn.execute("delete from jobs where document_id = ?", (document_id,)) + conn.execute("delete from documents where id = ?", (document_id,)) + + def list_documents(self, subject_id: int, week_number: int) -> list[dict[str, Any]]: + week = self.get_week(subject_id, week_number) + with self._connect() as conn: + rows = conn.execute( + """ + select documents.*, sources.original_name as source_original_name, + sources.source_type as source_type, sources.status as source_status, + sources.processor as source_processor, sources.page_ranges as source_page_ranges + from documents + join sources on sources.id = documents.source_id + where sources.week_id = ? + order by documents.created_at desc + """, + (week["id"],), + ).fetchall() + return [dict(row) for row in rows] + + def get_week_context(self, week_id: int) -> dict[str, Any]: + with self._connect() as conn: + row = conn.execute( + """ + select weeks.id as week_id, weeks.number as week_number, weeks.title as week_title, + subjects.id as subject_id, subjects.name as subject_name, subjects.slug as subject_slug + from weeks join subjects on subjects.id = weeks.subject_id + where weeks.id = ? + """, + (week_id,), + ).fetchone() + if not row: + raise ValueError("Contexto de semana no encontrado") + return dict(row) + + def export_week_markdown(self, subject_id: int, week_number: int) -> Path: + subject = self.get_subject(subject_id) + docs = self.list_documents(subject_id, week_number) + export_dir = self.week_dir(subject["slug"], week_number) / "exports" + export_dir.mkdir(parents=True, exist_ok=True) + export_path = export_dir / f"{subject['slug']}-week-{week_number:02d}.md" + parts = [ + "---\n" + f"subject: {subject['name']}\n" + f"subject_slug: {subject['slug']}\n" + f"week: {week_number}\n" + f"documents: {len(docs)}\n" + f"exported_at: {utc_now()}\n" + "---\n\n" + f"# {subject['name']} - Semana {week_number}\n" + ] + for doc in docs: + path = Path(doc["markdown_path"]) + content = self._strip_frontmatter(self._read_markdown(path)).strip() + parts.append(f"\n\n{content}\n") + export_path.write_text(self._sanitize_markdown("\n\n---\n\n".join(parts)), encoding="utf-8") + return export_path + + def export_week_markdown_zip(self, subject_id: int, week_number: int) -> Path: + subject = self.get_subject(subject_id) + docs = self.list_documents(subject_id, week_number) + export_dir = self.week_dir(subject["slug"], week_number) / "exports" + export_dir.mkdir(parents=True, exist_ok=True) + export_path = export_dir / f"{subject['slug']}-week-{week_number:02d}-documentos.zip" + used_names: set[str] = set() + with zipfile.ZipFile(export_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for index, doc in enumerate(docs, start=1): + path = Path(doc["markdown_path"]) + name = self._unique_archive_name(f"{index:02d}-{slugify(doc['title'])}.md", used_names) + archive.writestr(name, self._read_markdown(path)) + return export_path + + def _unique_archive_name(self, filename: str, used_names: set[str]) -> str: + path = Path(filename) + candidate = path.name + suffix = path.suffix + stem = path.stem + counter = 2 + while candidate in used_names: + candidate = f"{stem}-{counter}{suffix}" + counter += 1 + used_names.add(candidate) + return candidate + + def _read_markdown(self, path: Path) -> str: + return self._sanitize_markdown(path.read_text(encoding="utf-8", errors="replace")) + + def _sanitize_markdown(self, content: str) -> str: + return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "", content) + + def _strip_frontmatter(self, content: str) -> str: + lines = content.splitlines() + if not lines or lines[0].strip() != "---": + return content + for index, line in enumerate(lines[1:], start=1): + if line.strip() == "---": + return "\n".join(lines[index + 1 :]).lstrip() + return content + + def detect_source_type(self, path: Path) -> str: + ext = path.suffix.lower().lstrip(".") + if ext in {"txt", "md"}: + return "text" + if ext == "docx": + return "docx" + if ext == "pdf": + return "pdf" + if ext in {"mp3", "wav", "m4a", "ogg", "flac", "webm"}: + return "audio" + if ext in {"mp4", "mov", "mkv", "avi"}: + return "video" + return "unknown" + + def _unique_path(self, path: Path) -> Path: + if not path.exists(): + return path + stem = path.stem + suffix = path.suffix + for index in range(2, 10_000): + candidate = path.with_name(f"{stem}-{index}{suffix}") + if not candidate.exists(): + return candidate + raise RuntimeError("No se pudo generar un nombre de archivo unico") + + +def rows_to_models(rows: Iterable[dict[str, Any]]) -> list[dict[str, Any]]: + return [dict(row) for row in rows] diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py new file mode 100644 index 0000000..61a88bb --- /dev/null +++ b/backend/app/models/schemas.py @@ -0,0 +1,55 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, Field + + +class SubjectCreate(BaseModel): + name: str = Field(min_length=1) + + +class Subject(BaseModel): + id: int + name: str + slug: str + created_at: datetime + + +class WeekCreate(BaseModel): + number: int = Field(ge=1) + title: Optional[str] = None + + +class WeekUpdate(BaseModel): + number: int = Field(ge=1) + title: Optional[str] = None + + +class Week(BaseModel): + id: int + subject_id: int + number: int + title: Optional[str] + created_at: datetime + + +class MarkdownUpdate(BaseModel): + content: str + + +class Job(BaseModel): + id: int + status: str + message: str + source_id: Optional[int] = None + document_id: Optional[int] = None + created_at: datetime + updated_at: datetime + + +class Document(BaseModel): + id: int + source_id: int + title: str + markdown_path: str + created_at: datetime diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/services/ingestion_service.py b/backend/app/services/ingestion_service.py new file mode 100644 index 0000000..55cad32 --- /dev/null +++ b/backend/app/services/ingestion_service.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from pathlib import Path + +from backend.app.managers.knowledge_manager import KnowledgeManager +from backend.app.services.markdown_builder import build_markdown +from backend.app.services.processors.audio_processor import AudioProcessor +from backend.app.services.processors.docx_processor import DocxProcessor +from backend.app.services.processors.pdf_processor import PdfProcessor +from backend.app.services.processors.text_processor import TextProcessor +from backend.app.services.processors.video_processor import VideoProcessor + + +class IngestionService: + def __init__(self, manager: KnowledgeManager): + self.manager = manager + + def process_source(self, source_id: int, use_ocr: bool = False, page_ranges: str | None = None) -> dict: + source = self.manager.get_source(source_id) + context = self.manager.get_week_context(source["week_id"]) + path = Path(source["stored_path"]) + processor = self._processor_for(source["source_type"], use_ocr, page_ranges) + processed = processor.process(path) + markdown = build_markdown( + title=processed.title, + body=processed.body, + metadata={ + "subject": context["subject_name"], + "subject_slug": context["subject_slug"], + "week": context["week_number"], + "source_file": source["original_name"], + "source_type": source["source_type"], + "processor": processed.processor, + "page_ranges": page_ranges if source["source_type"] == "pdf" and page_ranges else None, + "language": "es", + }, + ) + return self.manager.create_document(source_id, processed.title, markdown, processor=processed.processor, page_ranges=page_ranges) + + def _processor_for(self, source_type: str, use_ocr: bool, page_ranges: str | None): + if source_type == "text": + return TextProcessor() + if source_type == "docx": + return DocxProcessor() + if source_type == "pdf": + return PdfProcessor(use_ocr=use_ocr, page_ranges=page_ranges) + if source_type == "audio": + return AudioProcessor() + if source_type == "video": + return VideoProcessor(self.manager.tmp_dir) + raise RuntimeError(f"Tipo de fuente no soportado: {source_type}") diff --git a/backend/app/services/markdown_builder.py b/backend/app/services/markdown_builder.py new file mode 100644 index 0000000..8930be3 --- /dev/null +++ b/backend/app/services/markdown_builder.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +def build_markdown(title: str, body: str, metadata: dict[str, Any]) -> str: + frontmatter = { + **metadata, + "processed_at": datetime.now(timezone.utc).isoformat(), + } + lines = ["---"] + for key, value in frontmatter.items(): + if value is not None: + safe_value = str(value).replace("\n", " ") + lines.append(f"{key}: {safe_value}") + lines.extend(["---", "", f"# {title}", "", body.strip(), ""]) + return "\n".join(lines) + + +def title_from_path(path: Path) -> str: + return path.stem.replace("-", " ").replace("_", " ").strip().title() or "Documento" diff --git a/backend/app/services/processors/__init__.py b/backend/app/services/processors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/services/processors/audio_processor.py b/backend/app/services/processors/audio_processor.py new file mode 100644 index 0000000..36201a7 --- /dev/null +++ b/backend/app/services/processors/audio_processor.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from pathlib import Path + +from backend.app.services.markdown_builder import title_from_path +from backend.app.services.processors.base import ProcessedContent +from backend.app.services.providers.deepgram_client import DeepgramClient + + +class AudioProcessor: + def process(self, path: Path) -> ProcessedContent: + client = DeepgramClient() + data = client.transcribe_file(path) + transcript, confidence = client.transcript_from_response(data) + confidence_line = f"\n\n_Confianza estimada: {confidence * 100:.1f}%_" if isinstance(confidence, (int, float)) else "" + return ProcessedContent( + title=f"Transcripcion {title_from_path(path)}", + body=f"## Transcripcion\n\n{transcript}{confidence_line}", + processor="deepgram", + ) diff --git a/backend/app/services/processors/base.py b/backend/app/services/processors/base.py new file mode 100644 index 0000000..34a93ea --- /dev/null +++ b/backend/app/services/processors/base.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class ProcessedContent: + title: str + body: str + processor: str diff --git a/backend/app/services/processors/docx_processor.py b/backend/app/services/processors/docx_processor.py new file mode 100644 index 0000000..c92d356 --- /dev/null +++ b/backend/app/services/processors/docx_processor.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + +from docx import Document + +from backend.app.services.markdown_builder import title_from_path +from backend.app.services.processors.base import ProcessedContent + + +class DocxProcessor: + def process(self, path: Path) -> ProcessedContent: + doc = Document(path) + blocks: list[str] = [] + for paragraph in doc.paragraphs: + text = paragraph.text.strip() + if text: + blocks.append(text) + for table in doc.tables: + rows = [[cell.text.strip().replace("\n", " ") for cell in row.cells] for row in table.rows] + if rows: + blocks.append(self._table_to_markdown(rows)) + return ProcessedContent(title=title_from_path(path), body="\n\n".join(blocks), processor="docx") + + def _table_to_markdown(self, rows: list[list[str]]) -> str: + header = rows[0] + lines = [f"| {' | '.join(header)} |", f"| {' | '.join(['---'] * len(header))} |"] + for row in rows[1:]: + lines.append(f"| {' | '.join(row)} |") + return "\n".join(lines) diff --git a/backend/app/services/processors/pdf_processor.py b/backend/app/services/processors/pdf_processor.py new file mode 100644 index 0000000..597b986 --- /dev/null +++ b/backend/app/services/processors/pdf_processor.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path + +import fitz + +from backend.app.services.markdown_builder import title_from_path +from backend.app.services.processors.base import ProcessedContent +from backend.app.services.providers.mistral_client import MistralClient + + +class PdfProcessor: + def __init__(self, use_ocr: bool = False, page_ranges: str | None = None): + self.use_ocr = use_ocr + self.page_ranges = page_ranges + + def process(self, path: Path) -> ProcessedContent: + if self.use_ocr: + return self._process_ocr(path) + doc = fitz.open(path) + pages: list[str] = [] + for index in self._selected_pages(len(doc)): + page = doc[index - 1] + text = page.get_text("text").strip() + if text: + pages.append(f"## Pagina {index}\n\n{text}") + body = "\n\n".join(pages).strip() + if len(body) < 80: + raise RuntimeError("El PDF tiene poco texto extraible; procesalo con OCR") + return ProcessedContent(title=title_from_path(path), body=body, processor="pdf-text") + + def _process_ocr(self, path: Path) -> ProcessedContent: + doc = fitz.open(path) + client = MistralClient() + parts: list[str] = [] + temp_paths: list[Path] = [] + try: + for index in self._selected_pages(len(doc)): + page = doc[index - 1] + pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False) + image_path = path.with_name(f"{path.stem}-page-{index}.jpg") + pix.save(image_path) + temp_paths.append(image_path) + parts.append(f"## Pagina {index}\n\n{client.ocr_image(image_path)}") + finally: + for temp_path in temp_paths: + temp_path.unlink(missing_ok=True) + return ProcessedContent(title=title_from_path(path), body="\n\n".join(parts), processor="mistral-ocr") + + def _selected_pages(self, total_pages: int) -> list[int]: + if total_pages < 1: + raise RuntimeError("El PDF no tiene paginas") + if not self.page_ranges or not self.page_ranges.strip(): + return list(range(1, total_pages + 1)) + + selected: list[int] = [] + for raw_part in self.page_ranges.split(","): + part = raw_part.strip() + if not part: + continue + if "-" in part: + start_text, end_text = [piece.strip() for piece in part.split("-", 1)] + if not start_text.isdigit() or not end_text.isdigit(): + raise RuntimeError(f"Rango de paginas invalido: {part}") + start = int(start_text) + end = int(end_text) + if start > end: + raise RuntimeError(f"Rango de paginas invertido: {part}") + selected.extend(range(start, end + 1)) + else: + if not part.isdigit(): + raise RuntimeError(f"Pagina invalida: {part}") + selected.append(int(part)) + + unique_pages = sorted(set(selected)) + invalid = [page for page in unique_pages if page < 1 or page > total_pages] + if invalid: + raise RuntimeError(f"Paginas fuera de rango: {', '.join(map(str, invalid))}. El PDF tiene {total_pages} paginas") + if not unique_pages: + raise RuntimeError("No se seleccionaron paginas para procesar") + return unique_pages diff --git a/backend/app/services/processors/text_processor.py b/backend/app/services/processors/text_processor.py new file mode 100644 index 0000000..fa8c0a7 --- /dev/null +++ b/backend/app/services/processors/text_processor.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pathlib import Path + +from backend.app.services.markdown_builder import title_from_path +from backend.app.services.processors.base import ProcessedContent + + +class TextProcessor: + def process(self, path: Path) -> ProcessedContent: + return ProcessedContent( + title=title_from_path(path), + body=path.read_text(encoding="utf-8", errors="replace"), + processor="text", + ) diff --git a/backend/app/services/processors/video_processor.py b/backend/app/services/processors/video_processor.py new file mode 100644 index 0000000..f5cf330 --- /dev/null +++ b/backend/app/services/processors/video_processor.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +from backend.app.services.processors.audio_processor import AudioProcessor +from backend.app.services.processors.base import ProcessedContent + + +class VideoProcessor: + def __init__(self, tmp_dir: Path): + self.tmp_dir = tmp_dir + + def process(self, path: Path) -> ProcessedContent: + self.tmp_dir.mkdir(parents=True, exist_ok=True) + audio_path = self.tmp_dir / f"{path.stem}.wav" + try: + subprocess.run( + [ + "ffmpeg", + "-y", + "-i", + str(path), + "-vn", + "-acodec", + "pcm_s16le", + "-ar", + "16000", + "-ac", + "1", + str(audio_path), + ], + check=True, + capture_output=True, + text=True, + ) + processed = AudioProcessor().process(audio_path) + processed.title = f"Transcripcion {path.stem}" + processed.processor = "ffmpeg+deepgram" + return processed + except FileNotFoundError as exc: + raise RuntimeError("ffmpeg no esta instalado o no esta disponible en PATH") from exc + except subprocess.CalledProcessError as exc: + raise RuntimeError(f"No se pudo extraer audio del video: {exc.stderr}") from exc + finally: + audio_path.unlink(missing_ok=True) diff --git a/backend/app/services/providers/__init__.py b/backend/app/services/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/services/providers/deepgram_client.py b/backend/app/services/providers/deepgram_client.py new file mode 100644 index 0000000..c1ed664 --- /dev/null +++ b/backend/app/services/providers/deepgram_client.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from pathlib import Path + +import requests + +from backend.app.config import settings + + +class DeepgramClient: + endpoint = "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=es" + + def transcribe_file(self, path: Path) -> dict: + if not settings.deepgram_api_key: + raise RuntimeError("DEEPGRAM_API_KEY no esta configurada") + + headers = { + "Authorization": f"Token {settings.deepgram_api_key}", + "Content-Type": self._content_type(path), + } + with path.open("rb") as audio_file: + response = requests.post(self.endpoint, headers=headers, data=audio_file, timeout=600) + + if response.status_code == 401: + raise RuntimeError("API Key de Deepgram invalida") + if not response.ok: + try: + error = response.json() + except ValueError: + error = response.text + raise RuntimeError(f"Deepgram error {response.status_code}: {error}") + return response.json() + + def transcript_from_response(self, data: dict) -> tuple[str, float | None]: + alternative = data.get("results", {}).get("channels", [{}])[0].get("alternatives", [{}])[0] + transcript = alternative.get("paragraphs", {}).get("transcript") or alternative.get("transcript") or "" + confidence = alternative.get("confidence") + if not transcript.strip(): + raise RuntimeError("Deepgram no devolvio transcripcion") + return transcript.strip(), confidence + + def _content_type(self, path: Path) -> str: + ext = path.suffix.lower() + return { + ".mp3": "audio/mpeg", + ".wav": "audio/wav", + ".m4a": "audio/mp4", + ".mp4": "audio/mp4", + ".webm": "audio/webm", + ".ogg": "audio/ogg", + ".flac": "audio/flac", + }.get(ext, "application/octet-stream") diff --git a/backend/app/services/providers/mistral_client.py b/backend/app/services/providers/mistral_client.py new file mode 100644 index 0000000..ad524ec --- /dev/null +++ b/backend/app/services/providers/mistral_client.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import base64 +from pathlib import Path + +import requests + +from backend.app.config import settings + + +class MistralClient: + endpoint = "https://api.mistral.ai/v1/chat/completions" + model = "pixtral-12b-2409" + + def ocr_image(self, image_path: Path) -> str: + if not settings.mistral_api_key: + raise RuntimeError("MISTRAL_API_KEY no esta configurada") + data_url = self._image_data_url(image_path) + response = requests.post( + self.endpoint, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {settings.mistral_api_key}", + }, + json={ + "model": self.model, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Extract all text from this image and format it as Markdown. If there are tables, represent them using Markdown table syntax. Do not include commentary.", + }, + {"type": "image_url", "image_url": data_url}, + ], + } + ], + "max_tokens": 3000, + }, + timeout=180, + ) + if response.status_code == 401: + raise RuntimeError("API Key de Mistral invalida") + if not response.ok: + raise RuntimeError(f"Mistral error {response.status_code}: {response.text}") + data = response.json() + return data["choices"][0]["message"]["content"] + + def _image_data_url(self, image_path: Path) -> str: + encoded = base64.b64encode(image_path.read_bytes()).decode("ascii") + return f"data:image/jpeg;base64,{encoded}" diff --git a/backend/app/utils/__init__.py b/backend/app/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/utils/slug.py b/backend/app/utils/slug.py new file mode 100644 index 0000000..781be9b --- /dev/null +++ b/backend/app/utils/slug.py @@ -0,0 +1,8 @@ +import re +import unicodedata + + +def slugify(value: str) -> str: + normalized = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") + slug = re.sub(r"[^a-zA-Z0-9]+", "-", normalized).strip("-").lower() + return slug or "sin-titulo" diff --git a/cerrar_estacion.bat b/cerrar_estacion.bat new file mode 100644 index 0000000..01e3ee8 --- /dev/null +++ b/cerrar_estacion.bat @@ -0,0 +1,12 @@ +@echo off +setlocal +title Knowledge Station - Cerrar + +set "ROOT=%~dp0" +set "ROOT=%ROOT:~0,-1%" +powershell -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\scripts\stop-platform.ps1" -Root "%ROOT%" + +echo. +echo Puedes cerrar esta ventana. +pause +exit /b 0 diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..a8ab826 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,43 @@ +services: + backend: + build: + context: . + dockerfile: Dockerfile + environment: + APP_ENV: production + APP_DATA_DIR: /app/data + DATABASE_PATH: /app/data/app.db + MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} + DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-} + volumes: + - station-data:/app/data + expose: + - "8000" + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + + frontend: + build: + context: ./frontend + dockerfile: Dockerfile + args: + NEXT_PUBLIC_API_URL: "" + BACKEND_INTERNAL_URL: http://backend:8000 + environment: + NODE_ENV: production + PORT: 3000 + HOSTNAME: 0.0.0.0 + expose: + - "3000" + depends_on: + backend: + condition: service_healthy + restart: unless-stopped + +volumes: + station-data: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..262a650 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +services: + backend: + build: . + ports: + - "8000:8000" + env_file: + - .env + volumes: + - ./data:/app/data + frontend: + build: + context: ./frontend + ports: + - "3000:3000" + environment: + NEXT_PUBLIC_API_URL: http://127.0.0.1:8000 + depends_on: + - backend diff --git a/estacion documentos.html b/estacion documentos.html new file mode 100644 index 0000000..80f4644 --- /dev/null +++ b/estacion documentos.html @@ -0,0 +1,1618 @@ + + + + + + + Estación de Documentos Premium + + + + + + + + + + + + + + + + + + + + + +
+
+

+ Estación de Documentos +

+
+ Arrastra PDF, DOCX, TXT, Audio, Video +
+
+ +
+ +
+ + +
+
+
+ +
+ +
+
+ Documentos (Entrada) + Listo +
+ +
+ +
+ +
+
+ +
+ + + + + + +
+
+ + +
+ +
+ + +
+ +
+ + + + + +
+ +
+
+ +
+ + + +
+
+ + Suelta para Importar
+ PDF, DOCX, TXT, Audio, Video +
+
+ +
+
+
+
+
+ + +
+
+ Vista Previa (Rich Text) + Sincronizado +
+
+
+
+
+
+
+ + +
+ + + +
+
+ Procesamiento de PDF +
+

+ Este PDF parece contener imágenes o ser un escaneo. ¿Cómo deseas procesarlo? +

+ +
+ +
+ + +
+
+ + +
+
+ + + +
+ + + +
+
+ + + + +
+
+ Configuración +
+
+ + + + La llave se guarda localmente en tu navegador. + +
+
+ + + + La llave se guarda localmente en tu navegador. + +
+
+ + +
+
+ + + + + + diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 0000000..bbf5bbf --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1,6 @@ +.next +node_modules +.env.local +.env.*.local +npm-debug.log* +Dockerfile* diff --git a/frontend/.env.example b/frontend/.env.example new file mode 100644 index 0000000..a704b71 --- /dev/null +++ b/frontend/.env.example @@ -0,0 +1 @@ +NEXT_PUBLIC_API_URL=http://127.0.0.1:8000 diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000..37dbf95 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,24 @@ +FROM node:20-alpine AS deps +WORKDIR /app +COPY package.json package-lock.json* ./ +RUN npm ci + +FROM node:20-alpine AS builder +WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules +COPY . . +ARG NEXT_PUBLIC_API_URL= +ARG BACKEND_INTERNAL_URL=http://backend:8000 +ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL +ENV BACKEND_INTERNAL_URL=$BACKEND_INTERNAL_URL +RUN npm run build + +FROM node:20-alpine AS runner +WORKDIR /app +ENV NODE_ENV=production +ENV HOSTNAME=0.0.0.0 +COPY --from=builder /app/.next/standalone ./ +COPY --from=builder /app/.next/static ./.next/static +COPY --from=builder /app/public ./public +EXPOSE 3000 +CMD ["node", "server.js"] diff --git a/frontend/app/documents/[documentId]/page.tsx b/frontend/app/documents/[documentId]/page.tsx new file mode 100644 index 0000000..fb676d9 --- /dev/null +++ b/frontend/app/documents/[documentId]/page.tsx @@ -0,0 +1,65 @@ +"use client"; + +import { useEffect, useState } from "react"; +import Link from "next/link"; +import ReactMarkdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import { api, DocumentItem } from "@/lib/api"; +import { Shell } from "@/components/Shell"; +import { Button, Card } from "@/components/ui"; + +export default function DocumentPage({ params }: { params: { documentId: string } }) { + const documentId = Number(params.documentId); + const [document, setDocument] = useState(null); + const [content, setContent] = useState(""); + const [savedContent, setSavedContent] = useState(""); + const [status, setStatus] = useState(""); + const dirty = content !== savedContent; + + useEffect(() => { + Promise.all([api.document(documentId), api.markdown(documentId)]).then(([doc, markdown]) => { + setDocument(doc); setContent(markdown); setSavedContent(markdown); + }).catch((err) => setStatus(err.message)); + }, [documentId]); + + async function save() { + setStatus("Guardando..."); + await api.saveMarkdown(documentId, content); + setSavedContent(content); + setStatus("Guardado"); + } + + async function copy() { + await navigator.clipboard.writeText(content); + setStatus("Copiado al portapapeles"); + } + + return ( + + Volver al dashboard +
+
+

{document?.title || "Documento"}

+

{dirty ? "Cambios sin guardar" : "Sin cambios pendientes"} {status && `- ${status}`}

+
+
+ + + Descargar .md +
+
+
+ +
Editor Markdown
+