This commit is contained in:
urieljarethbusiness-cpu
2026-05-17 10:14:14 -06:00
parent d8773b2508
commit 64b3d15b90
61 changed files with 7612 additions and 0 deletions
View File
View File
+16
View File
@@ -0,0 +1,16 @@
from pathlib import Path
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
app_env: str = "development"
app_data_dir: Path = Path("./data")
database_path: Path = Path("./data/app.db")
mistral_api_key: str = ""
deepgram_api_key: str = ""
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
settings = Settings()
+281
View File
@@ -0,0 +1,281 @@
from __future__ import annotations
from pathlib import Path
from fastapi import BackgroundTasks, FastAPI, File, HTTPException, Query, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, PlainTextResponse
from backend.app.config import settings
from backend.app.managers.knowledge_manager import DuplicateSourceError, KnowledgeManager
from backend.app.models.schemas import MarkdownUpdate, SubjectCreate, WeekCreate, WeekUpdate
from backend.app.services.ingestion_service import IngestionService
app = FastAPI(title="Knowledge Station", version="0.1.0")
app.add_middleware(
CORSMiddleware,
allow_origins=["http://127.0.0.1:3000", "http://localhost:3000"],
allow_origin_regex=r"http://(127\.0\.0\.1|localhost):\d+",
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
manager = KnowledgeManager()
ingestion = IngestionService(manager)
@app.get("/health")
def health() -> dict[str, str]:
return {"status": "ok"}
@app.get("/settings/status")
def settings_status() -> dict:
return {
"mistral_configured": bool(settings.mistral_api_key),
"deepgram_configured": bool(settings.deepgram_api_key),
"data_dir": str(settings.app_data_dir),
"database_path": str(settings.database_path),
}
@app.post("/subjects")
def create_subject(payload: SubjectCreate) -> dict:
try:
return manager.create_subject(payload.name)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.get("/subjects")
def list_subjects() -> list[dict]:
return manager.list_subjects()
@app.get("/subjects/{subject_id}")
def get_subject(subject_id: int) -> dict:
try:
return manager.get_subject(subject_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.delete("/subjects/{subject_id}")
def delete_subject(subject_id: int) -> dict:
try:
manager.delete_subject(subject_id)
return {"status": "deleted"}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/subjects/{subject_id}/weeks")
def create_week(subject_id: int, payload: WeekCreate) -> dict:
try:
return manager.create_week(subject_id, payload.number, payload.title)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks")
def list_weeks(subject_id: int) -> list[dict]:
try:
manager.get_subject(subject_id)
return manager.list_weeks(subject_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}")
def get_week(subject_id: int, week_number: int) -> dict:
try:
return manager.get_week(subject_id, week_number)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.put("/subjects/{subject_id}/weeks/{week_number}")
def update_week(subject_id: int, week_number: int, payload: WeekUpdate) -> dict:
try:
return manager.update_week(subject_id, week_number, payload.number, payload.title)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.delete("/subjects/{subject_id}/weeks/{week_number}")
def delete_week(subject_id: int, week_number: int) -> dict:
try:
manager.delete_week(subject_id, week_number)
return {"status": "deleted"}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/subjects/{subject_id}/weeks/{week_number}/files")
def upload_file(
subject_id: int,
week_number: int,
background_tasks: BackgroundTasks,
file: UploadFile = File(...),
use_ocr: bool = Query(False, description="Usar Mistral OCR para PDFs"),
page_ranges: str | None = Query(None, description="Rangos de paginas PDF, por ejemplo: 1-5,8,10-12"),
) -> dict:
try:
source = manager.save_upload(subject_id, week_number, file)
job = manager.create_job(source["id"], "Archivo recibido")
background_tasks.add_task(process_job, job["id"], source["id"], use_ocr, page_ranges)
return {"source": source, "job": job}
except DuplicateSourceError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/jobs/{job_id}")
def get_job(job_id: int) -> dict:
try:
return manager.get_job(job_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/jobs/{job_id}/retry")
def retry_job(
job_id: int,
background_tasks: BackgroundTasks,
use_ocr: bool = Query(False),
page_ranges: str | None = Query(None),
) -> dict:
try:
job = manager.get_job(job_id)
if not job.get("source_id"):
raise HTTPException(status_code=400, detail="El trabajo no tiene fuente asociada")
manager.update_job(job_id, "queued", "Reintento en cola")
background_tasks.add_task(process_job, job_id, job["source_id"], use_ocr, page_ranges)
return manager.get_job(job_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/jobs")
def list_week_jobs(subject_id: int, week_number: int) -> list[dict]:
try:
return manager.list_jobs_for_week(subject_id, week_number)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/sources")
def list_sources(subject_id: int, week_number: int) -> list[dict]:
try:
return manager.list_sources(subject_id, week_number)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.delete("/sources/{source_id}")
def delete_source(source_id: int) -> dict:
try:
manager.delete_source(source_id)
return {"status": "deleted"}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/sources/{source_id}/reprocess")
def reprocess_source(
source_id: int,
background_tasks: BackgroundTasks,
use_ocr: bool = Query(False),
page_ranges: str | None = Query(None),
) -> dict:
try:
source = manager.get_source(source_id)
if manager.source_has_active_job(source_id):
raise HTTPException(status_code=409, detail="La fuente ya tiene un trabajo en cola o en proceso")
job = manager.create_job(source["id"], "Reprocesamiento en cola")
background_tasks.add_task(process_job, job["id"], source["id"], use_ocr, page_ranges)
return {"source": source, "job": job}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/documents")
def list_documents(subject_id: int, week_number: int) -> list[dict]:
try:
return manager.list_documents(subject_id, week_number)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/documents/{document_id}")
def get_document(document_id: int) -> dict:
try:
return manager.get_document(document_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/documents/{document_id}/markdown", response_class=PlainTextResponse)
def get_document_markdown(document_id: int) -> str:
try:
document = manager.get_document(document_id)
return Path(document["markdown_path"]).read_text(encoding="utf-8")
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.put("/documents/{document_id}/markdown")
def update_document_markdown(document_id: int, payload: MarkdownUpdate) -> dict:
try:
return manager.update_document_markdown(document_id, payload.content)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/documents/{document_id}/download")
def download_document(document_id: int) -> FileResponse:
try:
document = manager.get_document(document_id)
path = Path(document["markdown_path"])
return FileResponse(path, filename=path.name, media_type="text/markdown")
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.delete("/documents/{document_id}")
def delete_document(document_id: int) -> dict:
try:
manager.delete_document(document_id)
return {"status": "deleted"}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/export")
def export_week(subject_id: int, week_number: int) -> FileResponse:
try:
path = manager.export_week_markdown(subject_id, week_number)
return FileResponse(path, filename=path.name, media_type="text/markdown")
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/export.zip")
def export_week_zip(subject_id: int, week_number: int) -> FileResponse:
try:
path = manager.export_week_markdown_zip(subject_id, week_number)
return FileResponse(path, filename=path.name, media_type="application/zip")
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
def process_job(job_id: int, source_id: int, use_ocr: bool, page_ranges: str | None = None) -> None:
manager.update_job(job_id, "processing", "Procesando archivo")
manager.update_source_status(source_id, "processing")
try:
document = ingestion.process_source(source_id, use_ocr=use_ocr, page_ranges=page_ranges)
manager.update_job(job_id, "completed", "Documento procesado", document_id=document["id"])
except Exception as exc:
manager.update_source_status(source_id, "failed")
manager.update_job(job_id, "failed", str(exc))
View File
+548
View File
@@ -0,0 +1,548 @@
from __future__ import annotations
import shutil
import sqlite3
import re
import hashlib
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
from fastapi import UploadFile
from backend.app.config import settings
from backend.app.utils.slug import slugify
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
class DuplicateSourceError(ValueError):
def __init__(self, existing_source: dict[str, Any]):
self.existing_source = existing_source
super().__init__(f"El archivo ya existe en esta semana: {existing_source['original_name']}")
class KnowledgeManager:
def __init__(self, data_dir: Path | None = None, database_path: Path | None = None):
self.data_dir = data_dir or settings.app_data_dir
self.database_path = database_path or settings.database_path
self.subjects_dir = self.data_dir / "subjects"
self.tmp_dir = self.data_dir / "tmp"
self.data_dir.mkdir(parents=True, exist_ok=True)
self.subjects_dir.mkdir(parents=True, exist_ok=True)
self.tmp_dir.mkdir(parents=True, exist_ok=True)
self.database_path.parent.mkdir(parents=True, exist_ok=True)
self._init_db()
def _connect(self) -> sqlite3.Connection:
conn = sqlite3.connect(self.database_path)
conn.row_factory = sqlite3.Row
return conn
def _init_db(self) -> None:
with self._connect() as conn:
conn.executescript(
"""
create table if not exists subjects (
id integer primary key autoincrement,
name text not null,
slug text not null unique,
created_at text not null
);
create table if not exists weeks (
id integer primary key autoincrement,
subject_id integer not null,
number integer not null,
title text,
created_at text not null,
unique(subject_id, number),
foreign key(subject_id) references subjects(id)
);
create table if not exists sources (
id integer primary key autoincrement,
week_id integer not null,
original_name text not null,
stored_path text not null,
source_type text not null,
status text not null,
file_size integer,
sha256 text,
mime_type text,
processor text,
page_ranges text,
processed_at text,
created_at text not null,
foreign key(week_id) references weeks(id)
);
create table if not exists documents (
id integer primary key autoincrement,
source_id integer not null,
title text not null,
markdown_path text not null,
created_at text not null,
foreign key(source_id) references sources(id)
);
create table if not exists jobs (
id integer primary key autoincrement,
status text not null,
message text not null,
source_id integer,
document_id integer,
created_at text not null,
updated_at text not null,
foreign key(source_id) references sources(id),
foreign key(document_id) references documents(id)
);
"""
)
self._ensure_columns(
conn,
"sources",
{
"file_size": "integer",
"sha256": "text",
"mime_type": "text",
"processor": "text",
"page_ranges": "text",
"processed_at": "text",
},
)
self._backfill_source_file_metadata(conn)
def _ensure_columns(self, conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None:
existing = {row["name"] for row in conn.execute(f"pragma table_info({table})").fetchall()}
for name, definition in columns.items():
if name not in existing:
conn.execute(f"alter table {table} add column {name} {definition}")
def _backfill_source_file_metadata(self, conn: sqlite3.Connection) -> None:
rows = conn.execute("select id, stored_path from sources where sha256 is null or file_size is null").fetchall()
for row in rows:
path = Path(row["stored_path"])
if not path.exists():
continue
sha256, file_size = self._file_hash_and_size(path)
conn.execute("update sources set sha256 = ?, file_size = ? where id = ?", (sha256, file_size, row["id"]))
def create_subject(self, name: str) -> dict[str, Any]:
base_slug = slugify(name)
slug = base_slug
suffix = 2
with self._connect() as conn:
while conn.execute("select 1 from subjects where slug = ?", (slug,)).fetchone():
slug = f"{base_slug}-{suffix}"
suffix += 1
now = utc_now()
cursor = conn.execute(
"insert into subjects (name, slug, created_at) values (?, ?, ?)",
(name, slug, now),
)
(self.subjects_dir / slug).mkdir(parents=True, exist_ok=True)
return self.get_subject(cursor.lastrowid)
def get_subject(self, subject_id: int) -> dict[str, Any]:
with self._connect() as conn:
row = conn.execute("select * from subjects where id = ?", (subject_id,)).fetchone()
if not row:
raise ValueError("Materia no encontrada")
return dict(row)
def delete_subject(self, subject_id: int) -> None:
subject = self.get_subject(subject_id)
with self._connect() as conn:
week_rows = conn.execute("select id from weeks where subject_id = ?", (subject_id,)).fetchall()
week_ids = [row["id"] for row in week_rows]
for week_id in week_ids:
source_rows = conn.execute("select id from sources where week_id = ?", (week_id,)).fetchall()
source_ids = [row["id"] for row in source_rows]
for source_id in source_ids:
conn.execute("delete from documents where source_id = ?", (source_id,))
conn.execute("delete from jobs where source_id = ?", (source_id,))
conn.execute("delete from sources where week_id = ?", (week_id,))
conn.execute("delete from weeks where subject_id = ?", (subject_id,))
conn.execute("delete from subjects where id = ?", (subject_id,))
shutil.rmtree(self.subjects_dir / subject["slug"], ignore_errors=True)
def list_subjects(self) -> list[dict[str, Any]]:
with self._connect() as conn:
rows = conn.execute("select * from subjects order by created_at desc").fetchall()
return [dict(row) for row in rows]
def create_week(self, subject_id: int, number: int, title: str | None = None) -> dict[str, Any]:
subject = self.get_subject(subject_id)
now = utc_now()
with self._connect() as conn:
conn.execute(
"insert or ignore into weeks (subject_id, number, title, created_at) values (?, ?, ?, ?)",
(subject_id, number, title, now),
)
if title:
conn.execute("update weeks set title = ? where subject_id = ? and number = ?", (title, subject_id, number))
row = conn.execute(
"select * from weeks where subject_id = ? and number = ?",
(subject_id, number),
).fetchone()
self.week_dir(subject["slug"], number).mkdir(parents=True, exist_ok=True)
for child in ("raw", "processed", "exports"):
(self.week_dir(subject["slug"], number) / child).mkdir(parents=True, exist_ok=True)
return dict(row)
def list_weeks(self, subject_id: int) -> list[dict[str, Any]]:
with self._connect() as conn:
rows = conn.execute("select * from weeks where subject_id = ? order by number", (subject_id,)).fetchall()
return [dict(row) for row in rows]
def delete_week(self, subject_id: int, number: int) -> None:
subject = self.get_subject(subject_id)
week = self.get_week(subject_id, number)
with self._connect() as conn:
source_rows = conn.execute("select id from sources where week_id = ?", (week["id"],)).fetchall()
for row in source_rows:
conn.execute("delete from documents where source_id = ?", (row["id"],))
conn.execute("delete from jobs where source_id = ?", (row["id"],))
conn.execute("delete from sources where week_id = ?", (week["id"],))
conn.execute("delete from weeks where id = ?", (week["id"],))
shutil.rmtree(self.week_dir(subject["slug"], number), ignore_errors=True)
def get_week(self, subject_id: int, number: int) -> dict[str, Any]:
with self._connect() as conn:
row = conn.execute(
"select * from weeks where subject_id = ? and number = ?",
(subject_id, number),
).fetchone()
if not row:
raise ValueError("Semana no encontrada")
return dict(row)
def update_week(self, subject_id: int, current_number: int, new_number: int, title: str | None = None) -> dict[str, Any]:
subject = self.get_subject(subject_id)
week = self.get_week(subject_id, current_number)
if new_number != current_number:
with self._connect() as conn:
existing = conn.execute(
"select 1 from weeks where subject_id = ? and number = ?",
(subject_id, new_number),
).fetchone()
if existing:
raise ValueError("Ya existe una semana con ese numero")
old_dir = self.week_dir(subject["slug"], current_number)
new_dir = self.week_dir(subject["slug"], new_number)
if old_dir.exists() and new_dir.exists():
raise ValueError("Ya existe una carpeta para ese numero de semana")
if old_dir.exists():
old_dir.rename(new_dir)
else:
for child in ("raw", "processed", "exports"):
(new_dir / child).mkdir(parents=True, exist_ok=True)
with self._connect() as conn:
conn.execute(
"update weeks set number = ?, title = ? where id = ?",
(new_number, title, week["id"]),
)
return self.get_week(subject_id, new_number)
def week_dir(self, subject_slug: str, week_number: int) -> Path:
return self.subjects_dir / subject_slug / f"week-{week_number:02d}"
def save_upload(self, subject_id: int, week_number: int, upload: UploadFile) -> dict[str, Any]:
subject = self.get_subject(subject_id)
week = self.get_week(subject_id, week_number)
filename = Path(upload.filename or "archivo").name
raw_dir = self.week_dir(subject["slug"], week_number) / "raw"
raw_dir.mkdir(parents=True, exist_ok=True)
stored_path = self._unique_path(raw_dir / filename)
digest = hashlib.sha256()
file_size = 0
with stored_path.open("wb") as out_file:
while chunk := upload.file.read(1024 * 1024):
file_size += len(chunk)
digest.update(chunk)
out_file.write(chunk)
sha256 = digest.hexdigest()
source_type = self.detect_source_type(stored_path)
now = utc_now()
with self._connect() as conn:
duplicate = conn.execute(
"select * from sources where week_id = ? and sha256 = ? limit 1",
(week["id"], sha256),
).fetchone()
if duplicate:
stored_path.unlink(missing_ok=True)
raise DuplicateSourceError(dict(duplicate))
cursor = conn.execute(
"""
insert into sources (week_id, original_name, stored_path, source_type, status, file_size, sha256, mime_type, created_at)
values (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(week["id"], filename, str(stored_path), source_type, "stored", file_size, sha256, upload.content_type, now),
)
return self.get_source(cursor.lastrowid)
def _file_hash_and_size(self, path: Path) -> tuple[str, int]:
digest = hashlib.sha256()
file_size = 0
with path.open("rb") as file:
while chunk := file.read(1024 * 1024):
file_size += len(chunk)
digest.update(chunk)
return digest.hexdigest(), file_size
def create_job(self, source_id: int, message: str = "En cola") -> dict[str, Any]:
now = utc_now()
with self._connect() as conn:
cursor = conn.execute(
"insert into jobs (status, message, source_id, created_at, updated_at) values (?, ?, ?, ?, ?)",
("queued", message, source_id, now, now),
)
return self.get_job(cursor.lastrowid)
def update_job(self, job_id: int, status: str, message: str, document_id: int | None = None) -> None:
with self._connect() as conn:
conn.execute(
"update jobs set status = ?, message = ?, document_id = coalesce(?, document_id), updated_at = ? where id = ?",
(status, message, document_id, utc_now(), job_id),
)
def update_source_status(self, source_id: int, status: str) -> None:
with self._connect() as conn:
conn.execute("update sources set status = ? where id = ?", (status, source_id))
def source_has_active_job(self, source_id: int) -> bool:
with self._connect() as conn:
row = conn.execute(
"select 1 from jobs where source_id = ? and status in ('queued', 'processing') limit 1",
(source_id,),
).fetchone()
return bool(row)
def get_job(self, job_id: int) -> dict[str, Any]:
with self._connect() as conn:
row = conn.execute("select * from jobs where id = ?", (job_id,)).fetchone()
if not row:
raise ValueError("Trabajo no encontrado")
return dict(row)
def list_jobs_for_week(self, subject_id: int, week_number: int) -> list[dict[str, Any]]:
week = self.get_week(subject_id, week_number)
with self._connect() as conn:
rows = conn.execute(
"""
select jobs.* from jobs
join sources on sources.id = jobs.source_id
where sources.week_id = ?
order by jobs.created_at desc
""",
(week["id"],),
).fetchall()
return [dict(row) for row in rows]
def get_source(self, source_id: int) -> dict[str, Any]:
with self._connect() as conn:
row = conn.execute("select * from sources where id = ?", (source_id,)).fetchone()
if not row:
raise ValueError("Fuente no encontrada")
return dict(row)
def list_sources(self, subject_id: int, week_number: int) -> list[dict[str, Any]]:
week = self.get_week(subject_id, week_number)
with self._connect() as conn:
rows = conn.execute(
"""
select sources.*, documents.id as document_id, documents.title as document_title,
documents.markdown_path as document_markdown_path, documents.created_at as document_created_at
from sources
left join documents on documents.source_id = sources.id
where sources.week_id = ?
order by sources.created_at desc
""",
(week["id"],),
).fetchall()
return [dict(row) for row in rows]
def delete_source(self, source_id: int) -> None:
source = self.get_source(source_id)
with self._connect() as conn:
doc_rows = conn.execute("select markdown_path from documents where source_id = ?", (source_id,)).fetchall()
conn.execute("delete from documents where source_id = ?", (source_id,))
conn.execute("delete from jobs where source_id = ?", (source_id,))
conn.execute("delete from sources where id = ?", (source_id,))
Path(source["stored_path"]).unlink(missing_ok=True)
for row in doc_rows:
Path(row["markdown_path"]).unlink(missing_ok=True)
def create_document(self, source_id: int, title: str, markdown: str, processor: str | None = None, page_ranges: str | None = None) -> dict[str, Any]:
source = self.get_source(source_id)
week_context = self.get_week_context(source["week_id"])
processed_dir = self.week_dir(week_context["subject_slug"], week_context["week_number"]) / "processed"
processed_dir.mkdir(parents=True, exist_ok=True)
markdown_path = self._unique_path(processed_dir / f"{slugify(title)}.md")
markdown_path.write_text(self._sanitize_markdown(markdown), encoding="utf-8")
now = utc_now()
with self._connect() as conn:
old_docs = conn.execute("select markdown_path from documents where source_id = ?", (source_id,)).fetchall()
conn.execute("delete from jobs where document_id in (select id from documents where source_id = ?)", (source_id,))
conn.execute("delete from documents where source_id = ?", (source_id,))
cursor = conn.execute(
"insert into documents (source_id, title, markdown_path, created_at) values (?, ?, ?, ?)",
(source_id, title, str(markdown_path), now),
)
conn.execute(
"update sources set status = ?, processor = ?, page_ranges = ?, processed_at = ? where id = ?",
("processed", processor, page_ranges, now, source_id),
)
for row in old_docs:
Path(row["markdown_path"]).unlink(missing_ok=True)
return self.get_document(cursor.lastrowid)
def get_document(self, document_id: int) -> dict[str, Any]:
with self._connect() as conn:
row = conn.execute("select * from documents where id = ?", (document_id,)).fetchone()
if not row:
raise ValueError("Documento no encontrado")
return dict(row)
def update_document_markdown(self, document_id: int, content: str) -> dict[str, Any]:
document = self.get_document(document_id)
Path(document["markdown_path"]).write_text(self._sanitize_markdown(content), encoding="utf-8")
return self.get_document(document_id)
def delete_document(self, document_id: int) -> None:
document = self.get_document(document_id)
Path(document["markdown_path"]).unlink(missing_ok=True)
with self._connect() as conn:
conn.execute("delete from jobs where document_id = ?", (document_id,))
conn.execute("delete from documents where id = ?", (document_id,))
def list_documents(self, subject_id: int, week_number: int) -> list[dict[str, Any]]:
week = self.get_week(subject_id, week_number)
with self._connect() as conn:
rows = conn.execute(
"""
select documents.*, sources.original_name as source_original_name,
sources.source_type as source_type, sources.status as source_status,
sources.processor as source_processor, sources.page_ranges as source_page_ranges
from documents
join sources on sources.id = documents.source_id
where sources.week_id = ?
order by documents.created_at desc
""",
(week["id"],),
).fetchall()
return [dict(row) for row in rows]
def get_week_context(self, week_id: int) -> dict[str, Any]:
with self._connect() as conn:
row = conn.execute(
"""
select weeks.id as week_id, weeks.number as week_number, weeks.title as week_title,
subjects.id as subject_id, subjects.name as subject_name, subjects.slug as subject_slug
from weeks join subjects on subjects.id = weeks.subject_id
where weeks.id = ?
""",
(week_id,),
).fetchone()
if not row:
raise ValueError("Contexto de semana no encontrado")
return dict(row)
def export_week_markdown(self, subject_id: int, week_number: int) -> Path:
subject = self.get_subject(subject_id)
docs = self.list_documents(subject_id, week_number)
export_dir = self.week_dir(subject["slug"], week_number) / "exports"
export_dir.mkdir(parents=True, exist_ok=True)
export_path = export_dir / f"{subject['slug']}-week-{week_number:02d}.md"
parts = [
"---\n"
f"subject: {subject['name']}\n"
f"subject_slug: {subject['slug']}\n"
f"week: {week_number}\n"
f"documents: {len(docs)}\n"
f"exported_at: {utc_now()}\n"
"---\n\n"
f"# {subject['name']} - Semana {week_number}\n"
]
for doc in docs:
path = Path(doc["markdown_path"])
content = self._strip_frontmatter(self._read_markdown(path)).strip()
parts.append(f"<!-- document_id: {doc['id']} -->\n\n{content}\n")
export_path.write_text(self._sanitize_markdown("\n\n---\n\n".join(parts)), encoding="utf-8")
return export_path
def export_week_markdown_zip(self, subject_id: int, week_number: int) -> Path:
subject = self.get_subject(subject_id)
docs = self.list_documents(subject_id, week_number)
export_dir = self.week_dir(subject["slug"], week_number) / "exports"
export_dir.mkdir(parents=True, exist_ok=True)
export_path = export_dir / f"{subject['slug']}-week-{week_number:02d}-documentos.zip"
used_names: set[str] = set()
with zipfile.ZipFile(export_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for index, doc in enumerate(docs, start=1):
path = Path(doc["markdown_path"])
name = self._unique_archive_name(f"{index:02d}-{slugify(doc['title'])}.md", used_names)
archive.writestr(name, self._read_markdown(path))
return export_path
def _unique_archive_name(self, filename: str, used_names: set[str]) -> str:
path = Path(filename)
candidate = path.name
suffix = path.suffix
stem = path.stem
counter = 2
while candidate in used_names:
candidate = f"{stem}-{counter}{suffix}"
counter += 1
used_names.add(candidate)
return candidate
def _read_markdown(self, path: Path) -> str:
return self._sanitize_markdown(path.read_text(encoding="utf-8", errors="replace"))
def _sanitize_markdown(self, content: str) -> str:
return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "", content)
def _strip_frontmatter(self, content: str) -> str:
lines = content.splitlines()
if not lines or lines[0].strip() != "---":
return content
for index, line in enumerate(lines[1:], start=1):
if line.strip() == "---":
return "\n".join(lines[index + 1 :]).lstrip()
return content
def detect_source_type(self, path: Path) -> str:
ext = path.suffix.lower().lstrip(".")
if ext in {"txt", "md"}:
return "text"
if ext == "docx":
return "docx"
if ext == "pdf":
return "pdf"
if ext in {"mp3", "wav", "m4a", "ogg", "flac", "webm"}:
return "audio"
if ext in {"mp4", "mov", "mkv", "avi"}:
return "video"
return "unknown"
def _unique_path(self, path: Path) -> Path:
if not path.exists():
return path
stem = path.stem
suffix = path.suffix
for index in range(2, 10_000):
candidate = path.with_name(f"{stem}-{index}{suffix}")
if not candidate.exists():
return candidate
raise RuntimeError("No se pudo generar un nombre de archivo unico")
def rows_to_models(rows: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
return [dict(row) for row in rows]
View File
+55
View File
@@ -0,0 +1,55 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel, Field
class SubjectCreate(BaseModel):
name: str = Field(min_length=1)
class Subject(BaseModel):
id: int
name: str
slug: str
created_at: datetime
class WeekCreate(BaseModel):
number: int = Field(ge=1)
title: Optional[str] = None
class WeekUpdate(BaseModel):
number: int = Field(ge=1)
title: Optional[str] = None
class Week(BaseModel):
id: int
subject_id: int
number: int
title: Optional[str]
created_at: datetime
class MarkdownUpdate(BaseModel):
content: str
class Job(BaseModel):
id: int
status: str
message: str
source_id: Optional[int] = None
document_id: Optional[int] = None
created_at: datetime
updated_at: datetime
class Document(BaseModel):
id: int
source_id: int
title: str
markdown_path: str
created_at: datetime
View File
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from pathlib import Path
from backend.app.managers.knowledge_manager import KnowledgeManager
from backend.app.services.markdown_builder import build_markdown
from backend.app.services.processors.audio_processor import AudioProcessor
from backend.app.services.processors.docx_processor import DocxProcessor
from backend.app.services.processors.pdf_processor import PdfProcessor
from backend.app.services.processors.text_processor import TextProcessor
from backend.app.services.processors.video_processor import VideoProcessor
class IngestionService:
def __init__(self, manager: KnowledgeManager):
self.manager = manager
def process_source(self, source_id: int, use_ocr: bool = False, page_ranges: str | None = None) -> dict:
source = self.manager.get_source(source_id)
context = self.manager.get_week_context(source["week_id"])
path = Path(source["stored_path"])
processor = self._processor_for(source["source_type"], use_ocr, page_ranges)
processed = processor.process(path)
markdown = build_markdown(
title=processed.title,
body=processed.body,
metadata={
"subject": context["subject_name"],
"subject_slug": context["subject_slug"],
"week": context["week_number"],
"source_file": source["original_name"],
"source_type": source["source_type"],
"processor": processed.processor,
"page_ranges": page_ranges if source["source_type"] == "pdf" and page_ranges else None,
"language": "es",
},
)
return self.manager.create_document(source_id, processed.title, markdown, processor=processed.processor, page_ranges=page_ranges)
def _processor_for(self, source_type: str, use_ocr: bool, page_ranges: str | None):
if source_type == "text":
return TextProcessor()
if source_type == "docx":
return DocxProcessor()
if source_type == "pdf":
return PdfProcessor(use_ocr=use_ocr, page_ranges=page_ranges)
if source_type == "audio":
return AudioProcessor()
if source_type == "video":
return VideoProcessor(self.manager.tmp_dir)
raise RuntimeError(f"Tipo de fuente no soportado: {source_type}")
+23
View File
@@ -0,0 +1,23 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
def build_markdown(title: str, body: str, metadata: dict[str, Any]) -> str:
frontmatter = {
**metadata,
"processed_at": datetime.now(timezone.utc).isoformat(),
}
lines = ["---"]
for key, value in frontmatter.items():
if value is not None:
safe_value = str(value).replace("\n", " ")
lines.append(f"{key}: {safe_value}")
lines.extend(["---", "", f"# {title}", "", body.strip(), ""])
return "\n".join(lines)
def title_from_path(path: Path) -> str:
return path.stem.replace("-", " ").replace("_", " ").strip().title() or "Documento"
@@ -0,0 +1,20 @@
from __future__ import annotations
from pathlib import Path
from backend.app.services.markdown_builder import title_from_path
from backend.app.services.processors.base import ProcessedContent
from backend.app.services.providers.deepgram_client import DeepgramClient
class AudioProcessor:
def process(self, path: Path) -> ProcessedContent:
client = DeepgramClient()
data = client.transcribe_file(path)
transcript, confidence = client.transcript_from_response(data)
confidence_line = f"\n\n_Confianza estimada: {confidence * 100:.1f}%_" if isinstance(confidence, (int, float)) else ""
return ProcessedContent(
title=f"Transcripcion {title_from_path(path)}",
body=f"## Transcripcion\n\n{transcript}{confidence_line}",
processor="deepgram",
)
+10
View File
@@ -0,0 +1,10 @@
from __future__ import annotations
from dataclasses import dataclass
@dataclass
class ProcessedContent:
title: str
body: str
processor: str
@@ -0,0 +1,30 @@
from __future__ import annotations
from pathlib import Path
from docx import Document
from backend.app.services.markdown_builder import title_from_path
from backend.app.services.processors.base import ProcessedContent
class DocxProcessor:
def process(self, path: Path) -> ProcessedContent:
doc = Document(path)
blocks: list[str] = []
for paragraph in doc.paragraphs:
text = paragraph.text.strip()
if text:
blocks.append(text)
for table in doc.tables:
rows = [[cell.text.strip().replace("\n", " ") for cell in row.cells] for row in table.rows]
if rows:
blocks.append(self._table_to_markdown(rows))
return ProcessedContent(title=title_from_path(path), body="\n\n".join(blocks), processor="docx")
def _table_to_markdown(self, rows: list[list[str]]) -> str:
header = rows[0]
lines = [f"| {' | '.join(header)} |", f"| {' | '.join(['---'] * len(header))} |"]
for row in rows[1:]:
lines.append(f"| {' | '.join(row)} |")
return "\n".join(lines)
@@ -0,0 +1,81 @@
from __future__ import annotations
from pathlib import Path
import fitz
from backend.app.services.markdown_builder import title_from_path
from backend.app.services.processors.base import ProcessedContent
from backend.app.services.providers.mistral_client import MistralClient
class PdfProcessor:
def __init__(self, use_ocr: bool = False, page_ranges: str | None = None):
self.use_ocr = use_ocr
self.page_ranges = page_ranges
def process(self, path: Path) -> ProcessedContent:
if self.use_ocr:
return self._process_ocr(path)
doc = fitz.open(path)
pages: list[str] = []
for index in self._selected_pages(len(doc)):
page = doc[index - 1]
text = page.get_text("text").strip()
if text:
pages.append(f"## Pagina {index}\n\n{text}")
body = "\n\n".join(pages).strip()
if len(body) < 80:
raise RuntimeError("El PDF tiene poco texto extraible; procesalo con OCR")
return ProcessedContent(title=title_from_path(path), body=body, processor="pdf-text")
def _process_ocr(self, path: Path) -> ProcessedContent:
doc = fitz.open(path)
client = MistralClient()
parts: list[str] = []
temp_paths: list[Path] = []
try:
for index in self._selected_pages(len(doc)):
page = doc[index - 1]
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
image_path = path.with_name(f"{path.stem}-page-{index}.jpg")
pix.save(image_path)
temp_paths.append(image_path)
parts.append(f"## Pagina {index}\n\n{client.ocr_image(image_path)}")
finally:
for temp_path in temp_paths:
temp_path.unlink(missing_ok=True)
return ProcessedContent(title=title_from_path(path), body="\n\n".join(parts), processor="mistral-ocr")
def _selected_pages(self, total_pages: int) -> list[int]:
if total_pages < 1:
raise RuntimeError("El PDF no tiene paginas")
if not self.page_ranges or not self.page_ranges.strip():
return list(range(1, total_pages + 1))
selected: list[int] = []
for raw_part in self.page_ranges.split(","):
part = raw_part.strip()
if not part:
continue
if "-" in part:
start_text, end_text = [piece.strip() for piece in part.split("-", 1)]
if not start_text.isdigit() or not end_text.isdigit():
raise RuntimeError(f"Rango de paginas invalido: {part}")
start = int(start_text)
end = int(end_text)
if start > end:
raise RuntimeError(f"Rango de paginas invertido: {part}")
selected.extend(range(start, end + 1))
else:
if not part.isdigit():
raise RuntimeError(f"Pagina invalida: {part}")
selected.append(int(part))
unique_pages = sorted(set(selected))
invalid = [page for page in unique_pages if page < 1 or page > total_pages]
if invalid:
raise RuntimeError(f"Paginas fuera de rango: {', '.join(map(str, invalid))}. El PDF tiene {total_pages} paginas")
if not unique_pages:
raise RuntimeError("No se seleccionaron paginas para procesar")
return unique_pages
@@ -0,0 +1,15 @@
from __future__ import annotations
from pathlib import Path
from backend.app.services.markdown_builder import title_from_path
from backend.app.services.processors.base import ProcessedContent
class TextProcessor:
def process(self, path: Path) -> ProcessedContent:
return ProcessedContent(
title=title_from_path(path),
body=path.read_text(encoding="utf-8", errors="replace"),
processor="text",
)
@@ -0,0 +1,46 @@
from __future__ import annotations
import subprocess
from pathlib import Path
from backend.app.services.processors.audio_processor import AudioProcessor
from backend.app.services.processors.base import ProcessedContent
class VideoProcessor:
def __init__(self, tmp_dir: Path):
self.tmp_dir = tmp_dir
def process(self, path: Path) -> ProcessedContent:
self.tmp_dir.mkdir(parents=True, exist_ok=True)
audio_path = self.tmp_dir / f"{path.stem}.wav"
try:
subprocess.run(
[
"ffmpeg",
"-y",
"-i",
str(path),
"-vn",
"-acodec",
"pcm_s16le",
"-ar",
"16000",
"-ac",
"1",
str(audio_path),
],
check=True,
capture_output=True,
text=True,
)
processed = AudioProcessor().process(audio_path)
processed.title = f"Transcripcion {path.stem}"
processed.processor = "ffmpeg+deepgram"
return processed
except FileNotFoundError as exc:
raise RuntimeError("ffmpeg no esta instalado o no esta disponible en PATH") from exc
except subprocess.CalledProcessError as exc:
raise RuntimeError(f"No se pudo extraer audio del video: {exc.stderr}") from exc
finally:
audio_path.unlink(missing_ok=True)
@@ -0,0 +1,52 @@
from __future__ import annotations
from pathlib import Path
import requests
from backend.app.config import settings
class DeepgramClient:
endpoint = "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=es"
def transcribe_file(self, path: Path) -> dict:
if not settings.deepgram_api_key:
raise RuntimeError("DEEPGRAM_API_KEY no esta configurada")
headers = {
"Authorization": f"Token {settings.deepgram_api_key}",
"Content-Type": self._content_type(path),
}
with path.open("rb") as audio_file:
response = requests.post(self.endpoint, headers=headers, data=audio_file, timeout=600)
if response.status_code == 401:
raise RuntimeError("API Key de Deepgram invalida")
if not response.ok:
try:
error = response.json()
except ValueError:
error = response.text
raise RuntimeError(f"Deepgram error {response.status_code}: {error}")
return response.json()
def transcript_from_response(self, data: dict) -> tuple[str, float | None]:
alternative = data.get("results", {}).get("channels", [{}])[0].get("alternatives", [{}])[0]
transcript = alternative.get("paragraphs", {}).get("transcript") or alternative.get("transcript") or ""
confidence = alternative.get("confidence")
if not transcript.strip():
raise RuntimeError("Deepgram no devolvio transcripcion")
return transcript.strip(), confidence
def _content_type(self, path: Path) -> str:
ext = path.suffix.lower()
return {
".mp3": "audio/mpeg",
".wav": "audio/wav",
".m4a": "audio/mp4",
".mp4": "audio/mp4",
".webm": "audio/webm",
".ogg": "audio/ogg",
".flac": "audio/flac",
}.get(ext, "application/octet-stream")
@@ -0,0 +1,52 @@
from __future__ import annotations
import base64
from pathlib import Path
import requests
from backend.app.config import settings
class MistralClient:
endpoint = "https://api.mistral.ai/v1/chat/completions"
model = "pixtral-12b-2409"
def ocr_image(self, image_path: Path) -> str:
if not settings.mistral_api_key:
raise RuntimeError("MISTRAL_API_KEY no esta configurada")
data_url = self._image_data_url(image_path)
response = requests.post(
self.endpoint,
headers={
"Content-Type": "application/json",
"Authorization": f"Bearer {settings.mistral_api_key}",
},
json={
"model": self.model,
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "Extract all text from this image and format it as Markdown. If there are tables, represent them using Markdown table syntax. Do not include commentary.",
},
{"type": "image_url", "image_url": data_url},
],
}
],
"max_tokens": 3000,
},
timeout=180,
)
if response.status_code == 401:
raise RuntimeError("API Key de Mistral invalida")
if not response.ok:
raise RuntimeError(f"Mistral error {response.status_code}: {response.text}")
data = response.json()
return data["choices"][0]["message"]["content"]
def _image_data_url(self, image_path: Path) -> str:
encoded = base64.b64encode(image_path.read_bytes()).decode("ascii")
return f"data:image/jpeg;base64,{encoded}"
View File
+8
View File
@@ -0,0 +1,8 @@
import re
import unicodedata
def slugify(value: str) -> str:
normalized = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii")
slug = re.sub(r"[^a-zA-Z0-9]+", "-", normalized).strip("-").lower()
return slug or "sin-titulo"