Files
urieljareth 239a7510c6 OCR Mistral: modelo mistral-ocr-4-0, batch + fallback con reintentos y errores etiquetados
- Modelo por defecto mistral-ocr-4-0 (.env.example, config.py, docker-compose)
- MistralClient: OCR en lote via API Batch (/v1/ocr) subiendo JSONL, polling y descarga de resultados
- Reintentos con backoff exponencial en llamadas a Mistral (429/5xx/red); sin reintento en 401/400
- PdfProcessor: lote cuando >= umbral de paginas, fallback sincrono pagina a pagina, placeholder rastreable por pagina fallida
- Errores etiquetados ([AUTH_MISTRAL], [BATCH], [NETWORK], [OCR_EMPTY], [PDF_PARSE]...) en el mensaje del job
- Nuevas variables: MISTRAL_OCR_BATCH_THRESHOLD/POLL_SECONDS/TIMEOUT_SECONDS/RETRY_ATTEMPTS/RETRY_BACKOFF_BASE
2026-07-26 19:58:38 -06:00

295 lines
11 KiB
Python

from __future__ import annotations
from pathlib import Path
from fastapi import BackgroundTasks, FastAPI, File, HTTPException, Query, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, PlainTextResponse
from backend.app.config import settings
from backend.app.managers.knowledge_manager import DuplicateSourceError, KnowledgeManager
from backend.app.models.schemas import MarkdownUpdate, SubjectCreate, SubjectUpdate, WeekCreate, WeekUpdate
from backend.app.services.errors import ProcessingError, classify_unknown_error
from backend.app.services.ingestion_service import IngestionService
app = FastAPI(title="Knowledge Station", version="0.1.0")
app.add_middleware(
CORSMiddleware,
allow_origins=["http://127.0.0.1:3000", "http://localhost:3000"],
allow_origin_regex=r"http://(127\.0\.0\.1|localhost):\d+",
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
manager = KnowledgeManager()
ingestion = IngestionService(manager)
@app.get("/health")
def health() -> dict[str, str]:
return {"status": "ok"}
@app.get("/settings/status")
def settings_status() -> dict:
return {
"mistral_configured": bool(settings.mistral_api_key),
"deepgram_configured": bool(settings.deepgram_api_key),
"data_dir": str(settings.app_data_dir),
"database_path": str(settings.database_path),
}
@app.post("/subjects")
def create_subject(payload: SubjectCreate) -> dict:
try:
return manager.create_subject(payload.name)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.get("/subjects")
def list_subjects() -> list[dict]:
return manager.list_subjects()
@app.get("/subjects/{subject_id}")
def get_subject(subject_id: int) -> dict:
try:
return manager.get_subject(subject_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.put("/subjects/{subject_id}")
def update_subject(subject_id: int, payload: SubjectUpdate) -> dict:
try:
return manager.update_subject(subject_id, payload.name)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.delete("/subjects/{subject_id}")
def delete_subject(subject_id: int) -> dict:
try:
manager.delete_subject(subject_id)
return {"status": "deleted"}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/subjects/{subject_id}/weeks")
def create_week(subject_id: int, payload: WeekCreate) -> dict:
try:
return manager.create_week(subject_id, payload.number, payload.title)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks")
def list_weeks(subject_id: int) -> list[dict]:
try:
manager.get_subject(subject_id)
return manager.list_weeks(subject_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}")
def get_week(subject_id: int, week_number: int) -> dict:
try:
return manager.get_week(subject_id, week_number)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.put("/subjects/{subject_id}/weeks/{week_number}")
def update_week(subject_id: int, week_number: int, payload: WeekUpdate) -> dict:
try:
return manager.update_week(subject_id, week_number, payload.number, payload.title)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.delete("/subjects/{subject_id}/weeks/{week_number}")
def delete_week(subject_id: int, week_number: int) -> dict:
try:
manager.delete_week(subject_id, week_number)
return {"status": "deleted"}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/subjects/{subject_id}/weeks/{week_number}/files")
def upload_file(
subject_id: int,
week_number: int,
background_tasks: BackgroundTasks,
file: UploadFile = File(...),
use_ocr: bool = Query(False, description="Usar Mistral OCR para PDFs"),
page_ranges: str | None = Query(None, description="Rangos de paginas PDF, por ejemplo: 1-5,8,10-12"),
) -> dict:
try:
source = manager.save_upload(subject_id, week_number, file)
job = manager.create_job(source["id"], "Archivo recibido")
background_tasks.add_task(process_job, job["id"], source["id"], use_ocr, page_ranges)
return {"source": source, "job": job}
except DuplicateSourceError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/jobs/{job_id}")
def get_job(job_id: int) -> dict:
try:
return manager.get_job(job_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/jobs/{job_id}/retry")
def retry_job(
job_id: int,
background_tasks: BackgroundTasks,
use_ocr: bool = Query(False),
page_ranges: str | None = Query(None),
) -> dict:
try:
job = manager.get_job(job_id)
if not job.get("source_id"):
raise HTTPException(status_code=400, detail="El trabajo no tiene fuente asociada")
manager.update_job(job_id, "queued", "Reintento en cola")
background_tasks.add_task(process_job, job_id, job["source_id"], use_ocr, page_ranges)
return manager.get_job(job_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/jobs")
def list_week_jobs(subject_id: int, week_number: int) -> list[dict]:
try:
return manager.list_jobs_for_week(subject_id, week_number)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/sources")
def list_sources(subject_id: int, week_number: int) -> list[dict]:
try:
return manager.list_sources(subject_id, week_number)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.delete("/sources/{source_id}")
def delete_source(source_id: int) -> dict:
try:
manager.delete_source(source_id)
return {"status": "deleted"}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post("/sources/{source_id}/reprocess")
def reprocess_source(
source_id: int,
background_tasks: BackgroundTasks,
use_ocr: bool = Query(False),
page_ranges: str | None = Query(None),
) -> dict:
try:
source = manager.get_source(source_id)
if manager.source_has_active_job(source_id):
raise HTTPException(status_code=409, detail="La fuente ya tiene un trabajo en cola o en proceso")
job = manager.create_job(source["id"], "Reprocesamiento en cola")
background_tasks.add_task(process_job, job["id"], source["id"], use_ocr, page_ranges)
return {"source": source, "job": job}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/documents")
def list_documents(subject_id: int, week_number: int) -> list[dict]:
try:
return manager.list_documents(subject_id, week_number)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/documents/{document_id}")
def get_document(document_id: int) -> dict:
try:
return manager.get_document(document_id)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/documents/{document_id}/markdown", response_class=PlainTextResponse)
def get_document_markdown(document_id: int) -> str:
try:
document = manager.get_document(document_id)
return Path(document["markdown_path"]).read_text(encoding="utf-8")
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.put("/documents/{document_id}/markdown")
def update_document_markdown(document_id: int, payload: MarkdownUpdate) -> dict:
try:
return manager.update_document_markdown(document_id, payload.content)
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/documents/{document_id}/download")
def download_document(document_id: int) -> FileResponse:
try:
document = manager.get_document(document_id)
path = Path(document["markdown_path"])
return FileResponse(path, filename=path.name, media_type="text/markdown")
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.delete("/documents/{document_id}")
def delete_document(document_id: int) -> dict:
try:
manager.delete_document(document_id)
return {"status": "deleted"}
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/export")
def export_week(subject_id: int, week_number: int) -> FileResponse:
try:
path = manager.export_week_markdown(subject_id, week_number)
return FileResponse(path, filename=path.name, media_type="text/markdown")
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/subjects/{subject_id}/weeks/{week_number}/export.zip")
def export_week_zip(subject_id: int, week_number: int) -> FileResponse:
try:
path = manager.export_week_markdown_zip(subject_id, week_number)
return FileResponse(path, filename=path.name, media_type="application/zip")
except ValueError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
def process_job(job_id: int, source_id: int, use_ocr: bool, page_ranges: str | None = None) -> None:
manager.update_job(job_id, "processing", "Procesando archivo")
manager.update_source_status(source_id, "processing")
try:
document = ingestion.process_source(source_id, use_ocr=use_ocr, page_ranges=page_ranges)
manager.update_job(job_id, "completed", "Documento procesado", document_id=document["id"])
except ProcessingError as exc:
manager.update_source_status(source_id, "failed")
manager.update_job(job_id, "failed", exc.labeled)
except Exception as exc:
manager.update_source_status(source_id, "failed")
code = classify_unknown_error(exc)
manager.update_job(job_id, "failed", f"[{code}] {exc}")