OCR Mistral: modelo mistral-ocr-4-0, batch + fallback con reintentos y errores etiquetados

- Modelo por defecto mistral-ocr-4-0 (.env.example, config.py, docker-compose)
- MistralClient: OCR en lote via API Batch (/v1/ocr) subiendo JSONL, polling y descarga de resultados
- Reintentos con backoff exponencial en llamadas a Mistral (429/5xx/red); sin reintento en 401/400
- PdfProcessor: lote cuando >= umbral de paginas, fallback sincrono pagina a pagina, placeholder rastreable por pagina fallida
- Errores etiquetados ([AUTH_MISTRAL], [BATCH], [NETWORK], [OCR_EMPTY], [PDF_PARSE]...) en el mensaje del job
- Nuevas variables: MISTRAL_OCR_BATCH_THRESHOLD/POLL_SECONDS/TIMEOUT_SECONDS/RETRY_ATTEMPTS/RETRY_BACKOFF_BASE

.env.example

@@ -2,5 +2,10 @@ APP_ENV=development
 APP_DATA_DIR=./data
 DATABASE_PATH=./data/app.db
 MISTRAL_API_KEY=
-MISTRAL_OCR_MODEL=mistral-ocr-latest
+MISTRAL_OCR_MODEL=mistral-ocr-4-0
+MISTRAL_OCR_BATCH_THRESHOLD=3
+MISTRAL_OCR_BATCH_POLL_SECONDS=10
+MISTRAL_OCR_BATCH_TIMEOUT_SECONDS=1800
+MISTRAL_OCR_RETRY_ATTEMPTS=3
+MISTRAL_OCR_RETRY_BACKOFF_BASE=2.0
 DEEPGRAM_API_KEY=

backend/app/config.py

@@ -8,7 +8,12 @@ class Settings(BaseSettings):
     app_data_dir: Path = Path("./data")
     database_path: Path = Path("./data/app.db")
     mistral_api_key: str = ""
-    mistral_ocr_model: str = "mistral-ocr-latest"
+    mistral_ocr_model: str = "mistral-ocr-4-0"
+    mistral_ocr_batch_threshold: int = 3
+    mistral_ocr_batch_poll_seconds: int = 10
+    mistral_ocr_batch_timeout_seconds: int = 1800
+    mistral_ocr_retry_attempts: int = 3
+    mistral_ocr_retry_backoff_base: float = 2.0
     deepgram_api_key: str = ""
 
     model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")

backend/app/main.py

@@ -9,6 +9,7 @@ from fastapi.responses import FileResponse, PlainTextResponse
 from backend.app.config import settings
 from backend.app.managers.knowledge_manager import DuplicateSourceError, KnowledgeManager
 from backend.app.models.schemas import MarkdownUpdate, SubjectCreate, SubjectUpdate, WeekCreate, WeekUpdate
+from backend.app.services.errors import ProcessingError, classify_unknown_error
 from backend.app.services.ingestion_service import IngestionService
 
 app = FastAPI(title="Knowledge Station", version="0.1.0")
@@ -284,6 +285,10 @@ def process_job(job_id: int, source_id: int, use_ocr: bool, page_ranges: str | N
     try:
         document = ingestion.process_source(source_id, use_ocr=use_ocr, page_ranges=page_ranges)
         manager.update_job(job_id, "completed", "Documento procesado", document_id=document["id"])
+    except ProcessingError as exc:
+        manager.update_source_status(source_id, "failed")
+        manager.update_job(job_id, "failed", exc.labeled)
     except Exception as exc:
         manager.update_source_status(source_id, "failed")
-        manager.update_job(job_id, "failed", str(exc))
+        code = classify_unknown_error(exc)
+        manager.update_job(job_id, "failed", f"[{code}] {exc}")

backend/app/services/errors.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import requests
+
+
+class ProcessingError(Exception):
+    """Error de procesamiento con un codigo corto y rastreable.
+
+    El atributo ``code`` es un identificador estable en MAYUSCULAS que se
+    antepone al mensaje del job (p.ej. ``[AUTH_MISTRAL] API Key invalida``)
+    para que el usuario pueda buscar y diagnosticar rapidamente el tipo de
+    fallo desde la UI.
+    """
+
+    def __init__(self, code: str, detail: str) -> None:
+        super().__init__(f"[{code}] {detail}")
+        self.code = code
+        self.detail = detail
+
+    @property
+    def labeled(self) -> str:
+        return f"[{self.code}] {self.detail}"
+
+
+class MistralAuthError(ProcessingError):
+    def __init__(self, detail: str = "API Key de Mistral invalida o no autorizada (401)") -> None:
+        super().__init__("AUTH_MISTRAL", detail)
+
+
+class MistralRateLimitError(ProcessingError):
+    def __init__(self, detail: str = "Limite de uso de Mistral alcanzado (429)") -> None:
+        super().__init__("RATE_LIMIT", detail)
+
+
+class MistralServerError(ProcessingError):
+    def __init__(self, status: int, body: str) -> None:
+        super().__init__("MISTRAL_SERVER", f"Mistral HTTP {status}: {body[:200]}")
+
+
+class MistralBatchError(ProcessingError):
+    def __init__(self, detail: str) -> None:
+        super().__init__("BATCH", detail)
+
+
+class MistralOCREmptyError(ProcessingError):
+    def __init__(self, detail: str = "OCR no devolvio contenido") -> None:
+        super().__init__("OCR_EMPTY", detail)
+
+
+class NetworkError(ProcessingError):
+    def __init__(self, detail: str) -> None:
+        super().__init__("NETWORK", detail)
+
+
+NON_RETRYABLE_CODES = {
+    "AUTH_MISTRAL",
+    "CONFIG",
+    "OCR_EMPTY",
+    "MISTRAL_CLIENT",
+    "BAD_PDF",
+    "FILE_IO",
+}
+
+
+def classify_requests_error(exc: requests.RequestException) -> ProcessingError:
+    if isinstance(exc, requests.Timeout):
+        return NetworkError(f"timeout de red con Mistral: {exc}")
+    if isinstance(exc, requests.ConnectionError):
+        return NetworkError(f"sin conexion a Mistral: {exc}")
+    return NetworkError(f"error de red con Mistral: {exc}")
+
+
+def classify_unknown_error(exc: Exception) -> str:
+    """Etiqueta breve para excepciones no envueltas en ProcessingError."""
+    name = type(exc).__name__.lower()
+    message = str(exc).lower()
+    if "fitz" in name or "pdf" in name or "pdf" in message or "muPDF" in message:
+        return "PDF_PARSE"
+    if isinstance(exc, FileNotFoundError):
+        return "FILE_IO"
+    if isinstance(exc, (PermissionError, OSError)):
+        return "FILE_IO"
+    if isinstance(exc, ValueError):
+        return "VALIDATION"
+    return "UNKNOWN"

backend/app/services/processors/pdf_processor.py

@@ -4,8 +4,10 @@ from pathlib import Path
 
 import fitz
 
+from backend.app.config import settings
 from backend.app.services.markdown_builder import title_from_path
 from backend.app.services.processors.base import ProcessedContent
+from backend.app.services.errors import ProcessingError
 from backend.app.services.providers.mistral_client import MistralClient
 
 
@@ -30,22 +32,77 @@ class PdfProcessor:
         return ProcessedContent(title=title_from_path(path), body=body, processor="pdf-text")
 
     def _process_ocr(self, path: Path) -> ProcessedContent:
-        doc = fitz.open(path)
-        client = MistralClient()
-        parts: list[str] = []
-        temp_paths: list[Path] = []
         try:
-            for index in self._selected_pages(len(doc)):
+            doc = fitz.open(path)
+        except Exception as exc:
+            raise ProcessingError("BAD_PDF", f"No se pudo abrir el PDF: {exc}") from exc
+        selected = self._selected_pages(len(doc))
+        client = MistralClient()
+        page_images: dict[int, Path] = {}
+        try:
+            for index in selected:
                 page = doc[index - 1]
                 pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
                 image_path = path.with_name(f"{path.stem}-page-{index}.jpg")
                 pix.save(image_path)
-                temp_paths.append(image_path)
+                page_images[index] = image_path
-                parts.append(f"## Pagina {index}\n\n{client.ocr_image(image_path)}")
+
+            results = self._ocr_pages(client, page_images)
         finally:
-            for temp_path in temp_paths:
+            for image_path in page_images.values():
-                temp_path.unlink(missing_ok=True)
+                image_path.unlink(missing_ok=True)
-        return ProcessedContent(title=title_from_path(path), body="\n\n".join(parts), processor="mistral-ocr")
+
+        parts: list[str] = []
+        for index in selected:
+            markdown = results.get(index, "")
+            parts.append(f"## Pagina {index}\n\n{markdown}")
+        return ProcessedContent(
+            title=title_from_path(path),
+            body="\n\n".join(parts),
+            processor="mistral-ocr",
+        )
+
+    def _ocr_pages(self, client: MistralClient, page_images: dict[int, Path]) -> dict[int, str]:
+        """OCR de las paginas con lote + fallback sincrono.
+
+        Estrategia:
+        1. Si hay >= ``batch_threshold`` paginas, intenta el job en lote.
+        2. Cualquier pagina no resuelta (o si el lote entero falla) cae a
+           OCR sincrono pagina a pagina con reintentos.
+        3. Las paginas que fallen de forma definitiva quedan como un
+           placeholder breve con el codigo de error rastreable.
+        """
+        indices = list(page_images.keys())
+        results: dict[int, str] = {}
+        page_errors: dict[int, str] = {}
+
+        if len(indices) >= settings.mistral_ocr_batch_threshold:
+            ordered = sorted(indices)
+            paths = [page_images[i] for i in ordered]
+            try:
+                batch_results = client.ocr_images_batch(paths)
+                for pos, index in enumerate(ordered):
+                    if pos in batch_results:
+                        results[index] = batch_results[pos]
+            except ProcessingError:
+                # El lote fallo; continua al fallback sincrono para todas
+                # las paginas no resueltas.
+                pass
+
+        for index in indices:
+            if index in results:
+                continue
+            try:
+                results[index] = client.ocr_image(page_images[index])
+            except ProcessingError as exc:
+                page_errors[index] = exc.labeled
+            except Exception as exc:
+                page_errors[index] = f"[OCR_ERROR] {exc}"
+
+        for index in indices:
+            if index not in results:
+                results[index] = f"> ⚠️ No se pudo procesar esta pagina: {page_errors.get(index, '[UNKNOWN]')}"
+        return results
 
     def _selected_pages(self, total_pages: int) -> list[int]:
         if total_pages < 1:

backend/app/services/providers/mistral_client.py

@@ -1,45 +1,287 @@
 from __future__ import annotations
 
 import base64
+import json
+import time
 from pathlib import Path
 
 import requests
 
 from backend.app.config import settings
+from backend.app.services.errors import (
+    MistralAuthError,
+    MistralBatchError,
+    MistralOCREmptyError,
+    MistralRateLimitError,
+    MistralServerError,
+    NetworkError,
+    NON_RETRYABLE_CODES,
+    ProcessingError,
+    classify_requests_error,
+)
 
 
 class MistralClient:
-    endpoint = "https://api.mistral.ai/v1/ocr"
+    """Cliente Mistral para OCR.
+
+    - ``ocr_image``: OCR sincrono de una pagina, con reintentos y backoff.
+    - ``ocr_images_batch``: OCR en lote via la API de Batch de Mistral
+      (endpoint ``/v1/ocr``), subiendo un fichero JSONL. Devuelve un dict
+      ``{page_index: markdown}``. Si el job falla o caduca se lanza
+      ``MistralBatchError`` para que el llamador caiga al modo sincrono.
+    """
+
+    base_url = "https://api.mistral.ai/v1"
+    ocr_url = f"{base_url}/ocr"
+    files_url = f"{base_url}/files"
+    batch_url = f"{base_url}/batch/jobs"
+
+    def __init__(self) -> None:
+        self._api_key = settings.mistral_api_key
+        self._model = settings.mistral_ocr_model
+
+    # ------------------------------------------------------------------
+    # API publica
+    # ------------------------------------------------------------------
 
     def ocr_image(self, image_path: Path) -> str:
-        if not settings.mistral_api_key:
+        self._require_key()
-            raise RuntimeError("MISTRAL_API_KEY no esta configurada")
         data_url = self._image_data_url(image_path)
-        response = requests.post(
+        return self._retry(lambda: self._ocr_sync(data_url))
-            self.endpoint,
+
-            headers={
+    def ocr_images_batch(self, image_paths: list[Path]) -> dict[int, str]:
-                "Content-Type": "application/json",
+        """OCR en lote. Devuelve ``{posicion_en_lista: markdown}``.
-                "Authorization": f"Bearer {settings.mistral_api_key}",
+
-            },
+        ``posicion_en_lista`` es el indice (0-based) dentro de ``image_paths``,
-            json={
+        NO el numero de pagina del PDF; el llamador hace esa traduccion.
-                "model": settings.mistral_ocr_model,
+        """
+        self._require_key()
+        if not image_paths:
+            return {}
+
+        payload = self._build_batch_payload(image_paths)
+        upload = self._retry(lambda: self._upload_batch_file(payload))
+        file_id = upload.get("id")
+        if not file_id:
+            raise MistralBatchError("la subida del fichero batch no devolvio id")
+
+        job = self._retry(lambda: self._create_batch_job(file_id))
+        job_id = job.get("id")
+        if not job_id:
+            raise MistralBatchError("la creacion del job batch no devolvio id")
+
+        final = self._poll_batch_job(job_id)
+        if final.get("status") != "SUCCESS":
+            reason = final.get("errors") or final.get("status") or "sin detalle"
+            raise MistralBatchError(f"job {job_id} finalizo como {final.get('status')}: {reason}")
+
+        output_file_id = final.get("output_file")
+        if not output_file_id:
+            raise MistralBatchError(f"job {job_id} sin output_file")
+
+        content = self._retry(lambda: self._download_file(output_file_id))
+        return self._parse_batch_output(content)
+
+    # ------------------------------------------------------------------
+    # OCR sincrono
+    # ------------------------------------------------------------------
+
+    def _ocr_sync(self, data_url: str) -> str:
+        try:
+            response = requests.post(
+                self.ocr_url,
+                headers=self._json_headers(),
+                json={
+                    "model": self._model,
+                    "document": {"type": "image_url", "image_url": data_url},
+                    "include_image_base64": False,
+                },
+                timeout=180,
+            )
+        except requests.RequestException as exc:
+            raise classify_requests_error(exc) from exc
+        self._raise_for_status(response)
+        markdown = self._extract_markdown(response.json())
+        if not markdown:
+            raise MistralOCREmptyError()
+        return markdown
+
+    # ------------------------------------------------------------------
+    # Batch: construccion, subida, job, polling, descarga, parseo
+    # ------------------------------------------------------------------
+
+    def _build_batch_payload(self, image_paths: list[Path]) -> bytes:
+        lines: list[str] = []
+        for idx, path in enumerate(image_paths):
+            data_url = self._image_data_url(path)
+            body = {
                 "document": {"type": "image_url", "image_url": data_url},
                 "include_image_base64": False,
-            },
+            }
-            timeout=180,
+            lines.append(
+                json.dumps(
+                    {"custom_id": str(idx), "body": body},
+                    separators=(",", ":"),
+                    ensure_ascii=False,
+                )
+            )
+        return ("\n".join(lines)).encode("utf-8")
+
+    def _upload_batch_file(self, payload: bytes) -> dict:
+        try:
+            response = requests.post(
+                self.files_url,
+                headers={"Authorization": f"Bearer {self._api_key}"},
+                data={"purpose": "batch"},
+                files={"file": ("ocr-batch.jsonl", payload, "application/jsonl")},
+                timeout=300,
+            )
+        except requests.RequestException as exc:
+            raise classify_requests_error(exc) from exc
+        self._raise_for_status(response)
+        return response.json()
+
+    def _create_batch_job(self, file_id: str) -> dict:
+        try:
+            response = requests.post(
+                self.batch_url,
+                headers=self._json_headers(),
+                json={
+                    "input_files": [file_id],
+                    "model": self._model,
+                    "endpoint": "/v1/ocr",
+                },
+                timeout=120,
+            )
+        except requests.RequestException as exc:
+            raise classify_requests_error(exc) from exc
+        self._raise_for_status(response)
+        return response.json()
+
+    def _poll_batch_job(self, job_id: str) -> dict:
+        terminal = {"SUCCESS", "FAILED", "TIMEOUT_EXCEEDED", "CANCELLED"}
+        deadline = time.time() + max(60, settings.mistral_ocr_batch_timeout_seconds)
+        delay = max(2, settings.mistral_ocr_batch_poll_seconds)
+        last: dict = {}
+        while time.time() < deadline:
+            try:
+                response = requests.get(
+                    f"{self.batch_url}/{job_id}",
+                    headers=self._json_headers(),
+                    timeout=60,
+                )
+            except requests.RequestException:
+                time.sleep(delay)
+                continue
+            if response.ok:
+                last = response.json()
+                if last.get("status") in terminal:
+                    return last
+            time.sleep(delay)
+        raise MistralBatchError(
+            f"job {job_id} excedio el timeout de polling "
+            f"({settings.mistral_ocr_batch_timeout_seconds}s); ultimo estado: "
+            f"{last.get('status') if last else 'desconocido'}"
         )
-        if response.status_code == 401:
+
-            raise RuntimeError("API Key de Mistral invalida")
+    def _download_file(self, file_id: str) -> bytes:
-        if not response.ok:
+        try:
-            raise RuntimeError(f"Mistral error {response.status_code}: {response.text}")
+            response = requests.get(
-        data = response.json()
+                f"{self.files_url}/{file_id}/content",
+                headers=self._json_headers(),
+                timeout=300,
+            )
+        except requests.RequestException as exc:
+            raise classify_requests_error(exc) from exc
+        self._raise_for_status(response)
+        return response.content
+
+    def _parse_batch_output(self, content: bytes) -> dict[int, str]:
+        results: dict[int, str] = {}
+        for raw in content.decode("utf-8", errors="replace").splitlines():
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            custom_id = obj.get("custom_id")
+            if custom_id is None:
+                continue
+            try:
+                idx = int(custom_id)
+            except (TypeError, ValueError):
+                continue
+            response = obj.get("response") or {}
+            status_code = response.get("status_code", 200)
+            body = response.get("body") or {}
+            if status_code >= 400 or obj.get("error"):
+                # Pagina fallida en lote: se omite y el llamador cae a sync.
+                continue
+            markdown = self._extract_markdown(body)
+            if markdown:
+                results[idx] = markdown
+        return results
+
+    # ------------------------------------------------------------------
+    # Helpers compartidos
+    # ------------------------------------------------------------------
+
+    def _require_key(self) -> None:
+        if not self._api_key:
+            raise ProcessingError("CONFIG", "MISTRAL_API_KEY no esta configurada")
+
+    def _json_headers(self) -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+        }
+
+    def _raise_for_status(self, response: requests.Response) -> None:
+        if response.ok:
+            return
+        try:
+            body = response.text
+        except Exception:
+            body = ""
+        status = response.status_code
+        if status == 401:
+            raise MistralAuthError()
+        if status == 429:
+            raise MistralRateLimitError()
+        if status >= 500:
+            raise MistralServerError(status, body)
+        raise ProcessingError("MISTRAL_CLIENT", f"HTTP {status}: {body[:200]}")
+
+    def _retry(self, fn):
+        attempts = max(1, settings.mistral_ocr_retry_attempts)
+        base_delay = max(0.5, settings.mistral_ocr_retry_backoff_base)
+        last_exc: Exception | None = None
+        for attempt in range(attempts):
+            try:
+                return fn()
+            except ProcessingError as exc:
+                if exc.code in NON_RETRYABLE_CODES:
+                    raise
+                last_exc = exc
+            except Exception as exc:
+                last_exc = exc
+            if attempt < attempts - 1:
+                time.sleep(base_delay * (2 ** attempt))
+        if last_exc is not None:
+            raise last_exc
+        raise MistralBatchError("reintentos agotados sin excepcion conocida")
+
+    def _extract_markdown(self, data: dict) -> str:
         pages = data.get("pages") or []
-        markdown_pages = [page.get("markdown", "").strip() for page in pages if page.get("markdown")]
+        chunks = [page.get("markdown", "").strip() for page in pages if page.get("markdown")]
-        if markdown_pages:
+        if chunks:
-            return "\n\n".join(markdown_pages)
+            return "\n\n".join(chunks)
         if data.get("markdown"):
-            return data["markdown"].strip()
+            return str(data["markdown"]).strip()
-        raise RuntimeError("Mistral OCR no devolvio Markdown")
+        return ""
 
     def _image_data_url(self, image_path: Path) -> str:
         encoded = base64.b64encode(image_path.read_bytes()).decode("ascii")

docker-compose.yml

@@ -11,7 +11,12 @@ services:
       APP_DATA_DIR: /app/data
       DATABASE_PATH: /app/data/app.db
       MISTRAL_API_KEY: ${MISTRAL_API_KEY:-}
-      MISTRAL_OCR_MODEL: ${MISTRAL_OCR_MODEL:-mistral-ocr-latest}
+      MISTRAL_OCR_MODEL: ${MISTRAL_OCR_MODEL:-mistral-ocr-4-0}
+      MISTRAL_OCR_BATCH_THRESHOLD: ${MISTRAL_OCR_BATCH_THRESHOLD:-3}
+      MISTRAL_OCR_BATCH_POLL_SECONDS: ${MISTRAL_OCR_BATCH_POLL_SECONDS:-10}
+      MISTRAL_OCR_BATCH_TIMEOUT_SECONDS: ${MISTRAL_OCR_BATCH_TIMEOUT_SECONDS:-1800}
+      MISTRAL_OCR_RETRY_ATTEMPTS: ${MISTRAL_OCR_RETRY_ATTEMPTS:-3}
+      MISTRAL_OCR_RETRY_BACKOFF_BASE: ${MISTRAL_OCR_RETRY_BACKOFF_BASE:-2.0}
       DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-}
       NODE_ENV: production
       PORT: 3000