OCR Mistral: modelo mistral-ocr-4-0, batch + fallback con reintentos y errores etiquetados
- Modelo por defecto mistral-ocr-4-0 (.env.example, config.py, docker-compose) - MistralClient: OCR en lote via API Batch (/v1/ocr) subiendo JSONL, polling y descarga de resultados - Reintentos con backoff exponencial en llamadas a Mistral (429/5xx/red); sin reintento en 401/400 - PdfProcessor: lote cuando >= umbral de paginas, fallback sincrono pagina a pagina, placeholder rastreable por pagina fallida - Errores etiquetados ([AUTH_MISTRAL], [BATCH], [NETWORK], [OCR_EMPTY], [PDF_PARSE]...) en el mensaje del job - Nuevas variables: MISTRAL_OCR_BATCH_THRESHOLD/POLL_SECONDS/TIMEOUT_SECONDS/RETRY_ATTEMPTS/RETRY_BACKOFF_BASE
This commit is contained in:
@@ -4,8 +4,10 @@ from pathlib import Path
|
||||
|
||||
import fitz
|
||||
|
||||
from backend.app.config import settings
|
||||
from backend.app.services.markdown_builder import title_from_path
|
||||
from backend.app.services.processors.base import ProcessedContent
|
||||
from backend.app.services.errors import ProcessingError
|
||||
from backend.app.services.providers.mistral_client import MistralClient
|
||||
|
||||
|
||||
@@ -30,22 +32,77 @@ class PdfProcessor:
|
||||
return ProcessedContent(title=title_from_path(path), body=body, processor="pdf-text")
|
||||
|
||||
def _process_ocr(self, path: Path) -> ProcessedContent:
|
||||
doc = fitz.open(path)
|
||||
client = MistralClient()
|
||||
parts: list[str] = []
|
||||
temp_paths: list[Path] = []
|
||||
try:
|
||||
for index in self._selected_pages(len(doc)):
|
||||
doc = fitz.open(path)
|
||||
except Exception as exc:
|
||||
raise ProcessingError("BAD_PDF", f"No se pudo abrir el PDF: {exc}") from exc
|
||||
selected = self._selected_pages(len(doc))
|
||||
client = MistralClient()
|
||||
page_images: dict[int, Path] = {}
|
||||
try:
|
||||
for index in selected:
|
||||
page = doc[index - 1]
|
||||
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
|
||||
image_path = path.with_name(f"{path.stem}-page-{index}.jpg")
|
||||
pix.save(image_path)
|
||||
temp_paths.append(image_path)
|
||||
parts.append(f"## Pagina {index}\n\n{client.ocr_image(image_path)}")
|
||||
page_images[index] = image_path
|
||||
|
||||
results = self._ocr_pages(client, page_images)
|
||||
finally:
|
||||
for temp_path in temp_paths:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
return ProcessedContent(title=title_from_path(path), body="\n\n".join(parts), processor="mistral-ocr")
|
||||
for image_path in page_images.values():
|
||||
image_path.unlink(missing_ok=True)
|
||||
|
||||
parts: list[str] = []
|
||||
for index in selected:
|
||||
markdown = results.get(index, "")
|
||||
parts.append(f"## Pagina {index}\n\n{markdown}")
|
||||
return ProcessedContent(
|
||||
title=title_from_path(path),
|
||||
body="\n\n".join(parts),
|
||||
processor="mistral-ocr",
|
||||
)
|
||||
|
||||
def _ocr_pages(self, client: MistralClient, page_images: dict[int, Path]) -> dict[int, str]:
|
||||
"""OCR de las paginas con lote + fallback sincrono.
|
||||
|
||||
Estrategia:
|
||||
1. Si hay >= ``batch_threshold`` paginas, intenta el job en lote.
|
||||
2. Cualquier pagina no resuelta (o si el lote entero falla) cae a
|
||||
OCR sincrono pagina a pagina con reintentos.
|
||||
3. Las paginas que fallen de forma definitiva quedan como un
|
||||
placeholder breve con el codigo de error rastreable.
|
||||
"""
|
||||
indices = list(page_images.keys())
|
||||
results: dict[int, str] = {}
|
||||
page_errors: dict[int, str] = {}
|
||||
|
||||
if len(indices) >= settings.mistral_ocr_batch_threshold:
|
||||
ordered = sorted(indices)
|
||||
paths = [page_images[i] for i in ordered]
|
||||
try:
|
||||
batch_results = client.ocr_images_batch(paths)
|
||||
for pos, index in enumerate(ordered):
|
||||
if pos in batch_results:
|
||||
results[index] = batch_results[pos]
|
||||
except ProcessingError:
|
||||
# El lote fallo; continua al fallback sincrono para todas
|
||||
# las paginas no resueltas.
|
||||
pass
|
||||
|
||||
for index in indices:
|
||||
if index in results:
|
||||
continue
|
||||
try:
|
||||
results[index] = client.ocr_image(page_images[index])
|
||||
except ProcessingError as exc:
|
||||
page_errors[index] = exc.labeled
|
||||
except Exception as exc:
|
||||
page_errors[index] = f"[OCR_ERROR] {exc}"
|
||||
|
||||
for index in indices:
|
||||
if index not in results:
|
||||
results[index] = f"> ⚠️ No se pudo procesar esta pagina: {page_errors.get(index, '[UNKNOWN]')}"
|
||||
return results
|
||||
|
||||
def _selected_pages(self, total_pages: int) -> list[int]:
|
||||
if total_pages < 1:
|
||||
|
||||
Reference in New Issue
Block a user