1

backend/app/services/ingestion_service.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from backend.app.managers.knowledge_manager import KnowledgeManager
+from backend.app.services.markdown_builder import build_markdown
+from backend.app.services.processors.audio_processor import AudioProcessor
+from backend.app.services.processors.docx_processor import DocxProcessor
+from backend.app.services.processors.pdf_processor import PdfProcessor
+from backend.app.services.processors.text_processor import TextProcessor
+from backend.app.services.processors.video_processor import VideoProcessor
+
+
+class IngestionService:
+    def __init__(self, manager: KnowledgeManager):
+        self.manager = manager
+
+    def process_source(self, source_id: int, use_ocr: bool = False, page_ranges: str | None = None) -> dict:
+        source = self.manager.get_source(source_id)
+        context = self.manager.get_week_context(source["week_id"])
+        path = Path(source["stored_path"])
+        processor = self._processor_for(source["source_type"], use_ocr, page_ranges)
+        processed = processor.process(path)
+        markdown = build_markdown(
+            title=processed.title,
+            body=processed.body,
+            metadata={
+                "subject": context["subject_name"],
+                "subject_slug": context["subject_slug"],
+                "week": context["week_number"],
+                "source_file": source["original_name"],
+                "source_type": source["source_type"],
+                "processor": processed.processor,
+                "page_ranges": page_ranges if source["source_type"] == "pdf" and page_ranges else None,
+                "language": "es",
+            },
+        )
+        return self.manager.create_document(source_id, processed.title, markdown, processor=processed.processor, page_ranges=page_ranges)
+
+    def _processor_for(self, source_type: str, use_ocr: bool, page_ranges: str | None):
+        if source_type == "text":
+            return TextProcessor()
+        if source_type == "docx":
+            return DocxProcessor()
+        if source_type == "pdf":
+            return PdfProcessor(use_ocr=use_ocr, page_ranges=page_ranges)
+        if source_type == "audio":
+            return AudioProcessor()
+        if source_type == "video":
+            return VideoProcessor(self.manager.tmp_dir)
+        raise RuntimeError(f"Tipo de fuente no soportado: {source_type}")

backend/app/services/markdown_builder.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def build_markdown(title: str, body: str, metadata: dict[str, Any]) -> str:
+    frontmatter = {
+        **metadata,
+        "processed_at": datetime.now(timezone.utc).isoformat(),
+    }
+    lines = ["---"]
+    for key, value in frontmatter.items():
+        if value is not None:
+            safe_value = str(value).replace("\n", " ")
+            lines.append(f"{key}: {safe_value}")
+    lines.extend(["---", "", f"# {title}", "", body.strip(), ""])
+    return "\n".join(lines)
+
+
+def title_from_path(path: Path) -> str:
+    return path.stem.replace("-", " ").replace("_", " ").strip().title() or "Documento"

backend/app/services/processors/audio_processor.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from backend.app.services.markdown_builder import title_from_path
+from backend.app.services.processors.base import ProcessedContent
+from backend.app.services.providers.deepgram_client import DeepgramClient
+
+
+class AudioProcessor:
+    def process(self, path: Path) -> ProcessedContent:
+        client = DeepgramClient()
+        data = client.transcribe_file(path)
+        transcript, confidence = client.transcript_from_response(data)
+        confidence_line = f"\n\n_Confianza estimada: {confidence * 100:.1f}%_" if isinstance(confidence, (int, float)) else ""
+        return ProcessedContent(
+            title=f"Transcripcion {title_from_path(path)}",
+            body=f"## Transcripcion\n\n{transcript}{confidence_line}",
+            processor="deepgram",
+        )

backend/app/services/processors/base.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class ProcessedContent:
+    title: str
+    body: str
+    processor: str

backend/app/services/processors/docx_processor.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from docx import Document
+
+from backend.app.services.markdown_builder import title_from_path
+from backend.app.services.processors.base import ProcessedContent
+
+
+class DocxProcessor:
+    def process(self, path: Path) -> ProcessedContent:
+        doc = Document(path)
+        blocks: list[str] = []
+        for paragraph in doc.paragraphs:
+            text = paragraph.text.strip()
+            if text:
+                blocks.append(text)
+        for table in doc.tables:
+            rows = [[cell.text.strip().replace("\n", " ") for cell in row.cells] for row in table.rows]
+            if rows:
+                blocks.append(self._table_to_markdown(rows))
+        return ProcessedContent(title=title_from_path(path), body="\n\n".join(blocks), processor="docx")
+
+    def _table_to_markdown(self, rows: list[list[str]]) -> str:
+        header = rows[0]
+        lines = [f"| {' | '.join(header)} |", f"| {' | '.join(['---'] * len(header))} |"]
+        for row in rows[1:]:
+            lines.append(f"| {' | '.join(row)} |")
+        return "\n".join(lines)

backend/app/services/processors/pdf_processor.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import fitz
+
+from backend.app.services.markdown_builder import title_from_path
+from backend.app.services.processors.base import ProcessedContent
+from backend.app.services.providers.mistral_client import MistralClient
+
+
+class PdfProcessor:
+    def __init__(self, use_ocr: bool = False, page_ranges: str | None = None):
+        self.use_ocr = use_ocr
+        self.page_ranges = page_ranges
+
+    def process(self, path: Path) -> ProcessedContent:
+        if self.use_ocr:
+            return self._process_ocr(path)
+        doc = fitz.open(path)
+        pages: list[str] = []
+        for index in self._selected_pages(len(doc)):
+            page = doc[index - 1]
+            text = page.get_text("text").strip()
+            if text:
+                pages.append(f"## Pagina {index}\n\n{text}")
+        body = "\n\n".join(pages).strip()
+        if len(body) < 80:
+            raise RuntimeError("El PDF tiene poco texto extraible; procesalo con OCR")
+        return ProcessedContent(title=title_from_path(path), body=body, processor="pdf-text")
+
+    def _process_ocr(self, path: Path) -> ProcessedContent:
+        doc = fitz.open(path)
+        client = MistralClient()
+        parts: list[str] = []
+        temp_paths: list[Path] = []
+        try:
+            for index in self._selected_pages(len(doc)):
+                page = doc[index - 1]
+                pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
+                image_path = path.with_name(f"{path.stem}-page-{index}.jpg")
+                pix.save(image_path)
+                temp_paths.append(image_path)
+                parts.append(f"## Pagina {index}\n\n{client.ocr_image(image_path)}")
+        finally:
+            for temp_path in temp_paths:
+                temp_path.unlink(missing_ok=True)
+        return ProcessedContent(title=title_from_path(path), body="\n\n".join(parts), processor="mistral-ocr")
+
+    def _selected_pages(self, total_pages: int) -> list[int]:
+        if total_pages < 1:
+            raise RuntimeError("El PDF no tiene paginas")
+        if not self.page_ranges or not self.page_ranges.strip():
+            return list(range(1, total_pages + 1))
+
+        selected: list[int] = []
+        for raw_part in self.page_ranges.split(","):
+            part = raw_part.strip()
+            if not part:
+                continue
+            if "-" in part:
+                start_text, end_text = [piece.strip() for piece in part.split("-", 1)]
+                if not start_text.isdigit() or not end_text.isdigit():
+                    raise RuntimeError(f"Rango de paginas invalido: {part}")
+                start = int(start_text)
+                end = int(end_text)
+                if start > end:
+                    raise RuntimeError(f"Rango de paginas invertido: {part}")
+                selected.extend(range(start, end + 1))
+            else:
+                if not part.isdigit():
+                    raise RuntimeError(f"Pagina invalida: {part}")
+                selected.append(int(part))
+
+        unique_pages = sorted(set(selected))
+        invalid = [page for page in unique_pages if page < 1 or page > total_pages]
+        if invalid:
+            raise RuntimeError(f"Paginas fuera de rango: {', '.join(map(str, invalid))}. El PDF tiene {total_pages} paginas")
+        if not unique_pages:
+            raise RuntimeError("No se seleccionaron paginas para procesar")
+        return unique_pages

backend/app/services/processors/text_processor.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from backend.app.services.markdown_builder import title_from_path
+from backend.app.services.processors.base import ProcessedContent
+
+
+class TextProcessor:
+    def process(self, path: Path) -> ProcessedContent:
+        return ProcessedContent(
+            title=title_from_path(path),
+            body=path.read_text(encoding="utf-8", errors="replace"),
+            processor="text",
+        )

backend/app/services/processors/video_processor.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from backend.app.services.processors.audio_processor import AudioProcessor
+from backend.app.services.processors.base import ProcessedContent
+
+
+class VideoProcessor:
+    def __init__(self, tmp_dir: Path):
+        self.tmp_dir = tmp_dir
+
+    def process(self, path: Path) -> ProcessedContent:
+        self.tmp_dir.mkdir(parents=True, exist_ok=True)
+        audio_path = self.tmp_dir / f"{path.stem}.wav"
+        try:
+            subprocess.run(
+                [
+                    "ffmpeg",
+                    "-y",
+                    "-i",
+                    str(path),
+                    "-vn",
+                    "-acodec",
+                    "pcm_s16le",
+                    "-ar",
+                    "16000",
+                    "-ac",
+                    "1",
+                    str(audio_path),
+                ],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+            processed = AudioProcessor().process(audio_path)
+            processed.title = f"Transcripcion {path.stem}"
+            processed.processor = "ffmpeg+deepgram"
+            return processed
+        except FileNotFoundError as exc:
+            raise RuntimeError("ffmpeg no esta instalado o no esta disponible en PATH") from exc
+        except subprocess.CalledProcessError as exc:
+            raise RuntimeError(f"No se pudo extraer audio del video: {exc.stderr}") from exc
+        finally:
+            audio_path.unlink(missing_ok=True)

backend/app/services/providers/deepgram_client.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import requests
+
+from backend.app.config import settings
+
+
+class DeepgramClient:
+    endpoint = "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=es"
+
+    def transcribe_file(self, path: Path) -> dict:
+        if not settings.deepgram_api_key:
+            raise RuntimeError("DEEPGRAM_API_KEY no esta configurada")
+
+        headers = {
+            "Authorization": f"Token {settings.deepgram_api_key}",
+            "Content-Type": self._content_type(path),
+        }
+        with path.open("rb") as audio_file:
+            response = requests.post(self.endpoint, headers=headers, data=audio_file, timeout=600)
+
+        if response.status_code == 401:
+            raise RuntimeError("API Key de Deepgram invalida")
+        if not response.ok:
+            try:
+                error = response.json()
+            except ValueError:
+                error = response.text
+            raise RuntimeError(f"Deepgram error {response.status_code}: {error}")
+        return response.json()
+
+    def transcript_from_response(self, data: dict) -> tuple[str, float | None]:
+        alternative = data.get("results", {}).get("channels", [{}])[0].get("alternatives", [{}])[0]
+        transcript = alternative.get("paragraphs", {}).get("transcript") or alternative.get("transcript") or ""
+        confidence = alternative.get("confidence")
+        if not transcript.strip():
+            raise RuntimeError("Deepgram no devolvio transcripcion")
+        return transcript.strip(), confidence
+
+    def _content_type(self, path: Path) -> str:
+        ext = path.suffix.lower()
+        return {
+            ".mp3": "audio/mpeg",
+            ".wav": "audio/wav",
+            ".m4a": "audio/mp4",
+            ".mp4": "audio/mp4",
+            ".webm": "audio/webm",
+            ".ogg": "audio/ogg",
+            ".flac": "audio/flac",
+        }.get(ext, "application/octet-stream")

backend/app/services/providers/mistral_client.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import base64
+from pathlib import Path
+
+import requests
+
+from backend.app.config import settings
+
+
+class MistralClient:
+    endpoint = "https://api.mistral.ai/v1/chat/completions"
+    model = "pixtral-12b-2409"
+
+    def ocr_image(self, image_path: Path) -> str:
+        if not settings.mistral_api_key:
+            raise RuntimeError("MISTRAL_API_KEY no esta configurada")
+        data_url = self._image_data_url(image_path)
+        response = requests.post(
+            self.endpoint,
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {settings.mistral_api_key}",
+            },
+            json={
+                "model": self.model,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "Extract all text from this image and format it as Markdown. If there are tables, represent them using Markdown table syntax. Do not include commentary.",
+                            },
+                            {"type": "image_url", "image_url": data_url},
+                        ],
+                    }
+                ],
+                "max_tokens": 3000,
+            },
+            timeout=180,
+        )
+        if response.status_code == 401:
+            raise RuntimeError("API Key de Mistral invalida")
+        if not response.ok:
+            raise RuntimeError(f"Mistral error {response.status_code}: {response.text}")
+        data = response.json()
+        return data["choices"][0]["message"]["content"]
+
+    def _image_data_url(self, image_path: Path) -> str:
+        encoded = base64.b64encode(image_path.read_bytes()).decode("ascii")
+        return f"data:image/jpeg;base64,{encoded}"