1
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
app_env: str = "development"
|
||||
app_data_dir: Path = Path("./data")
|
||||
database_path: Path = Path("./data/app.db")
|
||||
mistral_api_key: str = ""
|
||||
deepgram_api_key: str = ""
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,281 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import BackgroundTasks, FastAPI, File, HTTPException, Query, UploadFile
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, PlainTextResponse
|
||||
|
||||
from backend.app.config import settings
|
||||
from backend.app.managers.knowledge_manager import DuplicateSourceError, KnowledgeManager
|
||||
from backend.app.models.schemas import MarkdownUpdate, SubjectCreate, WeekCreate, WeekUpdate
|
||||
from backend.app.services.ingestion_service import IngestionService
|
||||
|
||||
app = FastAPI(title="Knowledge Station", version="0.1.0")
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["http://127.0.0.1:3000", "http://localhost:3000"],
|
||||
allow_origin_regex=r"http://(127\.0\.0\.1|localhost):\d+",
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
manager = KnowledgeManager()
|
||||
ingestion = IngestionService(manager)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/settings/status")
|
||||
def settings_status() -> dict:
|
||||
return {
|
||||
"mistral_configured": bool(settings.mistral_api_key),
|
||||
"deepgram_configured": bool(settings.deepgram_api_key),
|
||||
"data_dir": str(settings.app_data_dir),
|
||||
"database_path": str(settings.database_path),
|
||||
}
|
||||
|
||||
|
||||
@app.post("/subjects")
|
||||
def create_subject(payload: SubjectCreate) -> dict:
|
||||
try:
|
||||
return manager.create_subject(payload.name)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/subjects")
|
||||
def list_subjects() -> list[dict]:
|
||||
return manager.list_subjects()
|
||||
|
||||
|
||||
@app.get("/subjects/{subject_id}")
|
||||
def get_subject(subject_id: int) -> dict:
|
||||
try:
|
||||
return manager.get_subject(subject_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.delete("/subjects/{subject_id}")
|
||||
def delete_subject(subject_id: int) -> dict:
|
||||
try:
|
||||
manager.delete_subject(subject_id)
|
||||
return {"status": "deleted"}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.post("/subjects/{subject_id}/weeks")
|
||||
def create_week(subject_id: int, payload: WeekCreate) -> dict:
|
||||
try:
|
||||
return manager.create_week(subject_id, payload.number, payload.title)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/subjects/{subject_id}/weeks")
|
||||
def list_weeks(subject_id: int) -> list[dict]:
|
||||
try:
|
||||
manager.get_subject(subject_id)
|
||||
return manager.list_weeks(subject_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/subjects/{subject_id}/weeks/{week_number}")
|
||||
def get_week(subject_id: int, week_number: int) -> dict:
|
||||
try:
|
||||
return manager.get_week(subject_id, week_number)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.put("/subjects/{subject_id}/weeks/{week_number}")
|
||||
def update_week(subject_id: int, week_number: int, payload: WeekUpdate) -> dict:
|
||||
try:
|
||||
return manager.update_week(subject_id, week_number, payload.number, payload.title)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.delete("/subjects/{subject_id}/weeks/{week_number}")
|
||||
def delete_week(subject_id: int, week_number: int) -> dict:
|
||||
try:
|
||||
manager.delete_week(subject_id, week_number)
|
||||
return {"status": "deleted"}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.post("/subjects/{subject_id}/weeks/{week_number}/files")
|
||||
def upload_file(
|
||||
subject_id: int,
|
||||
week_number: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
file: UploadFile = File(...),
|
||||
use_ocr: bool = Query(False, description="Usar Mistral OCR para PDFs"),
|
||||
page_ranges: str | None = Query(None, description="Rangos de paginas PDF, por ejemplo: 1-5,8,10-12"),
|
||||
) -> dict:
|
||||
try:
|
||||
source = manager.save_upload(subject_id, week_number, file)
|
||||
job = manager.create_job(source["id"], "Archivo recibido")
|
||||
background_tasks.add_task(process_job, job["id"], source["id"], use_ocr, page_ranges)
|
||||
return {"source": source, "job": job}
|
||||
except DuplicateSourceError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/jobs/{job_id}")
|
||||
def get_job(job_id: int) -> dict:
|
||||
try:
|
||||
return manager.get_job(job_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.post("/jobs/{job_id}/retry")
|
||||
def retry_job(
|
||||
job_id: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
use_ocr: bool = Query(False),
|
||||
page_ranges: str | None = Query(None),
|
||||
) -> dict:
|
||||
try:
|
||||
job = manager.get_job(job_id)
|
||||
if not job.get("source_id"):
|
||||
raise HTTPException(status_code=400, detail="El trabajo no tiene fuente asociada")
|
||||
manager.update_job(job_id, "queued", "Reintento en cola")
|
||||
background_tasks.add_task(process_job, job_id, job["source_id"], use_ocr, page_ranges)
|
||||
return manager.get_job(job_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/subjects/{subject_id}/weeks/{week_number}/jobs")
|
||||
def list_week_jobs(subject_id: int, week_number: int) -> list[dict]:
|
||||
try:
|
||||
return manager.list_jobs_for_week(subject_id, week_number)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/subjects/{subject_id}/weeks/{week_number}/sources")
|
||||
def list_sources(subject_id: int, week_number: int) -> list[dict]:
|
||||
try:
|
||||
return manager.list_sources(subject_id, week_number)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.delete("/sources/{source_id}")
|
||||
def delete_source(source_id: int) -> dict:
|
||||
try:
|
||||
manager.delete_source(source_id)
|
||||
return {"status": "deleted"}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.post("/sources/{source_id}/reprocess")
|
||||
def reprocess_source(
|
||||
source_id: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
use_ocr: bool = Query(False),
|
||||
page_ranges: str | None = Query(None),
|
||||
) -> dict:
|
||||
try:
|
||||
source = manager.get_source(source_id)
|
||||
if manager.source_has_active_job(source_id):
|
||||
raise HTTPException(status_code=409, detail="La fuente ya tiene un trabajo en cola o en proceso")
|
||||
job = manager.create_job(source["id"], "Reprocesamiento en cola")
|
||||
background_tasks.add_task(process_job, job["id"], source["id"], use_ocr, page_ranges)
|
||||
return {"source": source, "job": job}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/subjects/{subject_id}/weeks/{week_number}/documents")
|
||||
def list_documents(subject_id: int, week_number: int) -> list[dict]:
|
||||
try:
|
||||
return manager.list_documents(subject_id, week_number)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/documents/{document_id}")
|
||||
def get_document(document_id: int) -> dict:
|
||||
try:
|
||||
return manager.get_document(document_id)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/documents/{document_id}/markdown", response_class=PlainTextResponse)
|
||||
def get_document_markdown(document_id: int) -> str:
|
||||
try:
|
||||
document = manager.get_document(document_id)
|
||||
return Path(document["markdown_path"]).read_text(encoding="utf-8")
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.put("/documents/{document_id}/markdown")
|
||||
def update_document_markdown(document_id: int, payload: MarkdownUpdate) -> dict:
|
||||
try:
|
||||
return manager.update_document_markdown(document_id, payload.content)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/documents/{document_id}/download")
|
||||
def download_document(document_id: int) -> FileResponse:
|
||||
try:
|
||||
document = manager.get_document(document_id)
|
||||
path = Path(document["markdown_path"])
|
||||
return FileResponse(path, filename=path.name, media_type="text/markdown")
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.delete("/documents/{document_id}")
|
||||
def delete_document(document_id: int) -> dict:
|
||||
try:
|
||||
manager.delete_document(document_id)
|
||||
return {"status": "deleted"}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/subjects/{subject_id}/weeks/{week_number}/export")
|
||||
def export_week(subject_id: int, week_number: int) -> FileResponse:
|
||||
try:
|
||||
path = manager.export_week_markdown(subject_id, week_number)
|
||||
return FileResponse(path, filename=path.name, media_type="text/markdown")
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@app.get("/subjects/{subject_id}/weeks/{week_number}/export.zip")
|
||||
def export_week_zip(subject_id: int, week_number: int) -> FileResponse:
|
||||
try:
|
||||
path = manager.export_week_markdown_zip(subject_id, week_number)
|
||||
return FileResponse(path, filename=path.name, media_type="application/zip")
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
|
||||
|
||||
def process_job(job_id: int, source_id: int, use_ocr: bool, page_ranges: str | None = None) -> None:
|
||||
manager.update_job(job_id, "processing", "Procesando archivo")
|
||||
manager.update_source_status(source_id, "processing")
|
||||
try:
|
||||
document = ingestion.process_source(source_id, use_ocr=use_ocr, page_ranges=page_ranges)
|
||||
manager.update_job(job_id, "completed", "Documento procesado", document_id=document["id"])
|
||||
except Exception as exc:
|
||||
manager.update_source_status(source_id, "failed")
|
||||
manager.update_job(job_id, "failed", str(exc))
|
||||
@@ -0,0 +1,548 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
import re
|
||||
import hashlib
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from fastapi import UploadFile
|
||||
|
||||
from backend.app.config import settings
|
||||
from backend.app.utils.slug import slugify
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class DuplicateSourceError(ValueError):
|
||||
def __init__(self, existing_source: dict[str, Any]):
|
||||
self.existing_source = existing_source
|
||||
super().__init__(f"El archivo ya existe en esta semana: {existing_source['original_name']}")
|
||||
|
||||
|
||||
class KnowledgeManager:
|
||||
def __init__(self, data_dir: Path | None = None, database_path: Path | None = None):
|
||||
self.data_dir = data_dir or settings.app_data_dir
|
||||
self.database_path = database_path or settings.database_path
|
||||
self.subjects_dir = self.data_dir / "subjects"
|
||||
self.tmp_dir = self.data_dir / "tmp"
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.subjects_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.database_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._init_db()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(self.database_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def _init_db(self) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.executescript(
|
||||
"""
|
||||
create table if not exists subjects (
|
||||
id integer primary key autoincrement,
|
||||
name text not null,
|
||||
slug text not null unique,
|
||||
created_at text not null
|
||||
);
|
||||
|
||||
create table if not exists weeks (
|
||||
id integer primary key autoincrement,
|
||||
subject_id integer not null,
|
||||
number integer not null,
|
||||
title text,
|
||||
created_at text not null,
|
||||
unique(subject_id, number),
|
||||
foreign key(subject_id) references subjects(id)
|
||||
);
|
||||
|
||||
create table if not exists sources (
|
||||
id integer primary key autoincrement,
|
||||
week_id integer not null,
|
||||
original_name text not null,
|
||||
stored_path text not null,
|
||||
source_type text not null,
|
||||
status text not null,
|
||||
file_size integer,
|
||||
sha256 text,
|
||||
mime_type text,
|
||||
processor text,
|
||||
page_ranges text,
|
||||
processed_at text,
|
||||
created_at text not null,
|
||||
foreign key(week_id) references weeks(id)
|
||||
);
|
||||
|
||||
create table if not exists documents (
|
||||
id integer primary key autoincrement,
|
||||
source_id integer not null,
|
||||
title text not null,
|
||||
markdown_path text not null,
|
||||
created_at text not null,
|
||||
foreign key(source_id) references sources(id)
|
||||
);
|
||||
|
||||
create table if not exists jobs (
|
||||
id integer primary key autoincrement,
|
||||
status text not null,
|
||||
message text not null,
|
||||
source_id integer,
|
||||
document_id integer,
|
||||
created_at text not null,
|
||||
updated_at text not null,
|
||||
foreign key(source_id) references sources(id),
|
||||
foreign key(document_id) references documents(id)
|
||||
);
|
||||
"""
|
||||
)
|
||||
self._ensure_columns(
|
||||
conn,
|
||||
"sources",
|
||||
{
|
||||
"file_size": "integer",
|
||||
"sha256": "text",
|
||||
"mime_type": "text",
|
||||
"processor": "text",
|
||||
"page_ranges": "text",
|
||||
"processed_at": "text",
|
||||
},
|
||||
)
|
||||
self._backfill_source_file_metadata(conn)
|
||||
|
||||
def _ensure_columns(self, conn: sqlite3.Connection, table: str, columns: dict[str, str]) -> None:
|
||||
existing = {row["name"] for row in conn.execute(f"pragma table_info({table})").fetchall()}
|
||||
for name, definition in columns.items():
|
||||
if name not in existing:
|
||||
conn.execute(f"alter table {table} add column {name} {definition}")
|
||||
|
||||
def _backfill_source_file_metadata(self, conn: sqlite3.Connection) -> None:
|
||||
rows = conn.execute("select id, stored_path from sources where sha256 is null or file_size is null").fetchall()
|
||||
for row in rows:
|
||||
path = Path(row["stored_path"])
|
||||
if not path.exists():
|
||||
continue
|
||||
sha256, file_size = self._file_hash_and_size(path)
|
||||
conn.execute("update sources set sha256 = ?, file_size = ? where id = ?", (sha256, file_size, row["id"]))
|
||||
|
||||
def create_subject(self, name: str) -> dict[str, Any]:
|
||||
base_slug = slugify(name)
|
||||
slug = base_slug
|
||||
suffix = 2
|
||||
with self._connect() as conn:
|
||||
while conn.execute("select 1 from subjects where slug = ?", (slug,)).fetchone():
|
||||
slug = f"{base_slug}-{suffix}"
|
||||
suffix += 1
|
||||
now = utc_now()
|
||||
cursor = conn.execute(
|
||||
"insert into subjects (name, slug, created_at) values (?, ?, ?)",
|
||||
(name, slug, now),
|
||||
)
|
||||
(self.subjects_dir / slug).mkdir(parents=True, exist_ok=True)
|
||||
return self.get_subject(cursor.lastrowid)
|
||||
|
||||
def get_subject(self, subject_id: int) -> dict[str, Any]:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("select * from subjects where id = ?", (subject_id,)).fetchone()
|
||||
if not row:
|
||||
raise ValueError("Materia no encontrada")
|
||||
return dict(row)
|
||||
|
||||
def delete_subject(self, subject_id: int) -> None:
|
||||
subject = self.get_subject(subject_id)
|
||||
with self._connect() as conn:
|
||||
week_rows = conn.execute("select id from weeks where subject_id = ?", (subject_id,)).fetchall()
|
||||
week_ids = [row["id"] for row in week_rows]
|
||||
for week_id in week_ids:
|
||||
source_rows = conn.execute("select id from sources where week_id = ?", (week_id,)).fetchall()
|
||||
source_ids = [row["id"] for row in source_rows]
|
||||
for source_id in source_ids:
|
||||
conn.execute("delete from documents where source_id = ?", (source_id,))
|
||||
conn.execute("delete from jobs where source_id = ?", (source_id,))
|
||||
conn.execute("delete from sources where week_id = ?", (week_id,))
|
||||
conn.execute("delete from weeks where subject_id = ?", (subject_id,))
|
||||
conn.execute("delete from subjects where id = ?", (subject_id,))
|
||||
shutil.rmtree(self.subjects_dir / subject["slug"], ignore_errors=True)
|
||||
|
||||
def list_subjects(self) -> list[dict[str, Any]]:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("select * from subjects order by created_at desc").fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def create_week(self, subject_id: int, number: int, title: str | None = None) -> dict[str, Any]:
|
||||
subject = self.get_subject(subject_id)
|
||||
now = utc_now()
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"insert or ignore into weeks (subject_id, number, title, created_at) values (?, ?, ?, ?)",
|
||||
(subject_id, number, title, now),
|
||||
)
|
||||
if title:
|
||||
conn.execute("update weeks set title = ? where subject_id = ? and number = ?", (title, subject_id, number))
|
||||
row = conn.execute(
|
||||
"select * from weeks where subject_id = ? and number = ?",
|
||||
(subject_id, number),
|
||||
).fetchone()
|
||||
self.week_dir(subject["slug"], number).mkdir(parents=True, exist_ok=True)
|
||||
for child in ("raw", "processed", "exports"):
|
||||
(self.week_dir(subject["slug"], number) / child).mkdir(parents=True, exist_ok=True)
|
||||
return dict(row)
|
||||
|
||||
def list_weeks(self, subject_id: int) -> list[dict[str, Any]]:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("select * from weeks where subject_id = ? order by number", (subject_id,)).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def delete_week(self, subject_id: int, number: int) -> None:
|
||||
subject = self.get_subject(subject_id)
|
||||
week = self.get_week(subject_id, number)
|
||||
with self._connect() as conn:
|
||||
source_rows = conn.execute("select id from sources where week_id = ?", (week["id"],)).fetchall()
|
||||
for row in source_rows:
|
||||
conn.execute("delete from documents where source_id = ?", (row["id"],))
|
||||
conn.execute("delete from jobs where source_id = ?", (row["id"],))
|
||||
conn.execute("delete from sources where week_id = ?", (week["id"],))
|
||||
conn.execute("delete from weeks where id = ?", (week["id"],))
|
||||
shutil.rmtree(self.week_dir(subject["slug"], number), ignore_errors=True)
|
||||
|
||||
def get_week(self, subject_id: int, number: int) -> dict[str, Any]:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"select * from weeks where subject_id = ? and number = ?",
|
||||
(subject_id, number),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise ValueError("Semana no encontrada")
|
||||
return dict(row)
|
||||
|
||||
def update_week(self, subject_id: int, current_number: int, new_number: int, title: str | None = None) -> dict[str, Any]:
|
||||
subject = self.get_subject(subject_id)
|
||||
week = self.get_week(subject_id, current_number)
|
||||
if new_number != current_number:
|
||||
with self._connect() as conn:
|
||||
existing = conn.execute(
|
||||
"select 1 from weeks where subject_id = ? and number = ?",
|
||||
(subject_id, new_number),
|
||||
).fetchone()
|
||||
if existing:
|
||||
raise ValueError("Ya existe una semana con ese numero")
|
||||
|
||||
old_dir = self.week_dir(subject["slug"], current_number)
|
||||
new_dir = self.week_dir(subject["slug"], new_number)
|
||||
if old_dir.exists() and new_dir.exists():
|
||||
raise ValueError("Ya existe una carpeta para ese numero de semana")
|
||||
if old_dir.exists():
|
||||
old_dir.rename(new_dir)
|
||||
else:
|
||||
for child in ("raw", "processed", "exports"):
|
||||
(new_dir / child).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"update weeks set number = ?, title = ? where id = ?",
|
||||
(new_number, title, week["id"]),
|
||||
)
|
||||
return self.get_week(subject_id, new_number)
|
||||
|
||||
def week_dir(self, subject_slug: str, week_number: int) -> Path:
|
||||
return self.subjects_dir / subject_slug / f"week-{week_number:02d}"
|
||||
|
||||
def save_upload(self, subject_id: int, week_number: int, upload: UploadFile) -> dict[str, Any]:
|
||||
subject = self.get_subject(subject_id)
|
||||
week = self.get_week(subject_id, week_number)
|
||||
filename = Path(upload.filename or "archivo").name
|
||||
raw_dir = self.week_dir(subject["slug"], week_number) / "raw"
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
stored_path = self._unique_path(raw_dir / filename)
|
||||
digest = hashlib.sha256()
|
||||
file_size = 0
|
||||
with stored_path.open("wb") as out_file:
|
||||
while chunk := upload.file.read(1024 * 1024):
|
||||
file_size += len(chunk)
|
||||
digest.update(chunk)
|
||||
out_file.write(chunk)
|
||||
sha256 = digest.hexdigest()
|
||||
source_type = self.detect_source_type(stored_path)
|
||||
now = utc_now()
|
||||
with self._connect() as conn:
|
||||
duplicate = conn.execute(
|
||||
"select * from sources where week_id = ? and sha256 = ? limit 1",
|
||||
(week["id"], sha256),
|
||||
).fetchone()
|
||||
if duplicate:
|
||||
stored_path.unlink(missing_ok=True)
|
||||
raise DuplicateSourceError(dict(duplicate))
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
insert into sources (week_id, original_name, stored_path, source_type, status, file_size, sha256, mime_type, created_at)
|
||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(week["id"], filename, str(stored_path), source_type, "stored", file_size, sha256, upload.content_type, now),
|
||||
)
|
||||
return self.get_source(cursor.lastrowid)
|
||||
|
||||
def _file_hash_and_size(self, path: Path) -> tuple[str, int]:
|
||||
digest = hashlib.sha256()
|
||||
file_size = 0
|
||||
with path.open("rb") as file:
|
||||
while chunk := file.read(1024 * 1024):
|
||||
file_size += len(chunk)
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest(), file_size
|
||||
|
||||
def create_job(self, source_id: int, message: str = "En cola") -> dict[str, Any]:
|
||||
now = utc_now()
|
||||
with self._connect() as conn:
|
||||
cursor = conn.execute(
|
||||
"insert into jobs (status, message, source_id, created_at, updated_at) values (?, ?, ?, ?, ?)",
|
||||
("queued", message, source_id, now, now),
|
||||
)
|
||||
return self.get_job(cursor.lastrowid)
|
||||
|
||||
def update_job(self, job_id: int, status: str, message: str, document_id: int | None = None) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"update jobs set status = ?, message = ?, document_id = coalesce(?, document_id), updated_at = ? where id = ?",
|
||||
(status, message, document_id, utc_now(), job_id),
|
||||
)
|
||||
|
||||
def update_source_status(self, source_id: int, status: str) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute("update sources set status = ? where id = ?", (status, source_id))
|
||||
|
||||
def source_has_active_job(self, source_id: int) -> bool:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"select 1 from jobs where source_id = ? and status in ('queued', 'processing') limit 1",
|
||||
(source_id,),
|
||||
).fetchone()
|
||||
return bool(row)
|
||||
|
||||
def get_job(self, job_id: int) -> dict[str, Any]:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("select * from jobs where id = ?", (job_id,)).fetchone()
|
||||
if not row:
|
||||
raise ValueError("Trabajo no encontrado")
|
||||
return dict(row)
|
||||
|
||||
def list_jobs_for_week(self, subject_id: int, week_number: int) -> list[dict[str, Any]]:
|
||||
week = self.get_week(subject_id, week_number)
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
select jobs.* from jobs
|
||||
join sources on sources.id = jobs.source_id
|
||||
where sources.week_id = ?
|
||||
order by jobs.created_at desc
|
||||
""",
|
||||
(week["id"],),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def get_source(self, source_id: int) -> dict[str, Any]:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("select * from sources where id = ?", (source_id,)).fetchone()
|
||||
if not row:
|
||||
raise ValueError("Fuente no encontrada")
|
||||
return dict(row)
|
||||
|
||||
def list_sources(self, subject_id: int, week_number: int) -> list[dict[str, Any]]:
|
||||
week = self.get_week(subject_id, week_number)
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
select sources.*, documents.id as document_id, documents.title as document_title,
|
||||
documents.markdown_path as document_markdown_path, documents.created_at as document_created_at
|
||||
from sources
|
||||
left join documents on documents.source_id = sources.id
|
||||
where sources.week_id = ?
|
||||
order by sources.created_at desc
|
||||
""",
|
||||
(week["id"],),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def delete_source(self, source_id: int) -> None:
|
||||
source = self.get_source(source_id)
|
||||
with self._connect() as conn:
|
||||
doc_rows = conn.execute("select markdown_path from documents where source_id = ?", (source_id,)).fetchall()
|
||||
conn.execute("delete from documents where source_id = ?", (source_id,))
|
||||
conn.execute("delete from jobs where source_id = ?", (source_id,))
|
||||
conn.execute("delete from sources where id = ?", (source_id,))
|
||||
Path(source["stored_path"]).unlink(missing_ok=True)
|
||||
for row in doc_rows:
|
||||
Path(row["markdown_path"]).unlink(missing_ok=True)
|
||||
|
||||
def create_document(self, source_id: int, title: str, markdown: str, processor: str | None = None, page_ranges: str | None = None) -> dict[str, Any]:
|
||||
source = self.get_source(source_id)
|
||||
week_context = self.get_week_context(source["week_id"])
|
||||
processed_dir = self.week_dir(week_context["subject_slug"], week_context["week_number"]) / "processed"
|
||||
processed_dir.mkdir(parents=True, exist_ok=True)
|
||||
markdown_path = self._unique_path(processed_dir / f"{slugify(title)}.md")
|
||||
markdown_path.write_text(self._sanitize_markdown(markdown), encoding="utf-8")
|
||||
now = utc_now()
|
||||
with self._connect() as conn:
|
||||
old_docs = conn.execute("select markdown_path from documents where source_id = ?", (source_id,)).fetchall()
|
||||
conn.execute("delete from jobs where document_id in (select id from documents where source_id = ?)", (source_id,))
|
||||
conn.execute("delete from documents where source_id = ?", (source_id,))
|
||||
cursor = conn.execute(
|
||||
"insert into documents (source_id, title, markdown_path, created_at) values (?, ?, ?, ?)",
|
||||
(source_id, title, str(markdown_path), now),
|
||||
)
|
||||
conn.execute(
|
||||
"update sources set status = ?, processor = ?, page_ranges = ?, processed_at = ? where id = ?",
|
||||
("processed", processor, page_ranges, now, source_id),
|
||||
)
|
||||
for row in old_docs:
|
||||
Path(row["markdown_path"]).unlink(missing_ok=True)
|
||||
return self.get_document(cursor.lastrowid)
|
||||
|
||||
def get_document(self, document_id: int) -> dict[str, Any]:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("select * from documents where id = ?", (document_id,)).fetchone()
|
||||
if not row:
|
||||
raise ValueError("Documento no encontrado")
|
||||
return dict(row)
|
||||
|
||||
def update_document_markdown(self, document_id: int, content: str) -> dict[str, Any]:
|
||||
document = self.get_document(document_id)
|
||||
Path(document["markdown_path"]).write_text(self._sanitize_markdown(content), encoding="utf-8")
|
||||
return self.get_document(document_id)
|
||||
|
||||
def delete_document(self, document_id: int) -> None:
|
||||
document = self.get_document(document_id)
|
||||
Path(document["markdown_path"]).unlink(missing_ok=True)
|
||||
with self._connect() as conn:
|
||||
conn.execute("delete from jobs where document_id = ?", (document_id,))
|
||||
conn.execute("delete from documents where id = ?", (document_id,))
|
||||
|
||||
def list_documents(self, subject_id: int, week_number: int) -> list[dict[str, Any]]:
|
||||
week = self.get_week(subject_id, week_number)
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
select documents.*, sources.original_name as source_original_name,
|
||||
sources.source_type as source_type, sources.status as source_status,
|
||||
sources.processor as source_processor, sources.page_ranges as source_page_ranges
|
||||
from documents
|
||||
join sources on sources.id = documents.source_id
|
||||
where sources.week_id = ?
|
||||
order by documents.created_at desc
|
||||
""",
|
||||
(week["id"],),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def get_week_context(self, week_id: int) -> dict[str, Any]:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
select weeks.id as week_id, weeks.number as week_number, weeks.title as week_title,
|
||||
subjects.id as subject_id, subjects.name as subject_name, subjects.slug as subject_slug
|
||||
from weeks join subjects on subjects.id = weeks.subject_id
|
||||
where weeks.id = ?
|
||||
""",
|
||||
(week_id,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise ValueError("Contexto de semana no encontrado")
|
||||
return dict(row)
|
||||
|
||||
def export_week_markdown(self, subject_id: int, week_number: int) -> Path:
|
||||
subject = self.get_subject(subject_id)
|
||||
docs = self.list_documents(subject_id, week_number)
|
||||
export_dir = self.week_dir(subject["slug"], week_number) / "exports"
|
||||
export_dir.mkdir(parents=True, exist_ok=True)
|
||||
export_path = export_dir / f"{subject['slug']}-week-{week_number:02d}.md"
|
||||
parts = [
|
||||
"---\n"
|
||||
f"subject: {subject['name']}\n"
|
||||
f"subject_slug: {subject['slug']}\n"
|
||||
f"week: {week_number}\n"
|
||||
f"documents: {len(docs)}\n"
|
||||
f"exported_at: {utc_now()}\n"
|
||||
"---\n\n"
|
||||
f"# {subject['name']} - Semana {week_number}\n"
|
||||
]
|
||||
for doc in docs:
|
||||
path = Path(doc["markdown_path"])
|
||||
content = self._strip_frontmatter(self._read_markdown(path)).strip()
|
||||
parts.append(f"<!-- document_id: {doc['id']} -->\n\n{content}\n")
|
||||
export_path.write_text(self._sanitize_markdown("\n\n---\n\n".join(parts)), encoding="utf-8")
|
||||
return export_path
|
||||
|
||||
def export_week_markdown_zip(self, subject_id: int, week_number: int) -> Path:
|
||||
subject = self.get_subject(subject_id)
|
||||
docs = self.list_documents(subject_id, week_number)
|
||||
export_dir = self.week_dir(subject["slug"], week_number) / "exports"
|
||||
export_dir.mkdir(parents=True, exist_ok=True)
|
||||
export_path = export_dir / f"{subject['slug']}-week-{week_number:02d}-documentos.zip"
|
||||
used_names: set[str] = set()
|
||||
with zipfile.ZipFile(export_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
for index, doc in enumerate(docs, start=1):
|
||||
path = Path(doc["markdown_path"])
|
||||
name = self._unique_archive_name(f"{index:02d}-{slugify(doc['title'])}.md", used_names)
|
||||
archive.writestr(name, self._read_markdown(path))
|
||||
return export_path
|
||||
|
||||
def _unique_archive_name(self, filename: str, used_names: set[str]) -> str:
|
||||
path = Path(filename)
|
||||
candidate = path.name
|
||||
suffix = path.suffix
|
||||
stem = path.stem
|
||||
counter = 2
|
||||
while candidate in used_names:
|
||||
candidate = f"{stem}-{counter}{suffix}"
|
||||
counter += 1
|
||||
used_names.add(candidate)
|
||||
return candidate
|
||||
|
||||
def _read_markdown(self, path: Path) -> str:
|
||||
return self._sanitize_markdown(path.read_text(encoding="utf-8", errors="replace"))
|
||||
|
||||
def _sanitize_markdown(self, content: str) -> str:
|
||||
return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "", content)
|
||||
|
||||
def _strip_frontmatter(self, content: str) -> str:
|
||||
lines = content.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return content
|
||||
for index, line in enumerate(lines[1:], start=1):
|
||||
if line.strip() == "---":
|
||||
return "\n".join(lines[index + 1 :]).lstrip()
|
||||
return content
|
||||
|
||||
def detect_source_type(self, path: Path) -> str:
|
||||
ext = path.suffix.lower().lstrip(".")
|
||||
if ext in {"txt", "md"}:
|
||||
return "text"
|
||||
if ext == "docx":
|
||||
return "docx"
|
||||
if ext == "pdf":
|
||||
return "pdf"
|
||||
if ext in {"mp3", "wav", "m4a", "ogg", "flac", "webm"}:
|
||||
return "audio"
|
||||
if ext in {"mp4", "mov", "mkv", "avi"}:
|
||||
return "video"
|
||||
return "unknown"
|
||||
|
||||
def _unique_path(self, path: Path) -> Path:
|
||||
if not path.exists():
|
||||
return path
|
||||
stem = path.stem
|
||||
suffix = path.suffix
|
||||
for index in range(2, 10_000):
|
||||
candidate = path.with_name(f"{stem}-{index}{suffix}")
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
raise RuntimeError("No se pudo generar un nombre de archivo unico")
|
||||
|
||||
|
||||
def rows_to_models(rows: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [dict(row) for row in rows]
|
||||
@@ -0,0 +1,55 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SubjectCreate(BaseModel):
|
||||
name: str = Field(min_length=1)
|
||||
|
||||
|
||||
class Subject(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
slug: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class WeekCreate(BaseModel):
|
||||
number: int = Field(ge=1)
|
||||
title: Optional[str] = None
|
||||
|
||||
|
||||
class WeekUpdate(BaseModel):
|
||||
number: int = Field(ge=1)
|
||||
title: Optional[str] = None
|
||||
|
||||
|
||||
class Week(BaseModel):
|
||||
id: int
|
||||
subject_id: int
|
||||
number: int
|
||||
title: Optional[str]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class MarkdownUpdate(BaseModel):
|
||||
content: str
|
||||
|
||||
|
||||
class Job(BaseModel):
|
||||
id: int
|
||||
status: str
|
||||
message: str
|
||||
source_id: Optional[int] = None
|
||||
document_id: Optional[int] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class Document(BaseModel):
|
||||
id: int
|
||||
source_id: int
|
||||
title: str
|
||||
markdown_path: str
|
||||
created_at: datetime
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from backend.app.managers.knowledge_manager import KnowledgeManager
|
||||
from backend.app.services.markdown_builder import build_markdown
|
||||
from backend.app.services.processors.audio_processor import AudioProcessor
|
||||
from backend.app.services.processors.docx_processor import DocxProcessor
|
||||
from backend.app.services.processors.pdf_processor import PdfProcessor
|
||||
from backend.app.services.processors.text_processor import TextProcessor
|
||||
from backend.app.services.processors.video_processor import VideoProcessor
|
||||
|
||||
|
||||
class IngestionService:
|
||||
def __init__(self, manager: KnowledgeManager):
|
||||
self.manager = manager
|
||||
|
||||
def process_source(self, source_id: int, use_ocr: bool = False, page_ranges: str | None = None) -> dict:
|
||||
source = self.manager.get_source(source_id)
|
||||
context = self.manager.get_week_context(source["week_id"])
|
||||
path = Path(source["stored_path"])
|
||||
processor = self._processor_for(source["source_type"], use_ocr, page_ranges)
|
||||
processed = processor.process(path)
|
||||
markdown = build_markdown(
|
||||
title=processed.title,
|
||||
body=processed.body,
|
||||
metadata={
|
||||
"subject": context["subject_name"],
|
||||
"subject_slug": context["subject_slug"],
|
||||
"week": context["week_number"],
|
||||
"source_file": source["original_name"],
|
||||
"source_type": source["source_type"],
|
||||
"processor": processed.processor,
|
||||
"page_ranges": page_ranges if source["source_type"] == "pdf" and page_ranges else None,
|
||||
"language": "es",
|
||||
},
|
||||
)
|
||||
return self.manager.create_document(source_id, processed.title, markdown, processor=processed.processor, page_ranges=page_ranges)
|
||||
|
||||
def _processor_for(self, source_type: str, use_ocr: bool, page_ranges: str | None):
|
||||
if source_type == "text":
|
||||
return TextProcessor()
|
||||
if source_type == "docx":
|
||||
return DocxProcessor()
|
||||
if source_type == "pdf":
|
||||
return PdfProcessor(use_ocr=use_ocr, page_ranges=page_ranges)
|
||||
if source_type == "audio":
|
||||
return AudioProcessor()
|
||||
if source_type == "video":
|
||||
return VideoProcessor(self.manager.tmp_dir)
|
||||
raise RuntimeError(f"Tipo de fuente no soportado: {source_type}")
|
||||
@@ -0,0 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def build_markdown(title: str, body: str, metadata: dict[str, Any]) -> str:
|
||||
frontmatter = {
|
||||
**metadata,
|
||||
"processed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
lines = ["---"]
|
||||
for key, value in frontmatter.items():
|
||||
if value is not None:
|
||||
safe_value = str(value).replace("\n", " ")
|
||||
lines.append(f"{key}: {safe_value}")
|
||||
lines.extend(["---", "", f"# {title}", "", body.strip(), ""])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def title_from_path(path: Path) -> str:
|
||||
return path.stem.replace("-", " ").replace("_", " ").strip().title() or "Documento"
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from backend.app.services.markdown_builder import title_from_path
|
||||
from backend.app.services.processors.base import ProcessedContent
|
||||
from backend.app.services.providers.deepgram_client import DeepgramClient
|
||||
|
||||
|
||||
class AudioProcessor:
|
||||
def process(self, path: Path) -> ProcessedContent:
|
||||
client = DeepgramClient()
|
||||
data = client.transcribe_file(path)
|
||||
transcript, confidence = client.transcript_from_response(data)
|
||||
confidence_line = f"\n\n_Confianza estimada: {confidence * 100:.1f}%_" if isinstance(confidence, (int, float)) else ""
|
||||
return ProcessedContent(
|
||||
title=f"Transcripcion {title_from_path(path)}",
|
||||
body=f"## Transcripcion\n\n{transcript}{confidence_line}",
|
||||
processor="deepgram",
|
||||
)
|
||||
@@ -0,0 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessedContent:
|
||||
title: str
|
||||
body: str
|
||||
processor: str
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from docx import Document
|
||||
|
||||
from backend.app.services.markdown_builder import title_from_path
|
||||
from backend.app.services.processors.base import ProcessedContent
|
||||
|
||||
|
||||
class DocxProcessor:
|
||||
def process(self, path: Path) -> ProcessedContent:
|
||||
doc = Document(path)
|
||||
blocks: list[str] = []
|
||||
for paragraph in doc.paragraphs:
|
||||
text = paragraph.text.strip()
|
||||
if text:
|
||||
blocks.append(text)
|
||||
for table in doc.tables:
|
||||
rows = [[cell.text.strip().replace("\n", " ") for cell in row.cells] for row in table.rows]
|
||||
if rows:
|
||||
blocks.append(self._table_to_markdown(rows))
|
||||
return ProcessedContent(title=title_from_path(path), body="\n\n".join(blocks), processor="docx")
|
||||
|
||||
def _table_to_markdown(self, rows: list[list[str]]) -> str:
|
||||
header = rows[0]
|
||||
lines = [f"| {' | '.join(header)} |", f"| {' | '.join(['---'] * len(header))} |"]
|
||||
for row in rows[1:]:
|
||||
lines.append(f"| {' | '.join(row)} |")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import fitz
|
||||
|
||||
from backend.app.services.markdown_builder import title_from_path
|
||||
from backend.app.services.processors.base import ProcessedContent
|
||||
from backend.app.services.providers.mistral_client import MistralClient
|
||||
|
||||
|
||||
class PdfProcessor:
|
||||
def __init__(self, use_ocr: bool = False, page_ranges: str | None = None):
|
||||
self.use_ocr = use_ocr
|
||||
self.page_ranges = page_ranges
|
||||
|
||||
def process(self, path: Path) -> ProcessedContent:
|
||||
if self.use_ocr:
|
||||
return self._process_ocr(path)
|
||||
doc = fitz.open(path)
|
||||
pages: list[str] = []
|
||||
for index in self._selected_pages(len(doc)):
|
||||
page = doc[index - 1]
|
||||
text = page.get_text("text").strip()
|
||||
if text:
|
||||
pages.append(f"## Pagina {index}\n\n{text}")
|
||||
body = "\n\n".join(pages).strip()
|
||||
if len(body) < 80:
|
||||
raise RuntimeError("El PDF tiene poco texto extraible; procesalo con OCR")
|
||||
return ProcessedContent(title=title_from_path(path), body=body, processor="pdf-text")
|
||||
|
||||
def _process_ocr(self, path: Path) -> ProcessedContent:
|
||||
doc = fitz.open(path)
|
||||
client = MistralClient()
|
||||
parts: list[str] = []
|
||||
temp_paths: list[Path] = []
|
||||
try:
|
||||
for index in self._selected_pages(len(doc)):
|
||||
page = doc[index - 1]
|
||||
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
|
||||
image_path = path.with_name(f"{path.stem}-page-{index}.jpg")
|
||||
pix.save(image_path)
|
||||
temp_paths.append(image_path)
|
||||
parts.append(f"## Pagina {index}\n\n{client.ocr_image(image_path)}")
|
||||
finally:
|
||||
for temp_path in temp_paths:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
return ProcessedContent(title=title_from_path(path), body="\n\n".join(parts), processor="mistral-ocr")
|
||||
|
||||
def _selected_pages(self, total_pages: int) -> list[int]:
|
||||
if total_pages < 1:
|
||||
raise RuntimeError("El PDF no tiene paginas")
|
||||
if not self.page_ranges or not self.page_ranges.strip():
|
||||
return list(range(1, total_pages + 1))
|
||||
|
||||
selected: list[int] = []
|
||||
for raw_part in self.page_ranges.split(","):
|
||||
part = raw_part.strip()
|
||||
if not part:
|
||||
continue
|
||||
if "-" in part:
|
||||
start_text, end_text = [piece.strip() for piece in part.split("-", 1)]
|
||||
if not start_text.isdigit() or not end_text.isdigit():
|
||||
raise RuntimeError(f"Rango de paginas invalido: {part}")
|
||||
start = int(start_text)
|
||||
end = int(end_text)
|
||||
if start > end:
|
||||
raise RuntimeError(f"Rango de paginas invertido: {part}")
|
||||
selected.extend(range(start, end + 1))
|
||||
else:
|
||||
if not part.isdigit():
|
||||
raise RuntimeError(f"Pagina invalida: {part}")
|
||||
selected.append(int(part))
|
||||
|
||||
unique_pages = sorted(set(selected))
|
||||
invalid = [page for page in unique_pages if page < 1 or page > total_pages]
|
||||
if invalid:
|
||||
raise RuntimeError(f"Paginas fuera de rango: {', '.join(map(str, invalid))}. El PDF tiene {total_pages} paginas")
|
||||
if not unique_pages:
|
||||
raise RuntimeError("No se seleccionaron paginas para procesar")
|
||||
return unique_pages
|
||||
@@ -0,0 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from backend.app.services.markdown_builder import title_from_path
|
||||
from backend.app.services.processors.base import ProcessedContent
|
||||
|
||||
|
||||
class TextProcessor:
|
||||
def process(self, path: Path) -> ProcessedContent:
|
||||
return ProcessedContent(
|
||||
title=title_from_path(path),
|
||||
body=path.read_text(encoding="utf-8", errors="replace"),
|
||||
processor="text",
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from backend.app.services.processors.audio_processor import AudioProcessor
|
||||
from backend.app.services.processors.base import ProcessedContent
|
||||
|
||||
|
||||
class VideoProcessor:
|
||||
def __init__(self, tmp_dir: Path):
|
||||
self.tmp_dir = tmp_dir
|
||||
|
||||
def process(self, path: Path) -> ProcessedContent:
|
||||
self.tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
audio_path = self.tmp_dir / f"{path.stem}.wav"
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
str(path),
|
||||
"-vn",
|
||||
"-acodec",
|
||||
"pcm_s16le",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-ac",
|
||||
"1",
|
||||
str(audio_path),
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
processed = AudioProcessor().process(audio_path)
|
||||
processed.title = f"Transcripcion {path.stem}"
|
||||
processed.processor = "ffmpeg+deepgram"
|
||||
return processed
|
||||
except FileNotFoundError as exc:
|
||||
raise RuntimeError("ffmpeg no esta instalado o no esta disponible en PATH") from exc
|
||||
except subprocess.CalledProcessError as exc:
|
||||
raise RuntimeError(f"No se pudo extraer audio del video: {exc.stderr}") from exc
|
||||
finally:
|
||||
audio_path.unlink(missing_ok=True)
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from backend.app.config import settings
|
||||
|
||||
|
||||
class DeepgramClient:
|
||||
endpoint = "https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=es"
|
||||
|
||||
def transcribe_file(self, path: Path) -> dict:
|
||||
if not settings.deepgram_api_key:
|
||||
raise RuntimeError("DEEPGRAM_API_KEY no esta configurada")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Token {settings.deepgram_api_key}",
|
||||
"Content-Type": self._content_type(path),
|
||||
}
|
||||
with path.open("rb") as audio_file:
|
||||
response = requests.post(self.endpoint, headers=headers, data=audio_file, timeout=600)
|
||||
|
||||
if response.status_code == 401:
|
||||
raise RuntimeError("API Key de Deepgram invalida")
|
||||
if not response.ok:
|
||||
try:
|
||||
error = response.json()
|
||||
except ValueError:
|
||||
error = response.text
|
||||
raise RuntimeError(f"Deepgram error {response.status_code}: {error}")
|
||||
return response.json()
|
||||
|
||||
def transcript_from_response(self, data: dict) -> tuple[str, float | None]:
|
||||
alternative = data.get("results", {}).get("channels", [{}])[0].get("alternatives", [{}])[0]
|
||||
transcript = alternative.get("paragraphs", {}).get("transcript") or alternative.get("transcript") or ""
|
||||
confidence = alternative.get("confidence")
|
||||
if not transcript.strip():
|
||||
raise RuntimeError("Deepgram no devolvio transcripcion")
|
||||
return transcript.strip(), confidence
|
||||
|
||||
def _content_type(self, path: Path) -> str:
|
||||
ext = path.suffix.lower()
|
||||
return {
|
||||
".mp3": "audio/mpeg",
|
||||
".wav": "audio/wav",
|
||||
".m4a": "audio/mp4",
|
||||
".mp4": "audio/mp4",
|
||||
".webm": "audio/webm",
|
||||
".ogg": "audio/ogg",
|
||||
".flac": "audio/flac",
|
||||
}.get(ext, "application/octet-stream")
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from backend.app.config import settings
|
||||
|
||||
|
||||
class MistralClient:
|
||||
endpoint = "https://api.mistral.ai/v1/chat/completions"
|
||||
model = "pixtral-12b-2409"
|
||||
|
||||
def ocr_image(self, image_path: Path) -> str:
|
||||
if not settings.mistral_api_key:
|
||||
raise RuntimeError("MISTRAL_API_KEY no esta configurada")
|
||||
data_url = self._image_data_url(image_path)
|
||||
response = requests.post(
|
||||
self.endpoint,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {settings.mistral_api_key}",
|
||||
},
|
||||
json={
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Extract all text from this image and format it as Markdown. If there are tables, represent them using Markdown table syntax. Do not include commentary.",
|
||||
},
|
||||
{"type": "image_url", "image_url": data_url},
|
||||
],
|
||||
}
|
||||
],
|
||||
"max_tokens": 3000,
|
||||
},
|
||||
timeout=180,
|
||||
)
|
||||
if response.status_code == 401:
|
||||
raise RuntimeError("API Key de Mistral invalida")
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"Mistral error {response.status_code}: {response.text}")
|
||||
data = response.json()
|
||||
return data["choices"][0]["message"]["content"]
|
||||
|
||||
def _image_data_url(self, image_path: Path) -> str:
|
||||
encoded = base64.b64encode(image_path.read_bytes()).decode("ascii")
|
||||
return f"data:image/jpeg;base64,{encoded}"
|
||||
@@ -0,0 +1,8 @@
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii")
|
||||
slug = re.sub(r"[^a-zA-Z0-9]+", "-", normalized).strip("-").lower()
|
||||
return slug or "sin-titulo"
|
||||
Reference in New Issue
Block a user