This commit is contained in:
urieljarethbusiness-cpu
2026-05-17 11:18:12 -06:00
parent 64b3d15b90
commit b964fcb2e6
8 changed files with 40 additions and 40 deletions
+1
View File
@@ -2,4 +2,5 @@ APP_ENV=development
APP_DATA_DIR=./data APP_DATA_DIR=./data
DATABASE_PATH=./data/app.db DATABASE_PATH=./data/app.db
MISTRAL_API_KEY= MISTRAL_API_KEY=
MISTRAL_OCR_MODEL=mistral-ocr-latest
DEEPGRAM_API_KEY= DEEPGRAM_API_KEY=
+1 -1
View File
@@ -18,7 +18,7 @@
- `KnowledgeManager` owns filesystem layout and SQLite metadata; processors should not write final Markdown directly. - `KnowledgeManager` owns filesystem layout and SQLite metadata; processors should not write final Markdown directly.
- `IngestionService` selects processors by `source_type` and wraps output with Markdown frontmatter for LLM ingestion. - `IngestionService` selects processors by `source_type` and wraps output with Markdown frontmatter for LLM ingestion.
- Text and Markdown are read directly; `.docx` uses `python-docx`; PDFs use PyMuPDF text extraction unless `use_ocr=true` is passed. PDF uploads can also pass `page_ranges` like `1-5,8,10-12`; ranges apply to both standard extraction and Mistral OCR. - Text and Markdown are read directly; `.docx` uses `python-docx`; PDFs use PyMuPDF text extraction unless `use_ocr=true` is passed. PDF uploads can also pass `page_ranges` like `1-5,8,10-12`; ranges apply to both standard extraction and Mistral OCR.
- OCR uses Mistral `https://api.mistral.ai/v1/chat/completions` with model `pixtral-12b-2409`. - OCR uses Mistral `https://api.mistral.ai/v1/ocr` with `MISTRAL_OCR_MODEL`, defaulting to `mistral-ocr-latest`.
- Audio uses Deepgram `https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=es`. - Audio uses Deepgram `https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=es`.
- Video processing requires `ffmpeg`; audio is extracted to `data/tmp/`, transcribed with Deepgram, then deleted. - Video processing requires `ffmpeg`; audio is extracted to `data/tmp/`, transcribed with Deepgram, then deleted.
- The frontend calls the backend through `NEXT_PUBLIC_API_URL`; when ports change, update `frontend/.env.local` or use the adaptive `.bat`. - The frontend calls the backend through `NEXT_PUBLIC_API_URL`; when ports change, update `frontend/.env.local` or use the adaptive `.bat`.
+1
View File
@@ -52,6 +52,7 @@ Para Coolify, configura backend y frontend como servicios separados. Monta un vo
- `APP_DATA_DIR`: carpeta de datos persistentes. - `APP_DATA_DIR`: carpeta de datos persistentes.
- `DATABASE_PATH`: ruta SQLite. - `DATABASE_PATH`: ruta SQLite.
- `MISTRAL_API_KEY`: OCR para PDFs escaneados. - `MISTRAL_API_KEY`: OCR para PDFs escaneados.
- `MISTRAL_OCR_MODEL`: modelo OCR de Mistral. Por defecto `mistral-ocr-latest`; puedes fijarlo a `mistral-ocr-2512`.
- `DEEPGRAM_API_KEY`: transcripción de audio/video. - `DEEPGRAM_API_KEY`: transcripción de audio/video.
No subas claves reales al repositorio. No subas claves reales al repositorio.
+1
View File
@@ -8,6 +8,7 @@ class Settings(BaseSettings):
app_data_dir: Path = Path("./data") app_data_dir: Path = Path("./data")
database_path: Path = Path("./data/app.db") database_path: Path = Path("./data/app.db")
mistral_api_key: str = "" mistral_api_key: str = ""
mistral_ocr_model: str = "mistral-ocr-latest"
deepgram_api_key: str = "" deepgram_api_key: str = ""
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
@@ -9,8 +9,7 @@ from backend.app.config import settings
class MistralClient: class MistralClient:
endpoint = "https://api.mistral.ai/v1/chat/completions" endpoint = "https://api.mistral.ai/v1/ocr"
model = "pixtral-12b-2409"
def ocr_image(self, image_path: Path) -> str: def ocr_image(self, image_path: Path) -> str:
if not settings.mistral_api_key: if not settings.mistral_api_key:
@@ -23,20 +22,9 @@ class MistralClient:
"Authorization": f"Bearer {settings.mistral_api_key}", "Authorization": f"Bearer {settings.mistral_api_key}",
}, },
json={ json={
"model": self.model, "model": settings.mistral_ocr_model,
"messages": [ "document": {"type": "image_url", "image_url": data_url},
{ "include_image_base64": False,
"role": "user",
"content": [
{
"type": "text",
"text": "Extract all text from this image and format it as Markdown. If there are tables, represent them using Markdown table syntax. Do not include commentary.",
},
{"type": "image_url", "image_url": data_url},
],
}
],
"max_tokens": 3000,
}, },
timeout=180, timeout=180,
) )
@@ -45,7 +33,13 @@ class MistralClient:
if not response.ok: if not response.ok:
raise RuntimeError(f"Mistral error {response.status_code}: {response.text}") raise RuntimeError(f"Mistral error {response.status_code}: {response.text}")
data = response.json() data = response.json()
return data["choices"][0]["message"]["content"] pages = data.get("pages") or []
markdown_pages = [page.get("markdown", "").strip() for page in pages if page.get("markdown")]
if markdown_pages:
return "\n\n".join(markdown_pages)
if data.get("markdown"):
return data["markdown"].strip()
raise RuntimeError("Mistral OCR no devolvio Markdown")
def _image_data_url(self, image_path: Path) -> str: def _image_data_url(self, image_path: Path) -> str:
encoded = base64.b64encode(image_path.read_bytes()).decode("ascii") encoded = base64.b64encode(image_path.read_bytes()).decode("ascii")
+12 -2
View File
@@ -8,11 +8,14 @@ services:
APP_DATA_DIR: /app/data APP_DATA_DIR: /app/data
DATABASE_PATH: /app/data/app.db DATABASE_PATH: /app/data/app.db
MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} MISTRAL_API_KEY: ${MISTRAL_API_KEY:-}
MISTRAL_OCR_MODEL: ${MISTRAL_OCR_MODEL:-mistral-ocr-latest}
DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-} DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-}
volumes: volumes:
- station-data:/app/data - station-data:/app/data
expose: expose:
- "8000" - "8000"
ports:
- "8000"
restart: unless-stopped restart: unless-stopped
healthcheck: healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -34,10 +37,17 @@ services:
HOSTNAME: 0.0.0.0 HOSTNAME: 0.0.0.0
expose: expose:
- "3000" - "3000"
ports:
- "3000"
depends_on: depends_on:
backend: - backend
condition: service_healthy
restart: unless-stopped restart: unless-stopped
healthcheck:
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"]
interval: 30s
timeout: 10s
retries: 3
start_period: 30s
volumes: volumes:
station-data: station-data:
+11 -18
View File
@@ -1306,28 +1306,19 @@
async function mistralOCR(dataUrl) { async function mistralOCR(dataUrl) {
try { try {
const res = await fetch('https://api.mistral.ai/v1/chat/completions', { const res = await fetch('https://api.mistral.ai/v1/ocr', {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
'Authorization': `Bearer ${MISTRAL_API_KEY}` 'Authorization': `Bearer ${MISTRAL_API_KEY}`
}, },
body: JSON.stringify({ body: JSON.stringify({
model: 'pixtral-12b-2409', model: 'mistral-ocr-latest',
messages: [{ document: {
role: 'user', type: 'image_url',
content: [ image_url: dataUrl // dataUrl ya incluye 'data:image/jpeg;base64,...'
{ },
type: 'text', include_image_base64: false
text: 'Extract all text from this image and format it as Markdown. If there are tables, represent them using Markdown table syntax. Do not include any commentary or introductions, just the document content.'
},
{
type: 'image_url',
image_url: dataUrl // dataUrl ya incluye 'data:image/jpeg;base64,...'
}
]
}],
max_tokens: 2000
}) })
}); });
@@ -1342,9 +1333,11 @@
} }
const data = await res.json(); const data = await res.json();
if (!data.choices || !data.choices[0]) throw new Error("Respuesta de API vacía"); const pages = data.pages || [];
const markdown = pages.map(page => page.markdown || '').filter(Boolean).join('\n\n');
if (!markdown && !data.markdown) throw new Error("Respuesta de API vacía");
return data.choices[0].message.content; return markdown || data.markdown;
} catch (error) { } catch (error) {
console.error("Mistral API Error:", error); console.error("Mistral API Error:", error);
throw error; throw error;
@@ -181,7 +181,7 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
<h1 className="text-4xl font-bold">Semana {weekNumber}</h1> <h1 className="text-4xl font-bold">Semana {weekNumber}</h1>
<p className="mt-2 text-slate-400">{week?.title || "Sube fuentes y genera Markdown procesado."}</p> <p className="mt-2 text-slate-400">{week?.title || "Sube fuentes y genera Markdown procesado."}</p>
</div> </div>
{error && <Card className="border-red-500/30 bg-red-500/10 text-red-200">{error}</Card>} {error && <Card className="whitespace-pre-wrap border-red-500/30 bg-red-500/10 text-red-200 [overflow-wrap:anywhere]">{error}</Card>}
<Card> <Card>
<div className="flex flex-wrap items-center justify-between gap-4"> <div className="flex flex-wrap items-center justify-between gap-4">
@@ -298,7 +298,7 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
<Card> <Card>
<h2 className="text-lg font-semibold">Trabajos</h2> <h2 className="text-lg font-semibold">Trabajos</h2>
<div className="mt-4 space-y-3"> <div className="mt-4 space-y-3">
{jobs.map((job) => <div key={job.id} className="rounded-xl bg-slate-900 p-3 text-sm"><div className="flex justify-between"><span>#{job.id}</span><StatusBadge status={job.status} /></div><p className="mt-2 text-slate-400">{job.message}</p></div>)} {jobs.map((job) => <div key={job.id} className="min-w-0 rounded-xl bg-slate-900 p-3 text-sm"><div className="flex items-start justify-between gap-2"><span>#{job.id}</span><StatusBadge status={job.status} /></div><p className="mt-2 whitespace-pre-wrap text-slate-400 [overflow-wrap:anywhere]">{job.message}</p></div>)}
{jobs.length === 0 && <p className="text-sm text-slate-400">Sin trabajos.</p>} {jobs.length === 0 && <p className="text-sm text-slate-400">Sin trabajos.</p>}
</div> </div>
</Card> </Card>