1
This commit is contained in:
@@ -2,4 +2,5 @@ APP_ENV=development
|
||||
APP_DATA_DIR=./data
|
||||
DATABASE_PATH=./data/app.db
|
||||
MISTRAL_API_KEY=
|
||||
MISTRAL_OCR_MODEL=mistral-ocr-latest
|
||||
DEEPGRAM_API_KEY=
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
- `KnowledgeManager` owns filesystem layout and SQLite metadata; processors should not write final Markdown directly.
|
||||
- `IngestionService` selects processors by `source_type` and wraps output with Markdown frontmatter for LLM ingestion.
|
||||
- Text and Markdown are read directly; `.docx` uses `python-docx`; PDFs use PyMuPDF text extraction unless `use_ocr=true` is passed. PDF uploads can also pass `page_ranges` like `1-5,8,10-12`; ranges apply to both standard extraction and Mistral OCR.
|
||||
- OCR uses Mistral `https://api.mistral.ai/v1/chat/completions` with model `pixtral-12b-2409`.
|
||||
- OCR uses Mistral `https://api.mistral.ai/v1/ocr` with `MISTRAL_OCR_MODEL`, defaulting to `mistral-ocr-latest`.
|
||||
- Audio uses Deepgram `https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=es`.
|
||||
- Video processing requires `ffmpeg`; audio is extracted to `data/tmp/`, transcribed with Deepgram, then deleted.
|
||||
- The frontend calls the backend through `NEXT_PUBLIC_API_URL`; when ports change, update `frontend/.env.local` or use the adaptive `.bat`.
|
||||
|
||||
@@ -52,6 +52,7 @@ Para Coolify, configura backend y frontend como servicios separados. Monta un vo
|
||||
- `APP_DATA_DIR`: carpeta de datos persistentes.
|
||||
- `DATABASE_PATH`: ruta SQLite.
|
||||
- `MISTRAL_API_KEY`: OCR para PDFs escaneados.
|
||||
- `MISTRAL_OCR_MODEL`: modelo OCR de Mistral. Por defecto `mistral-ocr-latest`; puedes fijarlo a `mistral-ocr-2512`.
|
||||
- `DEEPGRAM_API_KEY`: transcripción de audio/video.
|
||||
|
||||
No subas claves reales al repositorio.
|
||||
|
||||
@@ -8,6 +8,7 @@ class Settings(BaseSettings):
|
||||
app_data_dir: Path = Path("./data")
|
||||
database_path: Path = Path("./data/app.db")
|
||||
mistral_api_key: str = ""
|
||||
mistral_ocr_model: str = "mistral-ocr-latest"
|
||||
deepgram_api_key: str = ""
|
||||
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
||||
|
||||
@@ -9,8 +9,7 @@ from backend.app.config import settings
|
||||
|
||||
|
||||
class MistralClient:
|
||||
endpoint = "https://api.mistral.ai/v1/chat/completions"
|
||||
model = "pixtral-12b-2409"
|
||||
endpoint = "https://api.mistral.ai/v1/ocr"
|
||||
|
||||
def ocr_image(self, image_path: Path) -> str:
|
||||
if not settings.mistral_api_key:
|
||||
@@ -23,20 +22,9 @@ class MistralClient:
|
||||
"Authorization": f"Bearer {settings.mistral_api_key}",
|
||||
},
|
||||
json={
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Extract all text from this image and format it as Markdown. If there are tables, represent them using Markdown table syntax. Do not include commentary.",
|
||||
},
|
||||
{"type": "image_url", "image_url": data_url},
|
||||
],
|
||||
}
|
||||
],
|
||||
"max_tokens": 3000,
|
||||
"model": settings.mistral_ocr_model,
|
||||
"document": {"type": "image_url", "image_url": data_url},
|
||||
"include_image_base64": False,
|
||||
},
|
||||
timeout=180,
|
||||
)
|
||||
@@ -45,7 +33,13 @@ class MistralClient:
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"Mistral error {response.status_code}: {response.text}")
|
||||
data = response.json()
|
||||
return data["choices"][0]["message"]["content"]
|
||||
pages = data.get("pages") or []
|
||||
markdown_pages = [page.get("markdown", "").strip() for page in pages if page.get("markdown")]
|
||||
if markdown_pages:
|
||||
return "\n\n".join(markdown_pages)
|
||||
if data.get("markdown"):
|
||||
return data["markdown"].strip()
|
||||
raise RuntimeError("Mistral OCR no devolvio Markdown")
|
||||
|
||||
def _image_data_url(self, image_path: Path) -> str:
|
||||
encoded = base64.b64encode(image_path.read_bytes()).decode("ascii")
|
||||
|
||||
+12
-2
@@ -8,11 +8,14 @@ services:
|
||||
APP_DATA_DIR: /app/data
|
||||
DATABASE_PATH: /app/data/app.db
|
||||
MISTRAL_API_KEY: ${MISTRAL_API_KEY:-}
|
||||
MISTRAL_OCR_MODEL: ${MISTRAL_OCR_MODEL:-mistral-ocr-latest}
|
||||
DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-}
|
||||
volumes:
|
||||
- station-data:/app/data
|
||||
expose:
|
||||
- "8000"
|
||||
ports:
|
||||
- "8000"
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
||||
@@ -34,10 +37,17 @@ services:
|
||||
HOSTNAME: 0.0.0.0
|
||||
expose:
|
||||
- "3000"
|
||||
ports:
|
||||
- "3000"
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
|
||||
volumes:
|
||||
station-data:
|
||||
|
||||
@@ -1306,28 +1306,19 @@
|
||||
|
||||
async function mistralOCR(dataUrl) {
|
||||
try {
|
||||
const res = await fetch('https://api.mistral.ai/v1/chat/completions', {
|
||||
const res = await fetch('https://api.mistral.ai/v1/ocr', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${MISTRAL_API_KEY}`
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'pixtral-12b-2409',
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Extract all text from this image and format it as Markdown. If there are tables, represent them using Markdown table syntax. Do not include any commentary or introductions, just the document content.'
|
||||
},
|
||||
{
|
||||
model: 'mistral-ocr-latest',
|
||||
document: {
|
||||
type: 'image_url',
|
||||
image_url: dataUrl // dataUrl ya incluye 'data:image/jpeg;base64,...'
|
||||
}
|
||||
]
|
||||
}],
|
||||
max_tokens: 2000
|
||||
},
|
||||
include_image_base64: false
|
||||
})
|
||||
});
|
||||
|
||||
@@ -1342,9 +1333,11 @@
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
if (!data.choices || !data.choices[0]) throw new Error("Respuesta de API vacía");
|
||||
const pages = data.pages || [];
|
||||
const markdown = pages.map(page => page.markdown || '').filter(Boolean).join('\n\n');
|
||||
if (!markdown && !data.markdown) throw new Error("Respuesta de API vacía");
|
||||
|
||||
return data.choices[0].message.content;
|
||||
return markdown || data.markdown;
|
||||
} catch (error) {
|
||||
console.error("Mistral API Error:", error);
|
||||
throw error;
|
||||
|
||||
@@ -181,7 +181,7 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
||||
<h1 className="text-4xl font-bold">Semana {weekNumber}</h1>
|
||||
<p className="mt-2 text-slate-400">{week?.title || "Sube fuentes y genera Markdown procesado."}</p>
|
||||
</div>
|
||||
{error && <Card className="border-red-500/30 bg-red-500/10 text-red-200">{error}</Card>}
|
||||
{error && <Card className="whitespace-pre-wrap border-red-500/30 bg-red-500/10 text-red-200 [overflow-wrap:anywhere]">{error}</Card>}
|
||||
|
||||
<Card>
|
||||
<div className="flex flex-wrap items-center justify-between gap-4">
|
||||
@@ -298,7 +298,7 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
||||
<Card>
|
||||
<h2 className="text-lg font-semibold">Trabajos</h2>
|
||||
<div className="mt-4 space-y-3">
|
||||
{jobs.map((job) => <div key={job.id} className="rounded-xl bg-slate-900 p-3 text-sm"><div className="flex justify-between"><span>#{job.id}</span><StatusBadge status={job.status} /></div><p className="mt-2 text-slate-400">{job.message}</p></div>)}
|
||||
{jobs.map((job) => <div key={job.id} className="min-w-0 rounded-xl bg-slate-900 p-3 text-sm"><div className="flex items-start justify-between gap-2"><span>#{job.id}</span><StatusBadge status={job.status} /></div><p className="mt-2 whitespace-pre-wrap text-slate-400 [overflow-wrap:anywhere]">{job.message}</p></div>)}
|
||||
{jobs.length === 0 && <p className="text-sm text-slate-400">Sin trabajos.</p>}
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
Reference in New Issue
Block a user