From cd0b271bc9f8dbeccb2595c1667a993427efc316 Mon Sep 17 00:00:00 2001 From: urieljareth Date: Mon, 21 Sep 2026 11:47:49 -0600 Subject: [PATCH] Carga en lote concurrente con progreso visual: pool de 3 subidas simultaneas (PDF/audio/txt/imagenes), progreso real de bytes por archivo via XHR con panel de barras animadas, aislamiento de fallos por archivo y SQLite en modo WAL + busy_timeout para subidas concurrentes sin bloqueos. docker-compose.yaml sincronizado con docker-compose.yml --- backend/app/managers/knowledge_manager.py | 7 +- docker-compose.yaml | 5 + .../[subjectId]/weeks/[weekNumber]/page.tsx | 162 +++++++++++++++++- frontend/lib/api.ts | 47 ++++- frontend/tailwind.config.ts | 9 + 5 files changed, 216 insertions(+), 14 deletions(-) diff --git a/backend/app/managers/knowledge_manager.py b/backend/app/managers/knowledge_manager.py index ceaf4c1..e8f8862 100644 --- a/backend/app/managers/knowledge_manager.py +++ b/backend/app/managers/knowledge_manager.py @@ -38,8 +38,13 @@ class KnowledgeManager: self._init_db() def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.database_path) + conn = sqlite3.connect(self.database_path, timeout=30) conn.row_factory = sqlite3.Row + # WAL + busy_timeout: permite subidas concurrentes y trabajos en + # segundo plano sin errores "database is locked". + conn.execute("pragma busy_timeout = 30000") + conn.execute("pragma journal_mode = wal") + conn.execute("pragma synchronous = normal") return conn def _init_db(self) -> None: diff --git a/docker-compose.yaml b/docker-compose.yaml index 267be5a..b2a9383 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,6 +12,11 @@ services: DATABASE_PATH: /app/data/app.db MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} MISTRAL_OCR_MODEL: ${MISTRAL_OCR_MODEL:-mistral-ocr-latest} + MISTRAL_OCR_BATCH_THRESHOLD: ${MISTRAL_OCR_BATCH_THRESHOLD:-3} + MISTRAL_OCR_BATCH_POLL_SECONDS: ${MISTRAL_OCR_BATCH_POLL_SECONDS:-10} + MISTRAL_OCR_BATCH_TIMEOUT_SECONDS: ${MISTRAL_OCR_BATCH_TIMEOUT_SECONDS:-1800} + MISTRAL_OCR_RETRY_ATTEMPTS: ${MISTRAL_OCR_RETRY_ATTEMPTS:-3} + MISTRAL_OCR_RETRY_BACKOFF_BASE: ${MISTRAL_OCR_RETRY_BACKOFF_BASE:-2.0} DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-} NODE_ENV: production PORT: 3000 diff --git a/frontend/app/subjects/[subjectId]/weeks/[weekNumber]/page.tsx b/frontend/app/subjects/[subjectId]/weeks/[weekNumber]/page.tsx index 704e4d6..7fdea3b 100644 --- a/frontend/app/subjects/[subjectId]/weeks/[weekNumber]/page.tsx +++ b/frontend/app/subjects/[subjectId]/weeks/[weekNumber]/page.tsx @@ -2,7 +2,7 @@ import Link from "next/link"; import { ChangeEvent, DragEvent, useEffect, useMemo, useState } from "react"; -import { api, DocumentItem, Job, Source, Subject, Week } from "@/lib/api"; +import { api, DocumentItem, Job, Source, Subject, Week, uploadWithProgress } from "@/lib/api"; import { Shell } from "@/components/Shell"; import { Button, Card, ConfirmDialog, StatusBadge } from "@/components/ui"; @@ -10,6 +10,17 @@ type PendingDelete = | { type: "document"; item: DocumentItem } | { type: "source"; item: Source }; +type UploadItem = { + id: string; + name: string; + size: number; + progress: number; + status: "uploading" | "done" | "error"; + error?: string; +}; + +const UPLOAD_CONCURRENCY = 3; + export default function WeekPage({ params }: { params: { subjectId: string; weekNumber: string } }) { const subjectId = Number(params.subjectId); const weekNumber = Number(params.weekNumber); @@ -23,6 +34,7 @@ export default function WeekPage({ params }: { params: { subjectId: string; week const [pageRanges, setPageRanges] = useState(""); const [pendingFiles, setPendingFiles] = useState([]); const [uploading, setUploading] = useState(false); + const [uploadItems, setUploadItems] = useState([]); const [deletingId, setDeletingId] = useState(null); const [reprocessingId, setReprocessingId] = useState(null); const [pendingDelete, setPendingDelete] = useState(null); @@ -31,6 +43,15 @@ export default function WeekPage({ params }: { params: { subjectId: string; week const activeJobs = useMemo(() => jobs.some((job) => ["queued", "processing"].includes(job.status)), [jobs]); const pendingPdfCount = useMemo(() => pendingFiles.filter(isPdfFile).length, [pendingFiles]); + const uploadSummary = useMemo(() => { + if (!uploadItems.length) return null; + const finished = uploadItems.filter((item) => item.status !== "uploading").length; + const failed = uploadItems.filter((item) => item.status === "error").length; + const percent = Math.round( + uploadItems.reduce((acc, item) => acc + (item.status === "done" ? 100 : item.progress), 0) / uploadItems.length, + ); + return { finished, failed, total: uploadItems.length, percent }; + }, [uploadItems]); async function load() { const [subjectData, weekData, sourceList, jobList, documentList] = await Promise.all([ @@ -49,16 +70,63 @@ export default function WeekPage({ params }: { params: { subjectId: string; week async function uploadFileList(files: File[]) { if (!files.length) return; setUploading(true); setError(""); + const ranges = pageMode === "ranges" ? pageRanges : undefined; + const items: UploadItem[] = files.map((file, index) => ({ + id: `${Date.now()}-${index}-${file.name}`, + name: file.name, + size: file.size, + progress: 0, + status: "uploading", + })); + setUploadItems(items); + const patchItem = (id: string, patch: Partial) => { + setUploadItems((prev) => prev.map((item) => (item.id === id ? { ...item, ...patch } : item))); + }; + // Pool de subida: hasta UPLOAD_CONCURRENCY archivos en vuelo a la vez, + // y un archivo fallido no interrumpe el resto de la tanda. + const failures: string[] = []; + let cursor = 0; + async function runUploadQueue() { + while (cursor < files.length) { + const index = cursor++; + const file = files[index]; + const item = items[index]; + try { + await uploadWithProgress( + subjectId, + weekNumber, + file, + isImageFile(file) || pdfMethod === "mistral_ocr", + (progress) => patchItem(item.id, { progress: Math.round((progress.loaded / progress.total) * 100) }), + ranges, + ); + patchItem(item.id, { status: "done", progress: 100 }); + } catch (err) { + const message = err instanceof Error ? err.message : "No se pudo subir archivo"; + failures.push(`${file.name}: ${message}`); + patchItem(item.id, { status: "error", error: message }); + } + } + } + const loadErrors: string[] = []; try { - const ranges = pageMode === "ranges" ? pageRanges : undefined; - for (const file of files) await api.upload(subjectId, weekNumber, file, isImageFile(file) || pdfMethod === "mistral_ocr", ranges); + await Promise.all(Array.from({ length: Math.min(UPLOAD_CONCURRENCY, files.length) }, () => runUploadQueue())); await load(); } catch (err) { - setError(err instanceof Error ? err.message : "No se pudo subir archivo"); + loadErrors.push(err instanceof Error ? err.message : "No se pudo completar la subida"); } finally { setUploading(false); setDragActive(false); } + const problems = [...failures, ...loadErrors]; + if (problems.length) { + setError(problems.join("\n")); + } else { + // Ocultar el panel unos segundos despues de terminar si no hay fallos. + window.setTimeout(() => { + setUploadItems((prev) => (prev.some((item) => item.status === "uploading") ? prev : [])); + }, 6000); + } } function isPdfFile(file: File) { @@ -211,8 +279,92 @@ export default function WeekPage({ params }: { params: { subjectId: string; week

Arrastra archivos aqui

Puedes soltar varios archivos a la vez. Se procesaran en esta semana.

Soporta PDF, DOCX, TXT, Markdown, imagenes (OCR automatico), audio y video.

- {uploading &&

Subiendo archivos...

} + {uploading && ( +

+

+ )} + {uploadItems.length > 0 && uploadSummary && ( +
+
+

+ {uploading ? ( + <> +

+ {!uploading && ( + + )} +
+
+
+
+
    + {uploadItems.map((item) => ( +
  • +
    + + {item.status === "uploading" && ( + + + {item.status === "error" ? "fallo" : `${Math.round(item.progress)}%`} + +
    +
    + {item.status === "uploading" ? ( +
    +
    +
    +
    +
    + ) : ( +
    + )} +
    + {item.error &&

    {item.error}

    } +
    • + ))} +
+
+ )} {pendingFiles.length > 0 && (
diff --git a/frontend/lib/api.ts b/frontend/lib/api.ts index 70764a0..ddabea0 100644 --- a/frontend/lib/api.ts +++ b/frontend/lib/api.ts @@ -45,6 +45,44 @@ async function parseResponse(response: Response): Promise { return response.json() as Promise; } +export type UploadProgress = { loaded: number; total: number }; +export type UploadResult = { source: Source; job: Job }; + +function buildUploadUrl(subjectId: number, weekNumber: number, useOcr: boolean, pageRanges?: string) { + const params = new URLSearchParams({ use_ocr: String(useOcr) }); + if (pageRanges?.trim()) params.set("page_ranges", pageRanges.trim()); + return `${API_URL}/subjects/${subjectId}/weeks/${weekNumber}/files?${params.toString()}`; +} + +// fetch no expone progreso de subida; XHR si (xhr.upload.onprogress). +export function uploadWithProgress( + subjectId: number, + weekNumber: number, + file: File, + useOcr: boolean, + onProgress: (progress: UploadProgress) => void, + pageRanges?: string, +): Promise { + return new Promise((resolve, reject) => { + const form = new FormData(); + form.append("file", file); + const xhr = new XMLHttpRequest(); + xhr.open("POST", buildUploadUrl(subjectId, weekNumber, useOcr, pageRanges)); + xhr.upload.onprogress = (event) => { + if (event.lengthComputable && event.total > 0) onProgress({ loaded: event.loaded, total: event.total }); + }; + xhr.onload = () => { + let body: { detail?: string } | null = null; + try { body = JSON.parse(xhr.responseText || "null"); } catch { body = null; } + if (xhr.status >= 200 && xhr.status < 300 && body) resolve(body as UploadResult); + else reject(new Error(body?.detail || `Error ${xhr.status}`)); + }; + xhr.onerror = () => reject(new Error("Error de red al subir el archivo")); + xhr.onabort = () => reject(new Error("Subida cancelada")); + xhr.send(form); + }); +} + export const api = { async health() { return parseResponse<{ status: string }>(await fetch(`${API_URL}/health`, { cache: "no-store" })); @@ -119,14 +157,7 @@ export const api = { return parseResponse<{ source: Source; job: Job }>(await fetch(`${API_URL}/sources/${sourceId}/reprocess?${params.toString()}`, { method: "POST" })); }, async upload(subjectId: number, weekNumber: number, file: File, useOcr: boolean, pageRanges?: string) { - const form = new FormData(); - form.append("file", file); - const params = new URLSearchParams({ use_ocr: String(useOcr) }); - if (pageRanges?.trim()) params.set("page_ranges", pageRanges.trim()); - return parseResponse<{ source: Source; job: Job }>(await fetch(`${API_URL}/subjects/${subjectId}/weeks/${weekNumber}/files?${params.toString()}`, { - method: "POST", - body: form, - })); + return uploadWithProgress(subjectId, weekNumber, file, useOcr, () => {}, pageRanges); }, async markdown(documentId: number) { const response = await fetch(`${API_URL}/documents/${documentId}/markdown`, { cache: "no-store" }); diff --git a/frontend/tailwind.config.ts b/frontend/tailwind.config.ts index 847fe90..e840cc1 100644 --- a/frontend/tailwind.config.ts +++ b/frontend/tailwind.config.ts @@ -10,6 +10,15 @@ const config: Config = { panel: "#111827", brand: "#2563eb", }, + keyframes: { + "bar-shimmer": { + "0%": { transform: "translateX(-120%)" }, + "100%": { transform: "translateX(320%)" }, + }, + }, + animation: { + "bar-shimmer": "bar-shimmer 1.3s ease-in-out infinite", + }, }, }, plugins: [require("@tailwindcss/typography")],