Carga en lote concurrente con progreso visual: pool de 3 subidas simultaneas (PDF/audio/txt/imagenes), progreso real de bytes por archivo via XHR con panel de barras animadas, aislamiento de fallos por archivo y SQLite en modo WAL + busy_timeout para subidas concurrentes sin bloqueos. docker-compose.yaml sincronizado con docker-compose.yml
This commit is contained in:
urieljareth
@@ -38,8 +38,13 @@ class KnowledgeManager:
|
||||
self._init_db()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(self.database_path)
|
||||
conn = sqlite3.connect(self.database_path, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
# WAL + busy_timeout: permite subidas concurrentes y trabajos en
|
||||
# segundo plano sin errores "database is locked".
|
||||
conn.execute("pragma busy_timeout = 30000")
|
||||
conn.execute("pragma journal_mode = wal")
|
||||
conn.execute("pragma synchronous = normal")
|
||||
return conn
|
||||
|
||||
def _init_db(self) -> None:
|
||||
|
||||
@@ -12,6 +12,11 @@ services:
|
||||
DATABASE_PATH: /app/data/app.db
|
||||
MISTRAL_API_KEY: ${MISTRAL_API_KEY:-}
|
||||
MISTRAL_OCR_MODEL: ${MISTRAL_OCR_MODEL:-mistral-ocr-latest}
|
||||
MISTRAL_OCR_BATCH_THRESHOLD: ${MISTRAL_OCR_BATCH_THRESHOLD:-3}
|
||||
MISTRAL_OCR_BATCH_POLL_SECONDS: ${MISTRAL_OCR_BATCH_POLL_SECONDS:-10}
|
||||
MISTRAL_OCR_BATCH_TIMEOUT_SECONDS: ${MISTRAL_OCR_BATCH_TIMEOUT_SECONDS:-1800}
|
||||
MISTRAL_OCR_RETRY_ATTEMPTS: ${MISTRAL_OCR_RETRY_ATTEMPTS:-3}
|
||||
MISTRAL_OCR_RETRY_BACKOFF_BASE: ${MISTRAL_OCR_RETRY_BACKOFF_BASE:-2.0}
|
||||
DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-}
|
||||
NODE_ENV: production
|
||||
PORT: 3000
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import Link from "next/link";
|
||||
import { ChangeEvent, DragEvent, useEffect, useMemo, useState } from "react";
|
||||
import { api, DocumentItem, Job, Source, Subject, Week } from "@/lib/api";
|
||||
import { api, DocumentItem, Job, Source, Subject, Week, uploadWithProgress } from "@/lib/api";
|
||||
import { Shell } from "@/components/Shell";
|
||||
import { Button, Card, ConfirmDialog, StatusBadge } from "@/components/ui";
|
||||
|
||||
@@ -10,6 +10,17 @@ type PendingDelete =
|
||||
| { type: "document"; item: DocumentItem }
|
||||
| { type: "source"; item: Source };
|
||||
|
||||
type UploadItem = {
|
||||
id: string;
|
||||
name: string;
|
||||
size: number;
|
||||
progress: number;
|
||||
status: "uploading" | "done" | "error";
|
||||
error?: string;
|
||||
};
|
||||
|
||||
const UPLOAD_CONCURRENCY = 3;
|
||||
|
||||
export default function WeekPage({ params }: { params: { subjectId: string; weekNumber: string } }) {
|
||||
const subjectId = Number(params.subjectId);
|
||||
const weekNumber = Number(params.weekNumber);
|
||||
@@ -23,6 +34,7 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
||||
const [pageRanges, setPageRanges] = useState("");
|
||||
const [pendingFiles, setPendingFiles] = useState<File[]>([]);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const [uploadItems, setUploadItems] = useState<UploadItem[]>([]);
|
||||
const [deletingId, setDeletingId] = useState<string | null>(null);
|
||||
const [reprocessingId, setReprocessingId] = useState<number | null>(null);
|
||||
const [pendingDelete, setPendingDelete] = useState<PendingDelete | null>(null);
|
||||
@@ -31,6 +43,15 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
||||
|
||||
const activeJobs = useMemo(() => jobs.some((job) => ["queued", "processing"].includes(job.status)), [jobs]);
|
||||
const pendingPdfCount = useMemo(() => pendingFiles.filter(isPdfFile).length, [pendingFiles]);
|
||||
const uploadSummary = useMemo(() => {
|
||||
if (!uploadItems.length) return null;
|
||||
const finished = uploadItems.filter((item) => item.status !== "uploading").length;
|
||||
const failed = uploadItems.filter((item) => item.status === "error").length;
|
||||
const percent = Math.round(
|
||||
uploadItems.reduce((acc, item) => acc + (item.status === "done" ? 100 : item.progress), 0) / uploadItems.length,
|
||||
);
|
||||
return { finished, failed, total: uploadItems.length, percent };
|
||||
}, [uploadItems]);
|
||||
|
||||
async function load() {
|
||||
const [subjectData, weekData, sourceList, jobList, documentList] = await Promise.all([
|
||||
@@ -49,16 +70,63 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
||||
async function uploadFileList(files: File[]) {
|
||||
if (!files.length) return;
|
||||
setUploading(true); setError("");
|
||||
const ranges = pageMode === "ranges" ? pageRanges : undefined;
|
||||
const items: UploadItem[] = files.map((file, index) => ({
|
||||
id: `${Date.now()}-${index}-${file.name}`,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
progress: 0,
|
||||
status: "uploading",
|
||||
}));
|
||||
setUploadItems(items);
|
||||
const patchItem = (id: string, patch: Partial<UploadItem>) => {
|
||||
setUploadItems((prev) => prev.map((item) => (item.id === id ? { ...item, ...patch } : item)));
|
||||
};
|
||||
// Pool de subida: hasta UPLOAD_CONCURRENCY archivos en vuelo a la vez,
|
||||
// y un archivo fallido no interrumpe el resto de la tanda.
|
||||
const failures: string[] = [];
|
||||
let cursor = 0;
|
||||
async function runUploadQueue() {
|
||||
while (cursor < files.length) {
|
||||
const index = cursor++;
|
||||
const file = files[index];
|
||||
const item = items[index];
|
||||
try {
|
||||
await uploadWithProgress(
|
||||
subjectId,
|
||||
weekNumber,
|
||||
file,
|
||||
isImageFile(file) || pdfMethod === "mistral_ocr",
|
||||
(progress) => patchItem(item.id, { progress: Math.round((progress.loaded / progress.total) * 100) }),
|
||||
ranges,
|
||||
);
|
||||
patchItem(item.id, { status: "done", progress: 100 });
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "No se pudo subir archivo";
|
||||
failures.push(`${file.name}: ${message}`);
|
||||
patchItem(item.id, { status: "error", error: message });
|
||||
}
|
||||
}
|
||||
}
|
||||
const loadErrors: string[] = [];
|
||||
try {
|
||||
const ranges = pageMode === "ranges" ? pageRanges : undefined;
|
||||
for (const file of files) await api.upload(subjectId, weekNumber, file, isImageFile(file) || pdfMethod === "mistral_ocr", ranges);
|
||||
await Promise.all(Array.from({ length: Math.min(UPLOAD_CONCURRENCY, files.length) }, () => runUploadQueue()));
|
||||
await load();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "No se pudo subir archivo");
|
||||
loadErrors.push(err instanceof Error ? err.message : "No se pudo completar la subida");
|
||||
} finally {
|
||||
setUploading(false);
|
||||
setDragActive(false);
|
||||
}
|
||||
const problems = [...failures, ...loadErrors];
|
||||
if (problems.length) {
|
||||
setError(problems.join("\n"));
|
||||
} else {
|
||||
// Ocultar el panel unos segundos despues de terminar si no hay fallos.
|
||||
window.setTimeout(() => {
|
||||
setUploadItems((prev) => (prev.some((item) => item.status === "uploading") ? prev : []));
|
||||
}, 6000);
|
||||
}
|
||||
}
|
||||
|
||||
function isPdfFile(file: File) {
|
||||
@@ -211,8 +279,92 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
||||
<p className="mt-4 font-medium">Arrastra archivos aqui</p>
|
||||
<p className="mt-2 text-sm text-slate-400">Puedes soltar varios archivos a la vez. Se procesaran en esta semana.</p>
|
||||
<p className="mt-3 text-xs text-slate-500">Soporta PDF, DOCX, TXT, Markdown, imagenes (OCR automatico), audio y video.</p>
|
||||
{uploading && <p className="mt-4 text-sm text-blue-300">Subiendo archivos...</p>}
|
||||
{uploading && (
|
||||
<p className="mt-4 flex items-center justify-center gap-2 text-sm text-blue-300">
|
||||
<span className="h-4 w-4 animate-spin rounded-full border-2 border-blue-400 border-t-transparent" aria-hidden="true" />
|
||||
Subiendo archivos...
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
{uploadItems.length > 0 && uploadSummary && (
|
||||
<div className="mt-5 rounded-2xl border border-blue-400/30 bg-slate-950/60 p-4">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<p className="flex flex-wrap items-center gap-2 text-sm font-medium">
|
||||
{uploading ? (
|
||||
<>
|
||||
<span className="h-4 w-4 animate-spin rounded-full border-2 border-blue-400 border-t-transparent" aria-hidden="true" />
|
||||
<span>Subiendo archivos · {uploadSummary.finished}/{uploadSummary.total} completados</span>
|
||||
</>
|
||||
) : (
|
||||
<span className={uploadSummary.failed ? "text-amber-300" : "text-emerald-300"}>
|
||||
Subida terminada: {uploadSummary.total - uploadSummary.failed} de {uploadSummary.total} correctos
|
||||
</span>
|
||||
)}
|
||||
<span className="text-slate-400">{uploadSummary.percent}%</span>
|
||||
</p>
|
||||
{!uploading && (
|
||||
<button
|
||||
type="button"
|
||||
className="rounded-lg border border-white/10 px-2.5 py-1 text-xs text-slate-300 hover:bg-white/10"
|
||||
onClick={() => setUploadItems([])}
|
||||
>
|
||||
Ocultar
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div
|
||||
className="mt-3 h-2.5 w-full overflow-hidden rounded-full bg-white/10"
|
||||
role="progressbar"
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-valuenow={uploadSummary.percent}
|
||||
aria-label="Progreso general de subida"
|
||||
>
|
||||
<div
|
||||
className={`h-full rounded-full transition-all duration-300 ease-out ${
|
||||
uploading
|
||||
? "bg-gradient-to-r from-blue-500 to-sky-400"
|
||||
: uploadSummary.failed
|
||||
? "bg-amber-500"
|
||||
: "bg-emerald-500"
|
||||
}`}
|
||||
style={{ width: `${uploadSummary.percent}%` }}
|
||||
/>
|
||||
</div>
|
||||
<ul className="mt-4 space-y-3">
|
||||
{uploadItems.map((item) => (
|
||||
<li className="text-sm" key={item.id}>
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<span className="flex min-w-0 items-center gap-2">
|
||||
{item.status === "uploading" && (
|
||||
<span className="h-3.5 w-3.5 shrink-0 animate-spin rounded-full border-2 border-blue-400 border-t-transparent" aria-hidden="true" />
|
||||
)}
|
||||
{item.status === "done" && <span className="shrink-0 text-emerald-400" aria-hidden="true">✓</span>}
|
||||
{item.status === "error" && <span className="shrink-0 font-bold text-red-400" aria-hidden="true">✕</span>}
|
||||
<span className="truncate">{item.name}</span>
|
||||
<span className="shrink-0 text-xs text-slate-500">{formatFileSize(item.size)}</span>
|
||||
</span>
|
||||
<span className={`shrink-0 text-xs ${item.status === "error" ? "text-red-300" : item.status === "done" ? "text-emerald-300" : "text-blue-300"}`}>
|
||||
{item.status === "error" ? "fallo" : `${Math.round(item.progress)}%`}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-1.5 h-1.5 w-full overflow-hidden rounded-full bg-white/10">
|
||||
{item.status === "uploading" ? (
|
||||
<div className="relative h-full rounded-full bg-blue-500 transition-all duration-200 ease-out" style={{ width: `${Math.max(item.progress, 4)}%` }}>
|
||||
<div className="absolute inset-0 overflow-hidden rounded-full">
|
||||
<div className="h-full w-1/3 animate-bar-shimmer rounded-full bg-white/50" />
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className={`h-full rounded-full ${item.status === "done" ? "bg-emerald-500" : "bg-red-500"}`} style={{ width: "100%" }} />
|
||||
)}
|
||||
</div>
|
||||
{item.error && <p className="mt-1 text-xs text-red-300 [overflow-wrap:anywhere]">{item.error}</p>}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
{pendingFiles.length > 0 && (
|
||||
<div className="mt-5 rounded-2xl border border-blue-400/40 bg-blue-500/10 p-4">
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
|
||||
+39
-8
@@ -45,6 +45,44 @@ async function parseResponse<T>(response: Response): Promise<T> {
|
||||
return response.json() as Promise<T>;
|
||||
}
|
||||
|
||||
export type UploadProgress = { loaded: number; total: number };
|
||||
export type UploadResult = { source: Source; job: Job };
|
||||
|
||||
function buildUploadUrl(subjectId: number, weekNumber: number, useOcr: boolean, pageRanges?: string) {
|
||||
const params = new URLSearchParams({ use_ocr: String(useOcr) });
|
||||
if (pageRanges?.trim()) params.set("page_ranges", pageRanges.trim());
|
||||
return `${API_URL}/subjects/${subjectId}/weeks/${weekNumber}/files?${params.toString()}`;
|
||||
}
|
||||
|
||||
// fetch no expone progreso de subida; XHR si (xhr.upload.onprogress).
|
||||
export function uploadWithProgress(
|
||||
subjectId: number,
|
||||
weekNumber: number,
|
||||
file: File,
|
||||
useOcr: boolean,
|
||||
onProgress: (progress: UploadProgress) => void,
|
||||
pageRanges?: string,
|
||||
): Promise<UploadResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("POST", buildUploadUrl(subjectId, weekNumber, useOcr, pageRanges));
|
||||
xhr.upload.onprogress = (event) => {
|
||||
if (event.lengthComputable && event.total > 0) onProgress({ loaded: event.loaded, total: event.total });
|
||||
};
|
||||
xhr.onload = () => {
|
||||
let body: { detail?: string } | null = null;
|
||||
try { body = JSON.parse(xhr.responseText || "null"); } catch { body = null; }
|
||||
if (xhr.status >= 200 && xhr.status < 300 && body) resolve(body as UploadResult);
|
||||
else reject(new Error(body?.detail || `Error ${xhr.status}`));
|
||||
};
|
||||
xhr.onerror = () => reject(new Error("Error de red al subir el archivo"));
|
||||
xhr.onabort = () => reject(new Error("Subida cancelada"));
|
||||
xhr.send(form);
|
||||
});
|
||||
}
|
||||
|
||||
export const api = {
|
||||
async health() {
|
||||
return parseResponse<{ status: string }>(await fetch(`${API_URL}/health`, { cache: "no-store" }));
|
||||
@@ -119,14 +157,7 @@ export const api = {
|
||||
return parseResponse<{ source: Source; job: Job }>(await fetch(`${API_URL}/sources/${sourceId}/reprocess?${params.toString()}`, { method: "POST" }));
|
||||
},
|
||||
async upload(subjectId: number, weekNumber: number, file: File, useOcr: boolean, pageRanges?: string) {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
const params = new URLSearchParams({ use_ocr: String(useOcr) });
|
||||
if (pageRanges?.trim()) params.set("page_ranges", pageRanges.trim());
|
||||
return parseResponse<{ source: Source; job: Job }>(await fetch(`${API_URL}/subjects/${subjectId}/weeks/${weekNumber}/files?${params.toString()}`, {
|
||||
method: "POST",
|
||||
body: form,
|
||||
}));
|
||||
return uploadWithProgress(subjectId, weekNumber, file, useOcr, () => {}, pageRanges);
|
||||
},
|
||||
async markdown(documentId: number) {
|
||||
const response = await fetch(`${API_URL}/documents/${documentId}/markdown`, { cache: "no-store" });
|
||||
|
||||
@@ -10,6 +10,15 @@ const config: Config = {
|
||||
panel: "#111827",
|
||||
brand: "#2563eb",
|
||||
},
|
||||
keyframes: {
|
||||
"bar-shimmer": {
|
||||
"0%": { transform: "translateX(-120%)" },
|
||||
"100%": { transform: "translateX(320%)" },
|
||||
},
|
||||
},
|
||||
animation: {
|
||||
"bar-shimmer": "bar-shimmer 1.3s ease-in-out infinite",
|
||||
},
|
||||
},
|
||||
},
|
||||
plugins: [require("@tailwindcss/typography")],
|
||||
|
||||
Reference in New Issue
Block a user