Carga en lote concurrente con progreso visual: pool de 3 subidas simultaneas (PDF/audio/txt/imagenes), progreso real de bytes por archivo via XHR con panel de barras animadas, aislamiento de fallos por archivo y SQLite en modo WAL + busy_timeout para subidas concurrentes sin bloqueos. docker-compose.yaml sincronizado con docker-compose.yml
This commit is contained in:
@@ -38,8 +38,13 @@ class KnowledgeManager:
|
|||||||
self._init_db()
|
self._init_db()
|
||||||
|
|
||||||
def _connect(self) -> sqlite3.Connection:
|
def _connect(self) -> sqlite3.Connection:
|
||||||
conn = sqlite3.connect(self.database_path)
|
conn = sqlite3.connect(self.database_path, timeout=30)
|
||||||
conn.row_factory = sqlite3.Row
|
conn.row_factory = sqlite3.Row
|
||||||
|
# WAL + busy_timeout: permite subidas concurrentes y trabajos en
|
||||||
|
# segundo plano sin errores "database is locked".
|
||||||
|
conn.execute("pragma busy_timeout = 30000")
|
||||||
|
conn.execute("pragma journal_mode = wal")
|
||||||
|
conn.execute("pragma synchronous = normal")
|
||||||
return conn
|
return conn
|
||||||
|
|
||||||
def _init_db(self) -> None:
|
def _init_db(self) -> None:
|
||||||
|
|||||||
@@ -12,6 +12,11 @@ services:
|
|||||||
DATABASE_PATH: /app/data/app.db
|
DATABASE_PATH: /app/data/app.db
|
||||||
MISTRAL_API_KEY: ${MISTRAL_API_KEY:-}
|
MISTRAL_API_KEY: ${MISTRAL_API_KEY:-}
|
||||||
MISTRAL_OCR_MODEL: ${MISTRAL_OCR_MODEL:-mistral-ocr-latest}
|
MISTRAL_OCR_MODEL: ${MISTRAL_OCR_MODEL:-mistral-ocr-latest}
|
||||||
|
MISTRAL_OCR_BATCH_THRESHOLD: ${MISTRAL_OCR_BATCH_THRESHOLD:-3}
|
||||||
|
MISTRAL_OCR_BATCH_POLL_SECONDS: ${MISTRAL_OCR_BATCH_POLL_SECONDS:-10}
|
||||||
|
MISTRAL_OCR_BATCH_TIMEOUT_SECONDS: ${MISTRAL_OCR_BATCH_TIMEOUT_SECONDS:-1800}
|
||||||
|
MISTRAL_OCR_RETRY_ATTEMPTS: ${MISTRAL_OCR_RETRY_ATTEMPTS:-3}
|
||||||
|
MISTRAL_OCR_RETRY_BACKOFF_BASE: ${MISTRAL_OCR_RETRY_BACKOFF_BASE:-2.0}
|
||||||
DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-}
|
DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-}
|
||||||
NODE_ENV: production
|
NODE_ENV: production
|
||||||
PORT: 3000
|
PORT: 3000
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
import Link from "next/link";
|
import Link from "next/link";
|
||||||
import { ChangeEvent, DragEvent, useEffect, useMemo, useState } from "react";
|
import { ChangeEvent, DragEvent, useEffect, useMemo, useState } from "react";
|
||||||
import { api, DocumentItem, Job, Source, Subject, Week } from "@/lib/api";
|
import { api, DocumentItem, Job, Source, Subject, Week, uploadWithProgress } from "@/lib/api";
|
||||||
import { Shell } from "@/components/Shell";
|
import { Shell } from "@/components/Shell";
|
||||||
import { Button, Card, ConfirmDialog, StatusBadge } from "@/components/ui";
|
import { Button, Card, ConfirmDialog, StatusBadge } from "@/components/ui";
|
||||||
|
|
||||||
@@ -10,6 +10,17 @@ type PendingDelete =
|
|||||||
| { type: "document"; item: DocumentItem }
|
| { type: "document"; item: DocumentItem }
|
||||||
| { type: "source"; item: Source };
|
| { type: "source"; item: Source };
|
||||||
|
|
||||||
|
type UploadItem = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
size: number;
|
||||||
|
progress: number;
|
||||||
|
status: "uploading" | "done" | "error";
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const UPLOAD_CONCURRENCY = 3;
|
||||||
|
|
||||||
export default function WeekPage({ params }: { params: { subjectId: string; weekNumber: string } }) {
|
export default function WeekPage({ params }: { params: { subjectId: string; weekNumber: string } }) {
|
||||||
const subjectId = Number(params.subjectId);
|
const subjectId = Number(params.subjectId);
|
||||||
const weekNumber = Number(params.weekNumber);
|
const weekNumber = Number(params.weekNumber);
|
||||||
@@ -23,6 +34,7 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
|||||||
const [pageRanges, setPageRanges] = useState("");
|
const [pageRanges, setPageRanges] = useState("");
|
||||||
const [pendingFiles, setPendingFiles] = useState<File[]>([]);
|
const [pendingFiles, setPendingFiles] = useState<File[]>([]);
|
||||||
const [uploading, setUploading] = useState(false);
|
const [uploading, setUploading] = useState(false);
|
||||||
|
const [uploadItems, setUploadItems] = useState<UploadItem[]>([]);
|
||||||
const [deletingId, setDeletingId] = useState<string | null>(null);
|
const [deletingId, setDeletingId] = useState<string | null>(null);
|
||||||
const [reprocessingId, setReprocessingId] = useState<number | null>(null);
|
const [reprocessingId, setReprocessingId] = useState<number | null>(null);
|
||||||
const [pendingDelete, setPendingDelete] = useState<PendingDelete | null>(null);
|
const [pendingDelete, setPendingDelete] = useState<PendingDelete | null>(null);
|
||||||
@@ -31,6 +43,15 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
|||||||
|
|
||||||
const activeJobs = useMemo(() => jobs.some((job) => ["queued", "processing"].includes(job.status)), [jobs]);
|
const activeJobs = useMemo(() => jobs.some((job) => ["queued", "processing"].includes(job.status)), [jobs]);
|
||||||
const pendingPdfCount = useMemo(() => pendingFiles.filter(isPdfFile).length, [pendingFiles]);
|
const pendingPdfCount = useMemo(() => pendingFiles.filter(isPdfFile).length, [pendingFiles]);
|
||||||
|
const uploadSummary = useMemo(() => {
|
||||||
|
if (!uploadItems.length) return null;
|
||||||
|
const finished = uploadItems.filter((item) => item.status !== "uploading").length;
|
||||||
|
const failed = uploadItems.filter((item) => item.status === "error").length;
|
||||||
|
const percent = Math.round(
|
||||||
|
uploadItems.reduce((acc, item) => acc + (item.status === "done" ? 100 : item.progress), 0) / uploadItems.length,
|
||||||
|
);
|
||||||
|
return { finished, failed, total: uploadItems.length, percent };
|
||||||
|
}, [uploadItems]);
|
||||||
|
|
||||||
async function load() {
|
async function load() {
|
||||||
const [subjectData, weekData, sourceList, jobList, documentList] = await Promise.all([
|
const [subjectData, weekData, sourceList, jobList, documentList] = await Promise.all([
|
||||||
@@ -49,16 +70,63 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
|||||||
async function uploadFileList(files: File[]) {
|
async function uploadFileList(files: File[]) {
|
||||||
if (!files.length) return;
|
if (!files.length) return;
|
||||||
setUploading(true); setError("");
|
setUploading(true); setError("");
|
||||||
|
const ranges = pageMode === "ranges" ? pageRanges : undefined;
|
||||||
|
const items: UploadItem[] = files.map((file, index) => ({
|
||||||
|
id: `${Date.now()}-${index}-${file.name}`,
|
||||||
|
name: file.name,
|
||||||
|
size: file.size,
|
||||||
|
progress: 0,
|
||||||
|
status: "uploading",
|
||||||
|
}));
|
||||||
|
setUploadItems(items);
|
||||||
|
const patchItem = (id: string, patch: Partial<UploadItem>) => {
|
||||||
|
setUploadItems((prev) => prev.map((item) => (item.id === id ? { ...item, ...patch } : item)));
|
||||||
|
};
|
||||||
|
// Pool de subida: hasta UPLOAD_CONCURRENCY archivos en vuelo a la vez,
|
||||||
|
// y un archivo fallido no interrumpe el resto de la tanda.
|
||||||
|
const failures: string[] = [];
|
||||||
|
let cursor = 0;
|
||||||
|
async function runUploadQueue() {
|
||||||
|
while (cursor < files.length) {
|
||||||
|
const index = cursor++;
|
||||||
|
const file = files[index];
|
||||||
|
const item = items[index];
|
||||||
|
try {
|
||||||
|
await uploadWithProgress(
|
||||||
|
subjectId,
|
||||||
|
weekNumber,
|
||||||
|
file,
|
||||||
|
isImageFile(file) || pdfMethod === "mistral_ocr",
|
||||||
|
(progress) => patchItem(item.id, { progress: Math.round((progress.loaded / progress.total) * 100) }),
|
||||||
|
ranges,
|
||||||
|
);
|
||||||
|
patchItem(item.id, { status: "done", progress: 100 });
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : "No se pudo subir archivo";
|
||||||
|
failures.push(`${file.name}: ${message}`);
|
||||||
|
patchItem(item.id, { status: "error", error: message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const loadErrors: string[] = [];
|
||||||
try {
|
try {
|
||||||
const ranges = pageMode === "ranges" ? pageRanges : undefined;
|
await Promise.all(Array.from({ length: Math.min(UPLOAD_CONCURRENCY, files.length) }, () => runUploadQueue()));
|
||||||
for (const file of files) await api.upload(subjectId, weekNumber, file, isImageFile(file) || pdfMethod === "mistral_ocr", ranges);
|
|
||||||
await load();
|
await load();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err instanceof Error ? err.message : "No se pudo subir archivo");
|
loadErrors.push(err instanceof Error ? err.message : "No se pudo completar la subida");
|
||||||
} finally {
|
} finally {
|
||||||
setUploading(false);
|
setUploading(false);
|
||||||
setDragActive(false);
|
setDragActive(false);
|
||||||
}
|
}
|
||||||
|
const problems = [...failures, ...loadErrors];
|
||||||
|
if (problems.length) {
|
||||||
|
setError(problems.join("\n"));
|
||||||
|
} else {
|
||||||
|
// Ocultar el panel unos segundos despues de terminar si no hay fallos.
|
||||||
|
window.setTimeout(() => {
|
||||||
|
setUploadItems((prev) => (prev.some((item) => item.status === "uploading") ? prev : []));
|
||||||
|
}, 6000);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function isPdfFile(file: File) {
|
function isPdfFile(file: File) {
|
||||||
@@ -211,8 +279,92 @@ export default function WeekPage({ params }: { params: { subjectId: string; week
|
|||||||
<p className="mt-4 font-medium">Arrastra archivos aqui</p>
|
<p className="mt-4 font-medium">Arrastra archivos aqui</p>
|
||||||
<p className="mt-2 text-sm text-slate-400">Puedes soltar varios archivos a la vez. Se procesaran en esta semana.</p>
|
<p className="mt-2 text-sm text-slate-400">Puedes soltar varios archivos a la vez. Se procesaran en esta semana.</p>
|
||||||
<p className="mt-3 text-xs text-slate-500">Soporta PDF, DOCX, TXT, Markdown, imagenes (OCR automatico), audio y video.</p>
|
<p className="mt-3 text-xs text-slate-500">Soporta PDF, DOCX, TXT, Markdown, imagenes (OCR automatico), audio y video.</p>
|
||||||
{uploading && <p className="mt-4 text-sm text-blue-300">Subiendo archivos...</p>}
|
{uploading && (
|
||||||
|
<p className="mt-4 flex items-center justify-center gap-2 text-sm text-blue-300">
|
||||||
|
<span className="h-4 w-4 animate-spin rounded-full border-2 border-blue-400 border-t-transparent" aria-hidden="true" />
|
||||||
|
Subiendo archivos...
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
{uploadItems.length > 0 && uploadSummary && (
|
||||||
|
<div className="mt-5 rounded-2xl border border-blue-400/30 bg-slate-950/60 p-4">
|
||||||
|
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||||
|
<p className="flex flex-wrap items-center gap-2 text-sm font-medium">
|
||||||
|
{uploading ? (
|
||||||
|
<>
|
||||||
|
<span className="h-4 w-4 animate-spin rounded-full border-2 border-blue-400 border-t-transparent" aria-hidden="true" />
|
||||||
|
<span>Subiendo archivos · {uploadSummary.finished}/{uploadSummary.total} completados</span>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<span className={uploadSummary.failed ? "text-amber-300" : "text-emerald-300"}>
|
||||||
|
Subida terminada: {uploadSummary.total - uploadSummary.failed} de {uploadSummary.total} correctos
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
<span className="text-slate-400">{uploadSummary.percent}%</span>
|
||||||
|
</p>
|
||||||
|
{!uploading && (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="rounded-lg border border-white/10 px-2.5 py-1 text-xs text-slate-300 hover:bg-white/10"
|
||||||
|
onClick={() => setUploadItems([])}
|
||||||
|
>
|
||||||
|
Ocultar
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
className="mt-3 h-2.5 w-full overflow-hidden rounded-full bg-white/10"
|
||||||
|
role="progressbar"
|
||||||
|
aria-valuemin={0}
|
||||||
|
aria-valuemax={100}
|
||||||
|
aria-valuenow={uploadSummary.percent}
|
||||||
|
aria-label="Progreso general de subida"
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className={`h-full rounded-full transition-all duration-300 ease-out ${
|
||||||
|
uploading
|
||||||
|
? "bg-gradient-to-r from-blue-500 to-sky-400"
|
||||||
|
: uploadSummary.failed
|
||||||
|
? "bg-amber-500"
|
||||||
|
: "bg-emerald-500"
|
||||||
|
}`}
|
||||||
|
style={{ width: `${uploadSummary.percent}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<ul className="mt-4 space-y-3">
|
||||||
|
{uploadItems.map((item) => (
|
||||||
|
<li className="text-sm" key={item.id}>
|
||||||
|
<div className="flex items-center justify-between gap-3">
|
||||||
|
<span className="flex min-w-0 items-center gap-2">
|
||||||
|
{item.status === "uploading" && (
|
||||||
|
<span className="h-3.5 w-3.5 shrink-0 animate-spin rounded-full border-2 border-blue-400 border-t-transparent" aria-hidden="true" />
|
||||||
|
)}
|
||||||
|
{item.status === "done" && <span className="shrink-0 text-emerald-400" aria-hidden="true">✓</span>}
|
||||||
|
{item.status === "error" && <span className="shrink-0 font-bold text-red-400" aria-hidden="true">✕</span>}
|
||||||
|
<span className="truncate">{item.name}</span>
|
||||||
|
<span className="shrink-0 text-xs text-slate-500">{formatFileSize(item.size)}</span>
|
||||||
|
</span>
|
||||||
|
<span className={`shrink-0 text-xs ${item.status === "error" ? "text-red-300" : item.status === "done" ? "text-emerald-300" : "text-blue-300"}`}>
|
||||||
|
{item.status === "error" ? "fallo" : `${Math.round(item.progress)}%`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div className="mt-1.5 h-1.5 w-full overflow-hidden rounded-full bg-white/10">
|
||||||
|
{item.status === "uploading" ? (
|
||||||
|
<div className="relative h-full rounded-full bg-blue-500 transition-all duration-200 ease-out" style={{ width: `${Math.max(item.progress, 4)}%` }}>
|
||||||
|
<div className="absolute inset-0 overflow-hidden rounded-full">
|
||||||
|
<div className="h-full w-1/3 animate-bar-shimmer rounded-full bg-white/50" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className={`h-full rounded-full ${item.status === "done" ? "bg-emerald-500" : "bg-red-500"}`} style={{ width: "100%" }} />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
{item.error && <p className="mt-1 text-xs text-red-300 [overflow-wrap:anywhere]">{item.error}</p>}
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
{pendingFiles.length > 0 && (
|
{pendingFiles.length > 0 && (
|
||||||
<div className="mt-5 rounded-2xl border border-blue-400/40 bg-blue-500/10 p-4">
|
<div className="mt-5 rounded-2xl border border-blue-400/40 bg-blue-500/10 p-4">
|
||||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||||
|
|||||||
+39
-8
@@ -45,6 +45,44 @@ async function parseResponse<T>(response: Response): Promise<T> {
|
|||||||
return response.json() as Promise<T>;
|
return response.json() as Promise<T>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type UploadProgress = { loaded: number; total: number };
|
||||||
|
export type UploadResult = { source: Source; job: Job };
|
||||||
|
|
||||||
|
function buildUploadUrl(subjectId: number, weekNumber: number, useOcr: boolean, pageRanges?: string) {
|
||||||
|
const params = new URLSearchParams({ use_ocr: String(useOcr) });
|
||||||
|
if (pageRanges?.trim()) params.set("page_ranges", pageRanges.trim());
|
||||||
|
return `${API_URL}/subjects/${subjectId}/weeks/${weekNumber}/files?${params.toString()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetch no expone progreso de subida; XHR si (xhr.upload.onprogress).
|
||||||
|
export function uploadWithProgress(
|
||||||
|
subjectId: number,
|
||||||
|
weekNumber: number,
|
||||||
|
file: File,
|
||||||
|
useOcr: boolean,
|
||||||
|
onProgress: (progress: UploadProgress) => void,
|
||||||
|
pageRanges?: string,
|
||||||
|
): Promise<UploadResult> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const form = new FormData();
|
||||||
|
form.append("file", file);
|
||||||
|
const xhr = new XMLHttpRequest();
|
||||||
|
xhr.open("POST", buildUploadUrl(subjectId, weekNumber, useOcr, pageRanges));
|
||||||
|
xhr.upload.onprogress = (event) => {
|
||||||
|
if (event.lengthComputable && event.total > 0) onProgress({ loaded: event.loaded, total: event.total });
|
||||||
|
};
|
||||||
|
xhr.onload = () => {
|
||||||
|
let body: { detail?: string } | null = null;
|
||||||
|
try { body = JSON.parse(xhr.responseText || "null"); } catch { body = null; }
|
||||||
|
if (xhr.status >= 200 && xhr.status < 300 && body) resolve(body as UploadResult);
|
||||||
|
else reject(new Error(body?.detail || `Error ${xhr.status}`));
|
||||||
|
};
|
||||||
|
xhr.onerror = () => reject(new Error("Error de red al subir el archivo"));
|
||||||
|
xhr.onabort = () => reject(new Error("Subida cancelada"));
|
||||||
|
xhr.send(form);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export const api = {
|
export const api = {
|
||||||
async health() {
|
async health() {
|
||||||
return parseResponse<{ status: string }>(await fetch(`${API_URL}/health`, { cache: "no-store" }));
|
return parseResponse<{ status: string }>(await fetch(`${API_URL}/health`, { cache: "no-store" }));
|
||||||
@@ -119,14 +157,7 @@ export const api = {
|
|||||||
return parseResponse<{ source: Source; job: Job }>(await fetch(`${API_URL}/sources/${sourceId}/reprocess?${params.toString()}`, { method: "POST" }));
|
return parseResponse<{ source: Source; job: Job }>(await fetch(`${API_URL}/sources/${sourceId}/reprocess?${params.toString()}`, { method: "POST" }));
|
||||||
},
|
},
|
||||||
async upload(subjectId: number, weekNumber: number, file: File, useOcr: boolean, pageRanges?: string) {
|
async upload(subjectId: number, weekNumber: number, file: File, useOcr: boolean, pageRanges?: string) {
|
||||||
const form = new FormData();
|
return uploadWithProgress(subjectId, weekNumber, file, useOcr, () => {}, pageRanges);
|
||||||
form.append("file", file);
|
|
||||||
const params = new URLSearchParams({ use_ocr: String(useOcr) });
|
|
||||||
if (pageRanges?.trim()) params.set("page_ranges", pageRanges.trim());
|
|
||||||
return parseResponse<{ source: Source; job: Job }>(await fetch(`${API_URL}/subjects/${subjectId}/weeks/${weekNumber}/files?${params.toString()}`, {
|
|
||||||
method: "POST",
|
|
||||||
body: form,
|
|
||||||
}));
|
|
||||||
},
|
},
|
||||||
async markdown(documentId: number) {
|
async markdown(documentId: number) {
|
||||||
const response = await fetch(`${API_URL}/documents/${documentId}/markdown`, { cache: "no-store" });
|
const response = await fetch(`${API_URL}/documents/${documentId}/markdown`, { cache: "no-store" });
|
||||||
|
|||||||
@@ -10,6 +10,15 @@ const config: Config = {
|
|||||||
panel: "#111827",
|
panel: "#111827",
|
||||||
brand: "#2563eb",
|
brand: "#2563eb",
|
||||||
},
|
},
|
||||||
|
keyframes: {
|
||||||
|
"bar-shimmer": {
|
||||||
|
"0%": { transform: "translateX(-120%)" },
|
||||||
|
"100%": { transform: "translateX(320%)" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
animation: {
|
||||||
|
"bar-shimmer": "bar-shimmer 1.3s ease-in-out infinite",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
plugins: [require("@tailwindcss/typography")],
|
plugins: [require("@tailwindcss/typography")],
|
||||||
|
|||||||
Reference in New Issue
Block a user