diff --git a/scripts/start-server.ps1 b/scripts/start-server.ps1 index 91239b8..efe40fc 100644 --- a/scripts/start-server.ps1 +++ b/scripts/start-server.ps1 @@ -132,10 +132,18 @@ $proc = $null foreach ($candidate in $candidates) { Remove-Item '.run\server.log' -Force -ErrorAction SilentlyContinue + Remove-Item '.run\server.err.log' -Force -ErrorAction SilentlyContinue Out-Line " [...] Encendiendo el servidor (intento $candidate)..." Gray - $cmdLine = '/c "' + $PythonExe + '" -m uvicorn yt_scraper.webapp.app:app --host 127.0.0.1 --port ' + $candidate + ' --log-level info > ".run\server.log" 2>&1' + # Spawn directo de python: un wrapper cmd /c con la ruta entre comillas + # dispara el quote-stripping de cmd cuando la ruta tiene espacios y el + # comando muere en silencio. Start-Process con array de args no sufre eso, + # y ademas server.info registra el PID real de python (no el del wrapper). try { - $proc = Start-Process -FilePath 'cmd.exe' -ArgumentList $cmdLine -WorkingDirectory $root -WindowStyle Minimized -PassThru + $proc = Start-Process -FilePath $PythonExe ` + -ArgumentList @('-m','uvicorn','yt_scraper.webapp.app:app','--host','127.0.0.1','--port',"$candidate",'--log-level','info') ` + -WorkingDirectory $root -WindowStyle Minimized -PassThru ` + -RedirectStandardOutput "$root\.run\server.log" ` + -RedirectStandardError "$root\.run\server.err.log" } catch { Out-Line " [warn] Start-Process fallo en $candidate : $($_.Exception.Message)" Yellow continue @@ -158,12 +166,14 @@ foreach ($candidate in $candidates) { if (-not $bound) { Out-Line " [warn] Este intento no respondio; probando el siguiente..." Yellow - # Show the actual reason from the log so the user can diagnose. - if (Test-Path '.run\server.log') { - $tail = Get-Content '.run\server.log' -Tail 6 -ErrorAction SilentlyContinue - if ($tail) { - Out-Line " (ultimas lineas del log):" DarkGray - $tail | ForEach-Object { Out-Line " $_" DarkGray } + # Show the actual reason from the logs so the user can diagnose. + foreach ($logName in @('.run\server.err.log', '.run\server.log')) { + if (Test-Path $logName) { + $tail = Get-Content $logName -Tail 6 -ErrorAction SilentlyContinue + if ($tail) { + Out-Line " (ultimas lineas de $logName):" DarkGray + $tail | ForEach-Object { Out-Line " $_" DarkGray } + } } } try { Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue } catch {} diff --git a/src/yt_scraper/webapp/static/app.js b/src/yt_scraper/webapp/static/app.js index 6c08f57..87799df 100644 --- a/src/yt_scraper/webapp/static/app.js +++ b/src/yt_scraper/webapp/static/app.js @@ -34,17 +34,20 @@ health: false, // ----- data stores ----- - dash: { channels: [], status_breakdown: {}, uploads_over_time: [], duration_histogram: [], top_tags: [] }, + dash: { channels: [], status_breakdown: {}, top_tags: [] }, channels: { items: [], pending: {} }, videos: { items: [], total: 0, page: 1, size: 25, selected: [] }, + sidebarOpen: false, detail: { video: null, transcript: [], hasAudio: false, audioPlaying: false, activeSeg: -1 }, search: { q: "", channel: "", items: [], ran: false }, analysis: { tab: "wordcloud", channel: "", term: "", words: [], timeline: [] }, scrape: { - form: { channel_id: "", limit: "", since: "", languages: "", include_shorts: false, no_live: false }, - jobId: null, progress: { completed: 0, total: 0 }, log: [], jobs: [], es: null, done: false, error: null, + form: { channel_id: "", limit: "", since: "", languages: "", include_shorts: false, no_live: false, full: false }, + jobId: null, progress: { completed: 0, total: 0 }, log: [], jobs: [], es: null, done: false, error: null, result: null, kind: "", + // channel_id al que apunta el discovery activo; null = todos los canales. + targetChannel: null, }, cookies: { items: [], drag: false, testResult: {}, uploadMsg: "" }, @@ -54,19 +57,28 @@ // ----- stats panel ----- stats: { open: false, data: null, loading: false, channel: "" }, + // ----- in-app confirm dialog (never use window.confirm) ----- + confirmBox: { open: false, title: "", message: "", confirmLabel: "Confirm", cancelLabel: "Cancel", danger: false, _resolve: null }, + // ----- floating job widget ----- jobWidget: { collapsed: false, dismissed: false, _hideTimer: null }, // ----- tools / folders ----- folders: { markdown: { path: "", exists: false }, exports: { path: "", exists: false }, audio: { path: "", exists: false }, analysis: { path: "", exists: false }, database: { path: "", exists: false } }, - tools: { reRendering: false, msg: "", fmt: null, thumbScope: "all", audioScope: "all", busy: "", watchCmd: "" }, + tools: { reRendering: false, msg: "", fmt: null, thumbScope: "all", audioScope: "all", avatarScope: "all", busy: "", watchCmd: "" }, // ----- forms ----- forms: { channelUrl: "", channelError: "", exportFormat: "json", exportChannel: "" }, filters: { channel: "", status: "", from: "", to: "", min_dur: "", q: "", sort: "upload_date" }, // ----- loading flags ----- - loading: { dashboard: false, channels: false, videos: false, detail: false, transcript: false, search: false, analysis: false, jobs: false, cookies: false, addChannel: false, startScrape: false }, + loading: { dashboard: false, channels: false, videos: false, detail: false, transcript: false, search: false, analysis: false, jobs: false, cookies: false, addChannel: false, startScrape: false, reset: false, reconcile: false }, + + // ----- channel sync ----- + sync: { busy: "", lastResult: "" }, + + // ----- videos stuck in a retryable terminal status ----- + retryable: { counts: {}, total: 0 }, // ----- chart instances ----- charts: {}, @@ -76,9 +88,59 @@ // ================================================================= init() { setupChartDefaults(); + this.sidebarOpen = window.innerWidth >= 768 ? true : localStorage.getItem("sidebar-open") === "true"; + const size = Number(localStorage.getItem("videos-size")); + if ([10, 25, 50, 100].includes(size)) this.videos.size = size; + this.hydrateURL(); this.checkHealth(); this.loadDashboard(); this.loadChannels(); + if (this.view === "videos") this.loadVideos(this.videos.page); + if (this.view === "search" && this.search.q) this.loadSearch(); + this.loadRetryable(); + this.startLivePolling(); + }, + + hydrateURL() { + const p = new URLSearchParams(window.location.search); + const views = ["dashboard", "channels", "videos", "detail", "search", "analysis", "scrape", "cookies", "tools", "export"]; + if (views.includes(p.get("view"))) this.view = p.get("view"); + if (this.view === "videos") { + ["channel", "status", "from", "to", "min_dur", "q", "sort"].forEach(k => { if (p.has(k)) this.filters[k] = p.get(k); }); + const page = Number(p.get("page")); + const size = Number(p.get("size")); + if (page > 0) this.videos.page = page; + if ([10, 25, 50, 100].includes(size)) this.videos.size = size; + } + if (this.view === "search") { + this.search.q = p.get("q") || ""; + this.search.channel = p.get("channel") || ""; + } + }, + + syncURL() { + clearTimeout(this._urlTimer); + this._urlTimer = setTimeout(() => { + const p = new URLSearchParams({ view: this.view }); + if (this.view === "videos") { + Object.entries(this.filters).forEach(([k, v]) => { if (v) p.set(k, v); }); + p.set("page", this.videos.page); p.set("size", this.videos.size); + } else if (this.view === "search") { + if (this.search.q) p.set("q", this.search.q); + if (this.search.channel) p.set("channel", this.search.channel); + } + const next = "?" + p.toString(); + if (next !== window.location.search) history.replaceState(null, "", next); + }, 120); + }, + + toggleSidebar() { + this.sidebarOpen = !this.sidebarOpen; + localStorage.setItem("sidebar-open", String(this.sidebarOpen)); + }, + closeSidebar() { + this.sidebarOpen = false; + localStorage.setItem("sidebar-open", "false"); }, async api(url, opts) { @@ -94,17 +156,38 @@ }, async checkHealth() { - try { const d = await this.api("/healthz"); this.health = d && d.status === "ok"; } - catch (_) { this.health = false; } + this._healthAttempt = (this._healthAttempt || 0) + 1; + // Exponential backoff capped at 30s. + const delays = [1, 1, 2, 2, 4, 4, 8, 15, 30]; + const wait = delays[Math.min(this._healthAttempt - 1, delays.length - 1)] * 1000; + try { + const d = await this.api("/healthz"); + this.health = d && d.status === "ok"; + this.healthFails = 0; + if (!this.health) { + setTimeout(() => this.checkHealth(), wait); + } else { + // healthy: ping every 30s + setTimeout(() => this.checkHealth(), 30000); + } + } catch (_) { + this.health = false; + this.healthFails = (this.healthFails || 0) + 1; + setTimeout(() => this.checkHealth(), wait); + } }, setView(v) { this.view = v; + this.syncURL(); + if (window.innerWidth < 768) this.closeSidebar(); // destroy stray canvases when leaving chart-bearing views - if (v !== "dashboard") this.destroyCharts(["uploads", "duration", "tags"]); if (v !== "analysis") this.destroyCharts(["topwords", "timeline"]); - if (v === "dashboard") this.$nextTick(() => this.renderDashboardCharts()); - if (v === "videos" && this.videos.items.length === 0) this.loadVideos(1); + if (v === "dashboard") this.loadDashboard(); + // Always reload, not only when empty: statuses change underneath us + // (jobs, CLI runs, reconcile) and a cached list would show stale rows. + if (v === "videos") this.loadVideos(); + if (v === "channels") this.loadChannels(); if (v === "scrape") this.loadJobs(); if (v === "cookies") this.loadCookies(); if (v === "analysis") this.loadAnalysis(); @@ -119,33 +202,10 @@ try { const d = await this.api("/api/dashboard"); this.dash = d; - this.$nextTick(() => this.renderDashboardCharts()); } catch (e) { this.toast("Failed to load dashboard: " + e.message, "error"); } finally { this.loading.dashboard = false; } }, - renderDashboardCharts() { - if (!window.Chart) return; - const up = this.dash.uploads_over_time || []; - this.makeChart("uploads", document.getElementById("chart-uploads"), { - type: "line", - data: { labels: up.map(u => this.fmtMonth(u.month)), datasets: [{ label: "uploads", data: up.map(u => u.count), borderColor: "#f43f5e", backgroundColor: "rgba(244,63,94,0.18)", fill: true, tension: 0.35, pointRadius: 2 }] }, - options: this.lineOpts(), - }); - const dh = this.dash.duration_histogram || []; - this.makeChart("duration", document.getElementById("chart-duration"), { - type: "bar", - data: { labels: dh.map(d => d.bucket), datasets: [{ label: "videos", data: dh.map(d => d.count), backgroundColor: "rgba(244,63,94,0.6)", borderRadius: 4 }] }, - options: this.barOpts(), - }); - const tt = (this.dash.top_tags || []).slice(0, 12); - this.makeChart("tags", document.getElementById("chart-tags"), { - type: "bar", - data: { labels: tt.map(t => t.tag), datasets: [{ label: "count", data: tt.map(t => t.count), backgroundColor: "rgba(251,113,133,0.7)", borderRadius: 4 }] }, - options: this.barOpts(true), - }); - }, - // ================================================================= // channels // ================================================================= @@ -166,12 +226,42 @@ finally { this.loading.channels = false; } }, + async syncChannels(channelId) { + const body = channelId ? { channel_id: channelId } : {}; + const isAll = !channelId; + this.sync.busy = isAll ? "all" : channelId; + try { + const d = await this.api("/api/tools/sync-channels", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }); + const summary = "Synced " + (d.synced || 0) + ", avatars cached: " + (d.avatars_cached || 0) + + (d.avatars_recovered_via_deep_fallback ? " (recovered: " + d.avatars_recovered_via_deep_fallback + ")" : "") + + (d.errors && d.errors.length ? ", errors: " + d.errors.length : ""); + this._setSyncResult(summary, d.errors && d.errors.length ? "error" : "success"); + this.toast(summary, d.errors && d.errors.length ? "error" : "success"); + await this.loadChannels(); + await this.loadDashboard(); + } catch (e) { + this._setSyncResult("Sync failed: " + e.message, "error"); + this.toast("Sync failed: " + e.message, "error"); + } finally { + this.sync.busy = ""; + } + }, + _setSyncResult(text, kind) { + this.sync.lastResult = text; + this.sync.lastResultKind = kind || "ok"; + if (this._syncResultTimer) clearTimeout(this._syncResultTimer); + this._syncResultTimer = setTimeout(() => { + this.sync.lastResult = ""; + this.sync.lastResultKind = ""; + }, 8000); + }, + async loadChannelPending(id) { try { const d = await this.api("/api/videos?channel=" + encodeURIComponent(id) + "&status=pending&size=1"); if (!this.channels.pending) this.channels.pending = {}; this.channels.pending[id] = (d && d.total) || 0; - } catch (_) {} + } catch (_) { console.warn("loadChannelPending failed", id, _); } }, async addChannel() { @@ -183,8 +273,11 @@ const d = await this.api("/api/channels", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ url }) }); this.forms.channelUrl = ""; await this.loadChannels(); - this.toast("Added " + (d.name || "channel") + " (" + (d.video_count || 0) + " videos)"); - // Auto-cache thumbnails in the background so the catalog looks populated immediately + let msg = "Added " + (d.name || "channel") + " (" + (d.video_count || 0) + " videos)"; + if (d.avatar && d.avatar_cached) msg += " · avatar cached"; + else if (d.avatar) msg += " · avatar URL set (download pending)"; + else msg += " · avatar not available"; + this.toast(msg); if (d.channel_id) { this.api("/api/tools/thumbnails", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ channel_id: d.channel_id }) }) .then((r) => { if (r && r.downloaded) this.toast((r.downloaded) + " thumbnails cached"); }) @@ -195,7 +288,8 @@ }, async removeChannel(id) { - if (!confirm("Remove this channel and its videos?")) return; + const ok = await this.confirmDialog({ title: "Remove channel", message: "Remove this channel and its videos from the platform?", confirmLabel: "Remove", danger: true }); + if (!ok) return; try { await this.api("/api/channels/" + encodeURIComponent(id), { method: "DELETE" }); await this.loadChannels(); this.loadDashboard(); } catch (e) { this.toast("Remove failed: " + e.message, "error"); } }, @@ -203,13 +297,21 @@ // ================================================================= // videos // ================================================================= + // Sort keys live in the