diff --git a/dashboard/src/lib/stores/app.svelte.ts b/dashboard/src/lib/stores/app.svelte.ts index 1932011c..642d4d00 100644 --- a/dashboard/src/lib/stores/app.svelte.ts +++ b/dashboard/src/lib/stores/app.svelte.ts @@ -8,11 +8,6 @@ */ import { browser } from "$app/environment"; -import { addToast } from "$lib/stores/toast.svelte"; -import { - getDownloadTag, - extractModelIdFromDownload, -} from "$lib/utils/downloads"; // UUID generation fallback for browsers without crypto.randomUUID function generateUUID(): string { @@ -1332,7 +1327,6 @@ class AppStore { this.runners = data.runners; } if (data.downloads) { - this.detectEvictions(this.downloads, data.downloads); this.downloads = data.downloads; } if (data.nodeDisk) { @@ -1461,63 +1455,6 @@ class AppStore { } } - /** - * Detect models that were evicted (completed -> not completed) and show toasts. - */ - private detectEvictions( - oldDownloads: Record, - newDownloads: Record, - ) { - if (Object.keys(oldDownloads).length === 0) return; - - const wasCompleted = new Map>(); - for (const [nodeId, entries] of Object.entries(oldDownloads)) { - if (!Array.isArray(entries)) continue; - for (const entry of entries) { - const tagged = getDownloadTag(entry); - if (!tagged || tagged[0] !== "DownloadCompleted") continue; - const modelId = extractModelIdFromDownload(tagged[1]); - if (!modelId) continue; - if (!wasCompleted.has(nodeId)) wasCompleted.set(nodeId, new Set()); - wasCompleted.get(nodeId)!.add(modelId); - } - } - - for (const [nodeId, modelIds] of wasCompleted) { - const newEntries = newDownloads[nodeId]; - const stillCompleted = new Set(); - if (Array.isArray(newEntries)) { - for (const entry of newEntries) { - const tagged = getDownloadTag(entry); - if (!tagged || tagged[0] !== "DownloadCompleted") continue; - const mid = extractModelIdFromDownload(tagged[1]); - if (mid) stillCompleted.add(mid); - } - } - - for (const modelId of modelIds) { - if (!stillCompleted.has(modelId)) { - const shortName = modelId.split("/").pop() ?? modelId; - const nodeLabel = this.getNodeFriendlyName(nodeId); - addToast({ - type: "info", - message: `Model evicted on ${nodeLabel}: ${shortName}`, - duration: 6000, - }); - } - } - } - } - - private getNodeFriendlyName(nodeId: string): string { - const nodeInfo = this.topologyData?.nodes?.[nodeId]; - return ( - nodeInfo?.friendly_name ?? - nodeInfo?.system_info?.chip ?? - nodeId.slice(0, 4) - ); - } - /** * Handle topology changes - clean up filter and re-fetch if needed */ diff --git a/dashboard/src/routes/+page.svelte b/dashboard/src/routes/+page.svelte index ef05e729..3845adec 100644 --- a/dashboard/src/routes/+page.svelte +++ b/dashboard/src/routes/+page.svelte @@ -1582,6 +1582,7 @@ perNode: NodeDownloadStatus[]; failedError: string | null; rejectedError: string | null; + evicted: boolean; } { const empty = { isDownloading: false, @@ -1589,6 +1590,7 @@ perNode: [] as NodeDownloadStatus[], failedError: null, rejectedError: null, + evicted: false, }; if (!downloadsData || Object.keys(downloadsData).length === 0) { @@ -1631,6 +1633,7 @@ (downloadPayload.error_message as string) || "Download failed", rejectedError: null, + evicted: false, }; } @@ -1643,6 +1646,19 @@ failedError: null, rejectedError: (downloadPayload.reason as string) || "Storage limit exceeded", + evicted: false, + }; + } + + // DownloadEvicted — model was evicted from storage + if (downloadKind === "DownloadEvicted") { + return { + isDownloading: false, + progress: null, + perNode: Array.from(perNodeMap.values()), + failedError: null, + rejectedError: null, + evicted: true, }; } @@ -1738,6 +1754,7 @@ perNode, failedError: null, rejectedError: null, + evicted: false, }; } @@ -1759,6 +1776,7 @@ perNode, failedError: null, rejectedError: null, + evicted: false, }; } @@ -1854,6 +1872,17 @@ }; } + if (result.evicted) { + return { + isDownloading: false, + isFailed: false, + errorMessage: null, + progress: null, + statusText: "EVICTED", + perNode: [], + }; + } + if (!result.isDownloading) { const statusInfo = deriveInstanceStatus(instanceWrapped); return { @@ -2489,7 +2518,6 @@ // ── Instance status transition toasts ── // Track previous statuses so we can detect meaningful transitions and fire toasts. let previousInstanceStatuses: Record = {}; - let previousInstanceModelIds: Record = {}; $effect(() => { const currentStatuses: Record = {}; @@ -2556,34 +2584,19 @@ if (prevStatus !== "SHUTDOWN" && currentStatus === "SHUTDOWN") { addToast({ type: "info", message: `Model shut down: ${shortName}` }); } - } - } - // Detect instances that disappeared while in early states (e.g. rejected download) - if (Object.keys(prev).length > 0) { - for (const [id, prevStatus] of Object.entries(prev)) { - if (id in currentStatuses) continue; // still exists - if (prevStatus === "PREPARING" || prevStatus === "DOWNLOADING") { - const modelId = previousInstanceModelIds[id]; - const shortName = modelId - ? (modelId.split("/").pop() ?? modelId) - : id.slice(0, 8); + // Any -> Evicted + if (prevStatus !== "EVICTED" && currentStatus === "EVICTED") { addToast({ - type: "warning", - message: `Download cancelled: ${shortName} — insufficient storage`, - duration: 8000, + type: "info", + message: `Model evicted: ${shortName}`, + duration: 6000, }); } } } previousInstanceStatuses = currentStatuses; - const modelIds: Record = {}; - for (const [id, inst] of Object.entries(instanceData)) { - const mid = getInstanceModelId(inst); - if (mid) modelIds[id] = mid; - } - previousInstanceModelIds = modelIds; }); // ── Connection status toasts ── diff --git a/dashboard/src/routes/downloads/+page.svelte b/dashboard/src/routes/downloads/+page.svelte index 6dbc206d..5d472186 100644 --- a/dashboard/src/routes/downloads/+page.svelte +++ b/dashboard/src/routes/downloads/+page.svelte @@ -47,6 +47,7 @@ limitBytes: number; modelDirectory?: string; } + | { kind: "evicted"; evictedFor: string; modelDirectory?: string } | { kind: "not_present" }; type ModelCardInfo = { @@ -167,6 +168,7 @@ downloading: 4, pending: 3, rejected: 2, + evicted: 1, failed: 1, not_present: 0, }; @@ -353,6 +355,11 @@ }; } else if (tag === "DownloadFailed") { cell = { kind: "failed", modelDirectory }; + } else if (tag === "DownloadEvicted") { + const evictedFor = + ((payload.evicted_for ?? payload.evictedFor) as string) ?? + "unknown"; + cell = { kind: "evicted", evictedFor, modelDirectory }; } else { const downloaded = getBytes( payload.downloaded ?? @@ -787,6 +794,51 @@ {/if} + {:else if cell.kind === "evicted"} +
+ + + + + Evicted for {cell.evictedFor.split("/").pop()} + + {#if row.shardMetadata} + + {/if} +
{:else if cell.kind === "failed"}