diff --git a/dashboard/src/lib/stores/app.svelte.ts b/dashboard/src/lib/stores/app.svelte.ts index 352a4c07..1932011c 100644 --- a/dashboard/src/lib/stores/app.svelte.ts +++ b/dashboard/src/lib/stores/app.svelte.ts @@ -8,6 +8,11 @@ */ import { browser } from "$app/environment"; +import { addToast } from "$lib/stores/toast.svelte"; +import { + getDownloadTag, + extractModelIdFromDownload, +} from "$lib/utils/downloads"; // UUID generation fallback for browsers without crypto.randomUUID function generateUUID(): string { @@ -254,6 +259,13 @@ interface RawStateResponse { string, { total: { inBytes: number }; available: { inBytes: number } } >; + nodeStorageConfig?: Record< + string, + { + maxStorage: { inBytes: number } | null; + storagePolicy: "manual" | "auto-evict"; + } + >; } export interface MessageAttachment { @@ -568,6 +580,15 @@ class AppStore { > >({}); nodeRdmaCtl = $state>({}); + nodeStorageConfig = $state< + Record< + string, + { + maxStorage: { inBytes: number } | null; + storagePolicy: "manual" | "auto-evict"; + } + > + >({}); nodeThunderboltBridge = $state< Record< string, @@ -1311,6 +1332,7 @@ class AppStore { this.runners = data.runners; } if (data.downloads) { + this.detectEvictions(this.downloads, data.downloads); this.downloads = data.downloads; } if (data.nodeDisk) { @@ -1326,6 +1348,7 @@ class AppStore { this.thunderboltBridgeCycles = data.thunderboltBridgeCycles ?? []; // Thunderbolt bridge status per node this.nodeThunderboltBridge = data.nodeThunderboltBridge ?? {}; + this.nodeStorageConfig = data.nodeStorageConfig ?? {}; this.lastUpdate = Date.now(); // Connection recovered if (!this.isConnected) { @@ -1438,6 +1461,63 @@ class AppStore { } } + /** + * Detect models that were evicted (completed -> not completed) and show toasts. + */ + private detectEvictions( + oldDownloads: Record, + newDownloads: Record, + ) { + if (Object.keys(oldDownloads).length === 0) return; + + const wasCompleted = new Map>(); + for (const [nodeId, entries] of Object.entries(oldDownloads)) { + if (!Array.isArray(entries)) continue; + for (const entry of entries) { + const tagged = getDownloadTag(entry); + if (!tagged || tagged[0] !== "DownloadCompleted") continue; + const modelId = extractModelIdFromDownload(tagged[1]); + if (!modelId) continue; + if (!wasCompleted.has(nodeId)) wasCompleted.set(nodeId, new Set()); + wasCompleted.get(nodeId)!.add(modelId); + } + } + + for (const [nodeId, modelIds] of wasCompleted) { + const newEntries = newDownloads[nodeId]; + const stillCompleted = new Set(); + if (Array.isArray(newEntries)) { + for (const entry of newEntries) { + const tagged = getDownloadTag(entry); + if (!tagged || tagged[0] !== "DownloadCompleted") continue; + const mid = extractModelIdFromDownload(tagged[1]); + if (mid) stillCompleted.add(mid); + } + } + + for (const modelId of modelIds) { + if (!stillCompleted.has(modelId)) { + const shortName = modelId.split("/").pop() ?? modelId; + const nodeLabel = this.getNodeFriendlyName(nodeId); + addToast({ + type: "info", + message: `Model evicted on ${nodeLabel}: ${shortName}`, + duration: 6000, + }); + } + } + } + } + + private getNodeFriendlyName(nodeId: string): string { + const nodeInfo = this.topologyData?.nodes?.[nodeId]; + return ( + nodeInfo?.friendly_name ?? + nodeInfo?.system_info?.chip ?? + nodeId.slice(0, 4) + ); + } + /** * Handle topology changes - clean up filter and re-fetch if needed */ @@ -3279,6 +3359,29 @@ class AppStore { } } + async setStorageConfig( + nodeId: string, + maxStorageGb: number | null, + storagePolicy: "manual" | "auto-evict", + ): Promise { + try { + const response = await fetch(`/storage/${encodeURIComponent(nodeId)}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ maxStorageGb, storagePolicy }), + }); + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to set storage config: ${response.status} - ${errorText}`, + ); + } + } catch (error) { + console.error("Error setting storage config:", error); + throw error; + } + } + /** * List all available traces */ @@ -3356,6 +3459,7 @@ export const instances = () => appStore.instances; export const runners = () => appStore.runners; export const downloads = () => appStore.downloads; export const nodeDisk = () => appStore.nodeDisk; +export const nodeStorageConfig = () => appStore.nodeStorageConfig; export const placementPreviews = () => appStore.placementPreviews; export const selectedPreviewModelId = () => appStore.selectedPreviewModelId; export const isLoadingPreviews = () => appStore.isLoadingPreviews; @@ -3479,6 +3583,11 @@ export const startDownload = (nodeId: string, shardMetadata: object) => appStore.startDownload(nodeId, shardMetadata); export const deleteDownload = (nodeId: string, modelId: string) => appStore.deleteDownload(nodeId, modelId); +export const setStorageConfig = ( + nodeId: string, + maxStorageGb: number | null, + storagePolicy: "manual" | "auto-evict", +) => appStore.setStorageConfig(nodeId, maxStorageGb, storagePolicy); // Trace actions export const listTraces = () => appStore.listTraces(); diff --git a/dashboard/src/routes/+page.svelte b/dashboard/src/routes/+page.svelte index 965482e6..ef05e729 100644 --- a/dashboard/src/routes/+page.svelte +++ b/dashboard/src/routes/+page.svelte @@ -1581,12 +1581,14 @@ progress: DownloadProgress | null; perNode: NodeDownloadStatus[]; failedError: string | null; + rejectedError: string | null; } { const empty = { isDownloading: false, progress: null, perNode: [] as NodeDownloadStatus[], failedError: null, + rejectedError: null, }; if (!downloadsData || Object.keys(downloadsData).length === 0) { @@ -1628,6 +1630,19 @@ (downloadPayload.errorMessage as string) || (downloadPayload.error_message as string) || "Download failed", + rejectedError: null, + }; + } + + // DownloadRejected — storage limit exceeded + if (downloadKind === "DownloadRejected") { + return { + isDownloading: false, + progress: null, + perNode: Array.from(perNodeMap.values()), + failedError: null, + rejectedError: + (downloadPayload.reason as string) || "Storage limit exceeded", }; } @@ -1722,6 +1737,7 @@ progress: null, perNode, failedError: null, + rejectedError: null, }; } @@ -1742,6 +1758,7 @@ }, perNode, failedError: null, + rejectedError: null, }; } @@ -1826,6 +1843,17 @@ }; } + if (result.rejectedError) { + return { + isDownloading: false, + isFailed: true, + errorMessage: result.rejectedError, + progress: null, + statusText: "REJECTED", + perNode: [], + }; + } + if (!result.isDownloading) { const statusInfo = deriveInstanceStatus(instanceWrapped); return { @@ -2461,6 +2489,7 @@ // ── Instance status transition toasts ── // Track previous statuses so we can detect meaningful transitions and fire toasts. let previousInstanceStatuses: Record = {}; + let previousInstanceModelIds: Record = {}; $effect(() => { const currentStatuses: Record = {}; @@ -2475,7 +2504,13 @@ if (Object.keys(prev).length > 0) { for (const [id, currentStatus] of Object.entries(currentStatuses)) { const prevStatus = prev[id]; - if (!prevStatus || prevStatus === currentStatus) continue; + if (prevStatus === currentStatus) continue; + if ( + !prevStatus && + currentStatus !== "REJECTED" && + currentStatus !== "FAILED" + ) + continue; const modelId = getInstanceModelId(instanceData[id]); const shortName = modelId @@ -2509,6 +2544,14 @@ addToast({ type: "error", message: `Model failed: ${shortName}` }); } + if (prevStatus !== "REJECTED" && currentStatus === "REJECTED") { + addToast({ + type: "warning", + message: `Storage limit exceeded: ${shortName}`, + duration: 8000, + }); + } + // Any -> Shutdown if (prevStatus !== "SHUTDOWN" && currentStatus === "SHUTDOWN") { addToast({ type: "info", message: `Model shut down: ${shortName}` }); @@ -2516,7 +2559,31 @@ } } + // Detect instances that disappeared while in early states (e.g. rejected download) + if (Object.keys(prev).length > 0) { + for (const [id, prevStatus] of Object.entries(prev)) { + if (id in currentStatuses) continue; // still exists + if (prevStatus === "PREPARING" || prevStatus === "DOWNLOADING") { + const modelId = previousInstanceModelIds[id]; + const shortName = modelId + ? (modelId.split("/").pop() ?? modelId) + : id.slice(0, 8); + addToast({ + type: "warning", + message: `Download cancelled: ${shortName} — insufficient storage`, + duration: 8000, + }); + } + } + } + previousInstanceStatuses = currentStatuses; + const modelIds: Record = {}; + for (const [id, inst] of Object.entries(instanceData)) { + const mid = getInstanceModelId(inst); + if (mid) modelIds[id] = mid; + } + previousInstanceModelIds = modelIds; }); // ── Connection status toasts ── diff --git a/dashboard/src/routes/downloads/+page.svelte b/dashboard/src/routes/downloads/+page.svelte index e3420005..6dbc206d 100644 --- a/dashboard/src/routes/downloads/+page.svelte +++ b/dashboard/src/routes/downloads/+page.svelte @@ -6,16 +6,19 @@ topologyData, downloads, nodeDisk, + nodeStorageConfig, refreshState, lastUpdate as lastUpdateStore, startDownload, deleteDownload, + setStorageConfig, } from "$lib/stores/app.svelte"; import { getDownloadTag, extractModelIdFromDownload, extractShardMetadata, } from "$lib/utils/downloads"; + import { addToast } from "$lib/stores/toast.svelte"; import HeaderNav from "$lib/components/HeaderNav.svelte"; type CellStatus = @@ -36,6 +39,14 @@ modelDirectory?: string; } | { kind: "failed"; modelDirectory?: string } + | { + kind: "rejected"; + reason: string; + requiredBytes: number; + availableBytes: number; + limitBytes: number; + modelDirectory?: string; + } | { kind: "not_present" }; type ModelCardInfo = { @@ -61,11 +72,14 @@ label: string; diskAvailable?: number; diskTotal?: number; + storageLimit?: number; + storagePolicy?: "manual" | "auto-evict"; }; const data = $derived(topologyData()); const downloadsData = $derived(downloads()); const nodeDiskData = $derived(nodeDisk()); + const storageConfigData = $derived(nodeStorageConfig()); function getNodeLabel(nodeId: string): string { const node = data?.nodes?.[nodeId]; @@ -122,10 +136,37 @@ return Math.min(100, Math.max(0, value as number)); } + function getNodeUsedStorage(nodeId: string): number { + const nodeDownloads = downloadsData?.[nodeId]; + if (!nodeDownloads || !Array.isArray(nodeDownloads)) return 0; + let total = 0; + for (const entry of nodeDownloads) { + const tagged = getDownloadTag(entry); + if (!tagged) continue; + const [tag, payload] = tagged; + if (tag === "DownloadCompleted") { + total += getBytes(payload.total); + } else if (tag === "DownloadOngoing") { + const prog = (payload.download_progress ?? payload.downloadProgress) as + | Record + | undefined; + if (prog) total += getBytes(prog.downloaded); + } + } + return total; + } + + function storageBarColor(percent: number): string { + if (percent >= 90) return "bg-red-500"; + if (percent >= 70) return "bg-yellow-500"; + return "bg-green-500"; + } + const CELL_PRIORITY: Record = { - completed: 4, - downloading: 3, - pending: 2, + completed: 5, + downloading: 4, + pending: 3, + rejected: 2, failed: 1, not_present: 0, }; @@ -178,6 +219,43 @@ let nodeColumns = $state([]); let infoRow = $state(null); + // Storage config modal state + let storageConfigNode = $state(null); + let configMaxGb = $state(null); + let configNoLimit = $state(true); + let configPolicy = $state<"manual" | "auto-evict">("manual"); + let configSaving = $state(false); + + function openStorageConfig(col: NodeColumn) { + storageConfigNode = col; + if (col.storageLimit != null) { + configNoLimit = false; + configMaxGb = Math.round(col.storageLimit / 1024 ** 3); + } else { + configNoLimit = true; + configMaxGb = null; + } + configPolicy = col.storagePolicy ?? "manual"; + } + + async function saveStorageConfig() { + if (!storageConfigNode) return; + configSaving = true; + try { + const maxGb = configNoLimit ? null : configMaxGb; + await setStorageConfig(storageConfigNode.nodeId, maxGb, configPolicy); + storageConfigNode = null; + refreshState(); + } catch (error) { + addToast({ + type: "error", + message: `Failed to save storage config: ${error instanceof Error ? error.message : String(error)}`, + }); + } finally { + configSaving = false; + } + } + $effect(() => { try { if (!downloadsData || Object.keys(downloadsData).length === 0) { @@ -189,11 +267,14 @@ const allNodeIds = Object.keys(downloadsData); const columns: NodeColumn[] = allNodeIds.map((nodeId) => { const diskInfo = nodeDiskData?.[nodeId]; + const storageConfig = storageConfigData?.[nodeId]; return { nodeId, label: getNodeLabel(nodeId), diskAvailable: diskInfo?.available?.inBytes, diskTotal: diskInfo?.total?.inBytes, + storageLimit: storageConfig?.maxStorage?.inBytes ?? undefined, + storagePolicy: storageConfig?.storagePolicy, }; }); @@ -236,7 +317,11 @@ let cell: CellStatus; if (tag === "DownloadCompleted") { const totalBytes = getBytes(payload.total); - cell = { kind: "completed", totalBytes, modelDirectory }; + cell = { + kind: "completed", + totalBytes, + modelDirectory, + }; } else if (tag === "DownloadOngoing") { const rawProgress = payload.download_progress ?? payload.downloadProgress ?? {}; @@ -257,6 +342,15 @@ etaMs, modelDirectory, }; + } else if (tag === "DownloadRejected") { + cell = { + kind: "rejected", + reason: (payload.reason as string) ?? "Storage limit exceeded", + requiredBytes: getBytes(payload.required), + availableBytes: getBytes(payload.available), + limitBytes: getBytes(payload.limit), + modelDirectory, + }; } else if (tag === "DownloadFailed") { cell = { kind: "failed", modelDirectory }; } else { @@ -406,15 +500,52 @@ Model {#each nodeColumns as col} + {@const usedStorage = getNodeUsedStorage(col.nodeId)} + {@const storageMax = col.storageLimit ?? col.diskTotal ?? 0} + {@const storagePercent = + storageMax > 0 + ? Math.min(100, (usedStorage / storageMax) * 100) + : 0} -
{col.label}
- {#if col.diskAvailable != null} -
+ {col.label} + +
+ {#if storageMax > 0} +
+
+
+
+
+ {formatBytes(usedStorage)} / {formatBytes(storageMax)} +
{/if} @@ -610,6 +741,52 @@ ... {/if} + {:else if cell.kind === "rejected"} +
+ + + + Need {formatBytes(cell.requiredBytes)} + {formatBytes(cell.availableBytes)} avail + {#if row.shardMetadata} + + {/if} +
{:else if cell.kind === "failed"}
{col.label} {#if cellStatus.kind === "downloading" && "percentage" in cellStatus} @@ -839,6 +1018,136 @@
{/if} + +{#if storageConfigNode} +
(storageConfigNode = null)} + role="presentation" + >
+ +{/if} +