electron-rare/exo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use DownloadEvicted event

Browse Source
This commit is contained in:
ciaranbor
2026-03-09 16:56:32 +00:00
parent f87d705b9c
commit 60f6d8e6fc
6 changed files with 115 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dashboard/src/lib/stores/app.svelte.ts
-63
View File
@@ -8,11 +8,6 @@
*/
import { browser } from "$app/environment";
import { addToast } from "$lib/stores/toast.svelte";
import {
getDownloadTag,
extractModelIdFromDownload,
} from "$lib/utils/downloads";
// UUID generation fallback for browsers without crypto.randomUUID
function generateUUID(): string {
@@ -1332,7 +1327,6 @@ class AppStore {
this.runners = data.runners;
}
if (data.downloads) {
this.detectEvictions(this.downloads, data.downloads);
this.downloads = data.downloads;
}
if (data.nodeDisk) {
@@ -1461,63 +1455,6 @@ class AppStore {
}
}
/**
* Detect models that were evicted (completed -> not completed) and show toasts.
*/
private detectEvictions(
oldDownloads: Record<string, unknown[]>,
newDownloads: Record<string, unknown[]>,
) {
if (Object.keys(oldDownloads).length === 0) return;
const wasCompleted = new Map<string, Set<string>>();
for (const [nodeId, entries] of Object.entries(oldDownloads)) {
if (!Array.isArray(entries)) continue;
for (const entry of entries) {
const tagged = getDownloadTag(entry);
if (!tagged || tagged[0] !== "DownloadCompleted") continue;
const modelId = extractModelIdFromDownload(tagged[1]);
if (!modelId) continue;
if (!wasCompleted.has(nodeId)) wasCompleted.set(nodeId, new Set());
wasCompleted.get(nodeId)!.add(modelId);
}
}
for (const [nodeId, modelIds] of wasCompleted) {
const newEntries = newDownloads[nodeId];
const stillCompleted = new Set<string>();
if (Array.isArray(newEntries)) {
for (const entry of newEntries) {
const tagged = getDownloadTag(entry);
if (!tagged || tagged[0] !== "DownloadCompleted") continue;
const mid = extractModelIdFromDownload(tagged[1]);
if (mid) stillCompleted.add(mid);
}
}
for (const modelId of modelIds) {
if (!stillCompleted.has(modelId)) {
const shortName = modelId.split("/").pop() ?? modelId;
const nodeLabel = this.getNodeFriendlyName(nodeId);
addToast({
type: "info",
message: `Model evicted on ${nodeLabel}: ${shortName}`,
duration: 6000,
});
}
}
}
}
private getNodeFriendlyName(nodeId: string): string {
const nodeInfo = this.topologyData?.nodes?.[nodeId];
return (
nodeInfo?.friendly_name ??
nodeInfo?.system_info?.chip ??
nodeId.slice(0, 4)
);
}
/**
* Handle topology changes - clean up filter and re-fetch if needed
*/
dashboard/src/routes/+page.svelte
+34 -21
View File
@@ -1582,6 +1582,7 @@
perNode: NodeDownloadStatus[];
failedError: string | null;
rejectedError: string | null;
evicted: boolean;
} {
const empty = {
isDownloading: false,
@@ -1589,6 +1590,7 @@
perNode: [] as NodeDownloadStatus[],
failedError: null,
rejectedError: null,
evicted: false,
};
if (!downloadsData || Object.keys(downloadsData).length === 0) {
@@ -1631,6 +1633,7 @@
(downloadPayload.error_message as string) ||
"Download failed",
rejectedError: null,
evicted: false,
};
}
@@ -1643,6 +1646,19 @@
failedError: null,
rejectedError:
(downloadPayload.reason as string) || "Storage limit exceeded",
evicted: false,
};
}
// DownloadEvicted — model was evicted from storage
if (downloadKind === "DownloadEvicted") {
return {
isDownloading: false,
progress: null,
perNode: Array.from(perNodeMap.values()),
failedError: null,
rejectedError: null,
evicted: true,
};
}
@@ -1738,6 +1754,7 @@
perNode,
failedError: null,
rejectedError: null,
evicted: false,
};
}
@@ -1759,6 +1776,7 @@
perNode,
failedError: null,
rejectedError: null,
evicted: false,
};
}
@@ -1854,6 +1872,17 @@
};
}
if (result.evicted) {
return {
isDownloading: false,
isFailed: false,
errorMessage: null,
progress: null,
statusText: "EVICTED",
perNode: [],
};
}
if (!result.isDownloading) {
const statusInfo = deriveInstanceStatus(instanceWrapped);
return {
@@ -2489,7 +2518,6 @@
// ── Instance status transition toasts ──
// Track previous statuses so we can detect meaningful transitions and fire toasts.
let previousInstanceStatuses: Record<string, string> = {};
let previousInstanceModelIds: Record<string, string> = {};
$effect(() => {
const currentStatuses: Record<string, string> = {};
@@ -2556,34 +2584,19 @@
if (prevStatus !== "SHUTDOWN" && currentStatus === "SHUTDOWN") {
addToast({ type: "info", message: `Model shut down: ${shortName}` });
}
}
}
// Detect instances that disappeared while in early states (e.g. rejected download)
if (Object.keys(prev).length > 0) {
for (const [id, prevStatus] of Object.entries(prev)) {
if (id in currentStatuses) continue; // still exists
if (prevStatus === "PREPARING" || prevStatus === "DOWNLOADING") {
const modelId = previousInstanceModelIds[id];
const shortName = modelId
? (modelId.split("/").pop() ?? modelId)
: id.slice(0, 8);
// Any -> Evicted
if (prevStatus !== "EVICTED" && currentStatus === "EVICTED") {
addToast({
type: "warning",
message: `Download cancelled: ${shortName} — insufficient storage`,
duration: 8000,
type: "info",
message: `Model evicted: ${shortName}`,
duration: 6000,
});
}
}
}
previousInstanceStatuses = currentStatuses;
const modelIds: Record<string, string> = {};
for (const [id, inst] of Object.entries(instanceData)) {
const mid = getInstanceModelId(inst);
if (mid) modelIds[id] = mid;
}
previousInstanceModelIds = modelIds;
});
// ── Connection status toasts ──
dashboard/src/routes/downloads/+page.svelte
+52
View File
@@ -47,6 +47,7 @@
limitBytes: number;
modelDirectory?: string;
}
| { kind: "evicted"; evictedFor: string; modelDirectory?: string }
| { kind: "not_present" };
type ModelCardInfo = {
@@ -167,6 +168,7 @@
downloading: 4,
pending: 3,
rejected: 2,
evicted: 1,
failed: 1,
not_present: 0,
};
@@ -353,6 +355,11 @@
};
} else if (tag === "DownloadFailed") {
cell = { kind: "failed", modelDirectory };
} else if (tag === "DownloadEvicted") {
const evictedFor =
((payload.evicted_for ?? payload.evictedFor) as string) ??
"unknown";
cell = { kind: "evicted", evictedFor, modelDirectory };
} else {
const downloaded = getBytes(
payload.downloaded ??
@@ -787,6 +794,51 @@
</button>
{/if}
</div>
{:else if cell.kind === "evicted"}
<div
class="flex flex-col items-center gap-1"
title="Evicted for {cell.evictedFor}"
>
<svg
class="w-5 h-5 text-blue-400"
viewBox="0 0 20 20"
fill="currentColor"
>
<path
fill-rule="evenodd"
d="M18 10a8 8 0 11-16 0 8 8 0 0116 0zm-7-4a1 1 0 11-2 0 1 1 0 012 0zM9 9a1 1 0 000 2v3a1 1 0 001 1h1a1 1 0 100-2v-3a1 1 0 00-1-1H9z"
clip-rule="evenodd"
></path>
</svg>
<span
class="text-[10px] text-blue-400/80 leading-tight text-center max-w-[100px] truncate"
>
Evicted for {cell.evictedFor.split("/").pop()}
</span>
{#if row.shardMetadata}
<button
type="button"
class="text-white/50 hover:text-exo-yellow transition-colors cursor-pointer"
onclick={() =>
startDownload(col.nodeId, row.shardMetadata!)}
title="Re-download on this node"
>
<svg
class="w-5 h-5"
viewBox="0 0 20 20"
fill="none"
stroke="currentColor"
stroke-width="2"
>
<path
d="M10 3v10m0 0l-3-3m3 3l3-3M3 17h14"
stroke-linecap="round"
stroke-linejoin="round"
></path>
</svg>
</button>
{/if}
</div>
{:else if cell.kind === "failed"}
<div
class="flex flex-col items-center gap-1"
src/exo/download/coordinator.py
+20 -8
View File
@@ -54,6 +54,7 @@ from exo.shared.types.memory import Memory
from exo.shared.types.storage import StorageConfig
from exo.shared.types.worker.downloads import (
DownloadCompleted,
DownloadEvicted,
DownloadFailed,
DownloadOngoing,
DownloadPending,
@@ -82,7 +83,6 @@ class DownloadCoordinator:
# LRU tracking
_model_last_used: dict[ModelId, datetime] = field(default_factory=dict)
_active_model_ids: set[ModelId] = field(default_factory=set)
_recently_evicted: set[ModelId] = field(default_factory=set)
_tg: TaskGroup = field(init=False, default_factory=TaskGroup)
_stopped: anyio.Event = field(init=False, default_factory=anyio.Event)
@@ -450,8 +450,6 @@ class DownloadCoordinator:
logger.debug(
"DownloadCoordinator: Fetching and emitting existing download progress..."
)
# Track which evicted models the scanner still sees on disk
evicted_still_on_disk: set[ModelId] = set()
async for (
_,
@@ -459,8 +457,9 @@ class DownloadCoordinator:
) in self.shard_downloader.get_shard_download_status():
model_id = progress.shard.model_card.model_id
if model_id in self._recently_evicted:
evicted_still_on_disk.add(model_id)
# Don't overwrite DownloadEvicted — the model may still be on disk
# while deletion is finishing
if isinstance(self.download_status.get(model_id), DownloadEvicted):
continue
# Active downloads emit progress via the callback — don't overwrite
@@ -542,8 +541,6 @@ class DownloadCoordinator:
NodeDownloadProgress(download_progress=path_completed)
)
self._recently_evicted = evicted_still_on_disk
logger.debug(
"DownloadCoordinator: Done emitting existing download progress."
)
@@ -599,6 +596,8 @@ class DownloadCoordinator:
logger.info(
f"Auto-evicting model {evict_model_id} to free space for {model_id}"
)
# Capture shard_metadata before _delete_download removes it
evicted_status = self.download_status.get(evict_model_id)
success = await self._delete_download(evict_model_id)
if not success:
current_used = calculate_used_storage(
@@ -611,7 +610,20 @@ class DownloadCoordinator:
current_available,
)
return False
self._recently_evicted.add(evict_model_id)
# Overwrite the DownloadPending that _delete_download emitted
# with an explicit DownloadEvicted status
if evicted_status is not None:
evicted = DownloadEvicted(
shard_metadata=evicted_status.shard_metadata,
node_id=self.node_id,
model_directory=self._model_dir(evict_model_id),
evicted_for=model_id,
)
self.download_status[evict_model_id] = evicted
await self.event_sender.send(
NodeDownloadProgress(download_progress=evicted)
)
return True
src/exo/download/tests/test_auto_eviction.py
+3 -1
View File
@@ -20,6 +20,7 @@ from exo.shared.types.memory import Memory
from exo.shared.types.storage import StorageConfig
from exo.shared.types.worker.downloads import (
DownloadCompleted,
DownloadEvicted,
DownloadPending,
DownloadRejected,
)
@@ -128,7 +129,8 @@ class TestStartDownloadAutoEviction:
# MODEL_A (oldest) should have been evicted
mock_delete.assert_called_once_with(MODEL_A)
assert MODEL_A not in coordinator.download_status
assert isinstance(coordinator.download_status[MODEL_A], DownloadEvicted)
assert coordinator.download_status[MODEL_A].evicted_for == MODEL_NEW
@patch(
"exo.download.coordinator.delete_model",
src/exo/shared/types/worker/downloads.py
+6 -1
View File
@@ -3,7 +3,7 @@ from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, PositiveInt
from exo.shared.types.common import NodeId
from exo.shared.types.common import ModelId, NodeId
from exo.shared.types.memory import Memory
from exo.shared.types.worker.shards import ShardMetadata
from exo.utils.pydantic_ext import CamelCaseModel, TaggedModel
@@ -54,12 +54,17 @@ class DownloadRejected(BaseDownloadProgress):
limit: Memory
class DownloadEvicted(BaseDownloadProgress):
evicted_for: ModelId
DownloadProgress = (
DownloadPending
| DownloadCompleted
| DownloadFailed
| DownloadOngoing
| DownloadRejected
| DownloadEvicted
)