From 7bb5cb4fc7f2933be00cb5ea35141746a2d08f5d Mon Sep 17 00:00:00 2001 From: Ryuichi Leo Takashige Date: Thu, 12 Mar 2026 17:34:53 +0000 Subject: [PATCH] Allow memory profiling to be unstable --- src/exo/worker/engines/vllm/growable_cache.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/exo/worker/engines/vllm/growable_cache.py b/src/exo/worker/engines/vllm/growable_cache.py index aa2dbd41..1167f062 100644 --- a/src/exo/worker/engines/vllm/growable_cache.py +++ b/src/exo/worker/engines/vllm/growable_cache.py @@ -33,7 +33,13 @@ def _patch_determine_available_memory() -> None: @torch.inference_mode() def patched(self: "Worker") -> int: - original(self) + try: + original(self) + except AssertionError: + logger.warning( + "vLLM memory profiling assertion failed (free memory changed during init, " + "likely another process released GPU memory). Continuing with growable cache." + ) torch.cuda.empty_cache() free_bytes, _ = torch.cuda.mem_get_info() initial = max(int(free_bytes * INITIAL_FRACTION), 1)