From 8fe1d09207c671309a675aab9e5970d0763e8987 Mon Sep 17 00:00:00 2001
From: Awni Hannun <awni@apple.com>
Date: Fri, 6 Feb 2026 16:38:36 -0800
Subject: [PATCH] Fix residency set with user provided buffer (#3108)

---
 mlx/backend/metal/allocator.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlx/backend/metal/allocator.cpp b/mlx/backend/metal/allocator.cpp
index 06e3d9ae..c15bf3bd 100644
--- a/mlx/backend/metal/allocator.cpp
+++ b/mlx/backend/metal/allocator.cpp
@@ -226,6 +226,7 @@ void MetalAllocator::release(Buffer buffer) {
   std::unique_lock lk(mutex_);
   active_memory_ -= buf->length();
   num_resources_--;
+  residency_set_.erase(buf);
   lk.unlock();
   auto pool = metal::new_scoped_memory_pool();
   buf->release();