diff --git a/mlx_lm/generate.py b/mlx_lm/generate.py
index 8e85ab6..b37a13d 100644
--- a/mlx_lm/generate.py
+++ b/mlx_lm/generate.py
@@ -1081,6 +1081,7 @@ class BatchGenerator:
                         for uid, length in zip(uids, lengths)
                     ]
                 )
+                mx.clear_cache()
 
         # Further prompt processing so we need to
         #   1. Merge the KV caches and prepare for right padded prompts