diff --git a/python/mlx/nn/layers/distributed.py b/python/mlx/nn/layers/distributed.py
index 92acde8f..f33e3cbc 100644
--- a/python/mlx/nn/layers/distributed.py
+++ b/python/mlx/nn/layers/distributed.py
@@ -86,6 +86,8 @@ def _all_to_sharded(segments):
     representation becomes a sharded representation."""
 
     def _shard_fn(path, weight):
+        if path.endswith("bias"):
+            return -1, segments
         return max(weight.ndim - 2, 0), segments
 
     return _shard_fn