diff --git a/.github/actions/build-cuda-release/action.yml b/.github/actions/build-cuda-release/action.yml
index e306095b..b11b332f 100644
--- a/.github/actions/build-cuda-release/action.yml
+++ b/.github/actions/build-cuda-release/action.yml
@@ -18,7 +18,7 @@ runs:
       env:
         CMAKE_ARGS: -DMLX_BUILD_CUDA=ON
       run: |
-        pip install auditwheel build patchelf setuptools
+        pip install auditwheel "build<=1.4.2" patchelf setuptools
         python setup.py clean --all
         MLX_DISABLE_SM90A_KERNELS=1 MLX_BUILD_STAGE=2 python -m build -w
 
diff --git a/.github/actions/build-linux-release/action.yml b/.github/actions/build-linux-release/action.yml
index c8dec00b..2e938d85 100644
--- a/.github/actions/build-linux-release/action.yml
+++ b/.github/actions/build-linux-release/action.yml
@@ -25,7 +25,7 @@ runs:
     - name: Build Python package
       shell: bash
       run: |
-        pip install auditwheel patchelf build
+        pip install auditwheel patchelf "build<=1.4.2"
         python setup.py clean --all
         MLX_BUILD_STAGE=1 python -m build -w
         auditwheel repair dist/mlx-*.whl \
diff --git a/python/tests/test_conv.py b/python/tests/test_conv.py
index 80b7d6b6..b4760c92 100644
--- a/python/tests/test_conv.py
+++ b/python/tests/test_conv.py
@@ -1,6 +1,7 @@
 # Copyright © 2023-2024 Apple Inc.
 
 import math
+import os
 import unittest
 from itertools import permutations
 
@@ -1151,6 +1152,7 @@ class TestConv(mlx_tests.MLXTestCase):
             )
             self.assertEqual(grads.shape, k_shape)
 
+    @unittest.skipIf(mx.cuda.is_available() and "CI" in os.environ, "flaky in CI")
     def test_conv_1d_with_2d(self):
         x = mx.random.uniform(shape=(2, 10, 16))
         y = mx.random.normal(shape=(16, 3, 16))
diff --git a/python/tests/test_fast_sdpa.py b/python/tests/test_fast_sdpa.py
index 115fd5f8..7bd86708 100644
--- a/python/tests/test_fast_sdpa.py
+++ b/python/tests/test_fast_sdpa.py
@@ -1,4 +1,5 @@
 import math
+import os
 import unittest
 from itertools import product
 
@@ -443,6 +444,7 @@ class TestFastSDPA(mlx_tests.MLXTestCase):
                 self.assertLessEqual(mx.max(diff).item(), atol)
 
     @unittest.skipIf(not mx.is_available(mx.gpu), "too slow on CPU")
+    @unittest.skipIf(mx.cuda.is_available() and "CI" in os.environ, "not enough memory")
     def test_sdpa_long_masked_sequence(self):
         # Test for int16 overflow in steel_attention_nax.h mask
         # indexing (col_pos declared as short, overflows when kL > 32767).