diff --git a/src/exo/main.py b/src/exo/main.py
index c3f165b4..e1ef3a70 100644
--- a/src/exo/main.py
+++ b/src/exo/main.py
@@ -251,7 +251,8 @@ class Node:
 def main():
     args = Args.parse()
     soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
-    resource.setrlimit(resource.RLIMIT_NOFILE, (max(soft, 65535), hard))
+    target = min(max(soft, 65535), hard)
+    resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard))
 
     mp.set_start_method("spawn")
     # TODO: Refactor the current verbosity system
diff --git a/src/exo/worker/engines/mlx/utils_mlx.py b/src/exo/worker/engines/mlx/utils_mlx.py
index a1fa097d..8117c6cc 100644
--- a/src/exo/worker/engines/mlx/utils_mlx.py
+++ b/src/exo/worker/engines/mlx/utils_mlx.py
@@ -1,6 +1,5 @@
 import json
 import os
-import resource
 import sys
 import time
 from pathlib import Path
@@ -63,8 +62,6 @@ from exo.worker.engines.mlx.auto_parallel import (
 from exo.worker.runner.bootstrap import logger
 
 Group = mx.distributed.Group
-# Needed for 8 bit model
-resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 4096))
 
 
 # TODO: Test this
diff --git a/src/exo/worker/runner/runner.py b/src/exo/worker/runner/runner.py
index 2764f024..0b3cdb16 100644
--- a/src/exo/worker/runner/runner.py
+++ b/src/exo/worker/runner/runner.py
@@ -1,5 +1,6 @@
 import base64
 import json
+import resource
 import time
 from collections.abc import Generator
 from functools import cache
@@ -112,6 +113,9 @@ def main(
     event_sender: MpSender[Event],
     task_receiver: MpReceiver[Task],
 ):
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    resource.setrlimit(resource.RLIMIT_NOFILE, (min(max(soft, 2048), hard), hard))
+
     instance, runner_id, shard_metadata = (
         bound_instance.instance,
         bound_instance.bound_runner_id,