bug

2026-01-23 15:00:11 +00:00
parent 457debc338
commit 8798ab52ee
2 changed files with 3 additions and 3 deletions
@@ -39,8 +39,8 @@ class BatchedScoringHandler:
    tokenizer: TokenizerWrapper
    model_id: ModelId
    device_rank: int
-    max_batch_size: int = 32
-    batch_timeout_ms: int = 10
+    max_batch_size: int = 128
+    batch_timeout_ms: int = 20

    pending: list[PendingScoringRequest] = field(default_factory=list)
    pending_start_time: float | None = None
@@ -89,7 +89,7 @@ from exo.worker.runner.bootstrap import logger

 # Batching configuration
 BATCH_ENABLED = True
-BATCH_MAX_SIZE = 64
+BATCH_MAX_SIZE = 128


 def _should_use_serial_processing(