fix(nlf): bail out after persistent inference failures

Add FAIL_THRESHOLD=30 counter: after 30 consecutive inference
failures (NotImplementedError or any Exception), log once at ERROR
and exit the loop instead of spinning at full CPU. Reset on success.
This commit is contained in:
L'électron rare
2026-05-13 11:49:59 +02:00
parent 0d1e97865f
commit f58f1d40e8
2 changed files with 119 additions and 0 deletions
+24
View File
@@ -27,6 +27,8 @@ CKPT_S = CACHE / "nlf_s_multi.torchscript"
N_VERTS = 6890
N_JOINTS = 24
FAIL_THRESHOLD = 30 # ~1 s at 30 fps before giving up
class NLFWorker:
def __init__(self, state: State, num_persons: int = 4,
@@ -40,6 +42,7 @@ class NLFWorker:
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self._smooth_pos: list[list] = []
self.failure_count = 0
@staticmethod
def is_available() -> bool:
@@ -53,6 +56,9 @@ class NLFWorker:
def stop(self) -> None:
self._stop.set()
def _record_success(self) -> None:
self.failure_count = 0
def _run(self) -> None:
try:
import torch
@@ -116,11 +122,29 @@ class NLFWorker:
try:
with torch.inference_mode():
pred = model.detect_smpl_batched(frame_batch)
except NotImplementedError as e:
self.failure_count += 1
if self.failure_count >= FAIL_THRESHOLD:
LOG.error(
"NLF inference unsupported on device=%s after %d frames: %s. "
"TorchScript checkpoint is CUDA-only; install CUDA or switch backend.",
device, self.failure_count, e,
)
return
time.sleep(self.period)
continue
except Exception as e:
self.failure_count += 1
if self.failure_count >= FAIL_THRESHOLD:
LOG.error("NLF inference failed %d frames in a row, stopping: %s",
self.failure_count, e)
return
LOG.warning("inference failed: %s", e)
time.sleep(self.period)
continue
self._record_success()
verts_all = pred.get("vertices3d_nonparam")
joints_all = pred.get("joints3d_nonparam")
trans_all = pred.get("trans")
@@ -0,0 +1,95 @@
"""NLFWorker must bail out after FAIL_THRESHOLD consecutive inference failures."""
import sys
import threading
from unittest.mock import MagicMock
import pytest
from data_only_viz.nlf_worker import NLFWorker, FAIL_THRESHOLD
from data_only_viz.state import State
def _make_fake_torch(raises: bool = True):
"""Return a MagicMock that quacks like torch for _run()."""
fake_model = MagicMock()
if raises:
fake_model.detect_smpl_batched.side_effect = NotImplementedError("CUDA only")
else:
pred = MagicMock()
pred.get.return_value = None # verts_all is None → continues normally
fake_model.detect_smpl_batched.return_value = pred
fake_model.eval.return_value = fake_model
mock_torch = MagicMock()
mock_torch.jit.load.return_value = fake_model
mock_torch.backends.mps.is_available.return_value = False
tensor = MagicMock()
mock_torch.from_numpy.return_value = tensor
tensor.permute.return_value = tensor
tensor.unsqueeze.return_value = tensor
tensor.to.return_value = tensor
# inference_mode() used as context manager
ctx = MagicMock()
ctx.__enter__ = MagicMock(return_value=None)
ctx.__exit__ = MagicMock(return_value=False)
mock_torch.inference_mode.return_value = ctx
return mock_torch
def _make_fake_cv2():
mock_cv2 = MagicMock()
cap = MagicMock()
cap.isOpened.return_value = True
cap.read.return_value = (True, MagicMock())
mock_cv2.VideoCapture.return_value = cap
mock_cv2.cvtColor.return_value = MagicMock()
mock_cv2.CAP_PROP_FRAME_WIDTH = 3
mock_cv2.CAP_PROP_FRAME_HEIGHT = 4
mock_cv2.COLOR_BGR2RGB = 4
return mock_cv2
def test_bailout_after_threshold_failures(tmp_path):
# Inject fake torch and cv2 into sys.modules before _run() does its imports
fake_torch = _make_fake_torch(raises=True)
fake_cv2 = _make_fake_cv2()
# Patch ckpt_path to a fake existing file so the worker doesn't abort early
fake_ckpt = tmp_path / "fake.torchscript"
fake_ckpt.write_bytes(b"")
original_torch = sys.modules.get("torch")
original_cv2 = sys.modules.get("cv2")
sys.modules["torch"] = fake_torch
sys.modules["cv2"] = fake_cv2
try:
state = State()
worker = NLFWorker(state, num_persons=1, target_fps=1000.0, device="cpu")
worker.ckpt_path = fake_ckpt
t = threading.Thread(target=worker._run, daemon=True)
t.start()
t.join(timeout=4.0)
finally:
if original_torch is None:
sys.modules.pop("torch", None)
else:
sys.modules["torch"] = original_torch
if original_cv2 is None:
sys.modules.pop("cv2", None)
else:
sys.modules["cv2"] = original_cv2
assert not t.is_alive(), "worker should exit after threshold failures"
assert worker.failure_count >= FAIL_THRESHOLD
def test_failure_counter_resets_on_success():
state = State()
worker = NLFWorker(state, num_persons=1, target_fps=10.0, device="cpu")
worker.failure_count = FAIL_THRESHOLD - 1
worker._record_success()
assert worker.failure_count == 0