fix(viz): harden iphone-usb source

Fix 8 defects from whole-branch review (a79bc14..8c4d9f4): - write_hands param: MediaPipe owns persons_hands in iphone-usb mode; IphoneUSBSource gets write_hands=False in multi.py - Guard cap.start() failure: return early if USB source unavailable - Stale frame on disconnect: read() returns (False,None) when _opened=False after thread exits - ARKit validity: update only valid joints, matching OSC listener - Decode exception: except Exception replaces removal-prone av.AVError - release() unblock: socket.shutdown() before join wakes recv-blocked reader immediately - Log stutter: remove redundant "multi —" prefix from USB log line - Unit tests: 10 pure no-device tests for _to_annexb + _decode_hands; add iphone-usb optional extra (av>=12.0) to pyproject.toml
2026-06-27 15:27:06 +02:00
parent 8c4d9f4d7b
commit 57e3a200bc
5 changed files with 203 additions and 22 deletions
@@ -5,11 +5,15 @@ State. Substitutes for cv2.VideoCapture in multi.py under --iphone-usb."""
 from __future__ import annotations

 import logging
+import socket
 import struct
 import threading
 import time

-import av
+try:
+    import av
+except ImportError:  # pragma: no cover — av missing is caught at runtime by caller
+    av = None  # type: ignore[assignment]
 import cv2
 import numpy as np

@@ -23,6 +27,7 @@ TAG_HANDS = 4
 LOG = logging.getLogger("iphone_usb_source")

 _HAND_BYTES = 1 + 21 * 12   # chirality:u8 + 21 × (x,y,z) × 4 bytes BE f32
+_ARKIT_JOINTS = 91


 def _make_point(x: float, y: float, z: float) -> PoseKp:
@@ -67,14 +72,17 @@ def _to_annexb(data: bytes) -> bytes:


 class IphoneUSBSource:
-    def __init__(self, state=None, target_size=(640, 480)) -> None:
+    def __init__(self, state=None, target_size=(640, 480),
+                 write_hands: bool = True) -> None:
        self.state = state
        self.target_w, self.target_h = target_size
-        self._codec = av.codec.CodecContext.create("hevc", "r")
+        self._write_hands = write_hands
+        self._codec = av.codec.CodecContext.create("hevc", "r") if av is not None else None
        self._lock = threading.Lock()
        self._frame = None            # latest BGR np.ndarray
        self._stop = threading.Event()
        self._thread = None
+        self._sock = None             # stored for shutdown() in release()
        self._opened = False

    def start(self) -> bool:
@@ -82,6 +90,7 @@ class IphoneUSBSource:
        if sock is None:
            LOG.error("iphone usb: no device / connect failed")
            return False
+        self._sock = sock
        self._opened = True
        self._thread = threading.Thread(
            target=self._run, args=(sock,), name="iphone_usb_src", daemon=True)
@@ -98,27 +107,30 @@ class IphoneUSBSource:
                    break
                if tag == TAG_VIDEO and len(payload) > 1:
                    annexb = _to_annexb(payload[1:])
-                    try:
-                        for fr in self._codec.decode(av.Packet(annexb)):
-                            img = fr.to_ndarray(format="bgr24")
-                            img = cv2.resize(img, (self.target_w, self.target_h))
-                            with self._lock:
-                                self._frame = img
-                    except av.AVError as e:
-                        LOG.debug("hevc decode: %s", e)
+                    if self._codec is not None and av is not None:
+                        try:
+                            for fr in self._codec.decode(av.Packet(annexb)):
+                                img = fr.to_ndarray(format="bgr24")
+                                img = cv2.resize(img, (self.target_w, self.target_h))
+                                with self._lock:
+                                    self._frame = img
+                        except Exception as e:  # av.AVError is a removal-prone alias
+                            LOG.debug("hevc decode: %s", e)
                elif tag == TAG_SKELETON and self.state is not None:
                    joints = decode_skeleton(payload)
                    if joints is not None:
-                        arr = np.array(
-                            [[j[0], j[1], j[2]] for j in joints],
-                            dtype=np.float32,
-                        )
                        with self.state.lock():
+                            arr = self.state.persons_arkit_joints.get(pid)
+                            if arr is None or arr.shape != (_ARKIT_JOINTS, 3):
+                                arr = np.zeros((_ARKIT_JOINTS, 3), dtype=np.float32)
+                            for i, (x, y, z, valid) in enumerate(joints):
+                                if valid:
+                                    arr[i] = (x, y, z)
                            self.state.persons_arkit_joints[pid] = arr
                            self.state.persons_arkit_last_t[pid] = time.perf_counter()
-                elif tag == TAG_HANDS and self.state is not None:
+                elif tag == TAG_HANDS:
                    hands = _decode_hands(payload)
-                    if hands is not None:
+                    if hands is not None and self._write_hands and self.state is not None:
                        with self.state.lock():
                            self.state.persons_hands = hands
        except OSError as e:
@@ -132,7 +144,7 @@ class IphoneUSBSource:

    def read(self):
        with self._lock:
-            if self._frame is None:
+            if self._frame is None or not self._opened:
                return False, None
            return True, self._frame.copy()

@@ -141,6 +153,11 @@ class IphoneUSBSource:

    def release(self) -> None:
        self._stop.set()
+        if self._sock is not None:
+            try:
+                self._sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
        if self._thread is not None:
            self._thread.join(timeout=2.0)
        self._opened = False
@@ -346,9 +346,11 @@ class MultiWorker:

        if self.iphone_usb:
            from .iphone_usb_source import IphoneUSBSource  # noqa: PLC0415
-            cap = IphoneUSBSource(self.state)
-            cap.start()
-            LOG.info("multi — iphone USB source")
+            cap = IphoneUSBSource(self.state, write_hands=False)
+            if not cap.start():
+                LOG.error("iphone USB source unavailable (app running? phone unlocked?)")
+                return
+            LOG.info("iphone USB source")
        else:
            cap = cv2.VideoCapture(self.camera_index)
            cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
@@ -73,6 +73,10 @@ multihmr = [
 # via git submodule third_party/SMPLer-X (fork electron-rare).
 # mmcv-lite suffit pour Config (le repo vendorise sa propre mmpose).
 # Le detector mmdet est remplace par YOLO Ultralytics (extras pose).
+iphone-usb = [
+    "av>=12.0",
+    "opencv-python>=4.10",
+]
 smplerx = [
    "torch>=2.4",
    "torchvision>=0.19",
@@ -0,0 +1,124 @@
+"""Pure (no-device) unit tests for iphone_usb_source helpers.
+
+Tests _to_annexb and _decode_hands without any hardware or network connection.
+"""
+import struct
+
+import pytest
+
+from data_only_viz.iphone_usb_source import _to_annexb, _decode_hands, _HAND_BYTES
+
+
+# ---------------------------------------------------------------------------
+# helpers
+# ---------------------------------------------------------------------------
+
+def _avcc(nals: list[bytes]) -> bytes:
+    """Build an AVCC-style buffer with 4-byte big-endian length prefixes."""
+    out = bytearray()
+    for nal in nals:
+        out += struct.pack(">I", len(nal)) + nal
+    return bytes(out)
+
+
+def _make_hands_payload(hands: list[list[tuple[float, float, float]]]) -> bytes:
+    """Build a synthetic HandsPayload matching the wire layout.
+
+    count:u8, then per hand: chirality:u8 (1=right) + 21 × (x,y,z) big-endian f32.
+    """
+    out = bytearray()
+    out += bytes([len(hands)])
+    for hand_kps in hands:
+        out += bytes([1])          # chirality = right
+        for x, y, z in hand_kps:
+            out += struct.pack(">fff", x, y, z)
+    return bytes(out)
+
+
+# ---------------------------------------------------------------------------
+# _to_annexb
+# ---------------------------------------------------------------------------
+
+def test_to_annexb_two_nals():
+    nal1 = b"\x67\x01\x02\x03"    # SPS-like
+    nal2 = b"\x41\xAB\xCD"
+    avcc = _avcc([nal1, nal2])
+    result = _to_annexb(avcc)
+    # First unit: Annex-B start code + nal1
+    assert result[:4] == b"\x00\x00\x00\x01"
+    assert result[4:4 + len(nal1)] == nal1
+    # Second unit: Annex-B start code + nal2
+    rest = result[4 + len(nal1):]
+    assert rest[:4] == b"\x00\x00\x00\x01"
+    assert rest[4:4 + len(nal2)] == nal2
+    assert len(result) == 4 + len(nal1) + 4 + len(nal2)
+
+
+def test_to_annexb_payload_bytes_preserved():
+    nal = bytes(range(32))
+    result = _to_annexb(_avcc([nal]))
+    assert result[4:] == nal
+
+
+def test_to_annexb_empty():
+    assert _to_annexb(b"") == b""
+
+
+def test_to_annexb_truncated_nal_is_dropped():
+    # Length prefix claims 10 bytes but only 3 are present
+    buf = struct.pack(">I", 10) + b"\x00" * 3
+    result = _to_annexb(buf)
+    assert result == b""
+
+
+def test_to_annexb_single_nal():
+    nal = b"\x65\xB8"
+    result = _to_annexb(_avcc([nal]))
+    assert result == b"\x00\x00\x00\x01" + nal
+
+
+# ---------------------------------------------------------------------------
+# _decode_hands
+# ---------------------------------------------------------------------------
+
+def test_decode_hands_empty_payload():
+    assert _decode_hands(b"") is None
+
+
+def test_decode_hands_count_zero():
+    assert _decode_hands(bytes([0])) == []
+
+
+def test_decode_hands_two_hands_length_and_coords():
+    hand_a = [(float(i) * 0.1, float(i) * 0.2, float(i) * 0.05) for i in range(21)]
+    hand_b = [(1.0 - float(i) * 0.03, 0.5, float(i) * 0.01) for i in range(21)]
+    payload = _make_hands_payload([hand_a, hand_b])
+    result = _decode_hands(payload)
+    assert isinstance(result, list)
+    assert len(result) == 2
+    for hand in result:
+        assert len(hand) == 21
+    # x/y spot-checks for hand A
+    assert abs(result[0][0].x - hand_a[0][0]) < 1e-5
+    assert abs(result[0][5].y - hand_a[5][1]) < 1e-5
+    # x/y spot-checks for hand B
+    assert abs(result[1][3].x - hand_b[3][0]) < 1e-5
+    assert abs(result[1][20].y - hand_b[20][1]) < 1e-5
+
+
+def test_decode_hands_one_hand():
+    kps = [(0.1 * i, 0.2 * i, 0.0) for i in range(21)]
+    result = _decode_hands(_make_hands_payload([kps]))
+    assert isinstance(result, list)
+    assert len(result) == 1
+    assert len(result[0]) == 21
+
+
+def test_decode_hands_truncated_does_not_raise():
+    hand_kps = [(0.1, 0.2, 0.3)] * 21
+    full = _make_hands_payload([hand_kps, hand_kps])
+    # Truncate to half — second hand will be incomplete
+    truncated = full[:len(full) // 2]
+    # Must not raise
+    result = _decode_hands(truncated)
+    assert isinstance(result, (list, type(None)))
@@ -71,6 +71,34 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" },
 ]

+[[package]]
+name = "av"
+version = "17.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5e/e3/477fa20578c284abeda08d91b63ee9abaebc93445d8feeb989d3d444bae1/av-17.1.0.tar.gz", hash = "sha256:7f1e71ff621b66253333926f948e00faae11d855b2442133c65128bca64cdeb3", size = 4288546, upload-time = "2026-06-07T05:52:55.999Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ec/87/8036b5c781bc3639ea04ef42d4e26da253bd4bd4311d8705b6a1c8824047/av-17.1.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:ad7b4aa011093324b7118245f50ac6db244cfe9900d4072508a5245a2b0d3f41", size = 22460847, upload-time = "2026-06-07T05:52:04.261Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/af/dfdf6fc7b17814b50d0aa9e7a7e37b87be91be3890f44b0d525433cd1fd1/av-17.1.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:43ebbe977f19a7f2d2bd1a4e119675a0b15e05852cf7309846b6ab922ba7ffe9", size = 18159115, upload-time = "2026-06-07T05:52:06.64Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/13/64f6c466471cea225b8b2f4cdc51a571f8a286984b55a08d169b932fda5d/av-17.1.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6a20658ec7d96a70e14b1196eff00b7cdd8831ac3b99868e16b8ba8b24090847", size = 33224427, upload-time = "2026-06-07T05:52:09.165Z" },
+    { url = "https://files.pythonhosted.org/packages/77/43/96b35170bf2e64e00a41748c6400ff73232dc0fc62ded283679fb07c7fe0/av-17.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f9a65d1f48b818323fb411e80358f89d77dec340b01d27c6b2dfbb9cbf4b779f", size = 35370183, upload-time = "2026-06-07T05:52:11.959Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/b3/8e8b4b6498731bfbd88e8399a756543f8088f1bd33d08eab678b5aebe728/av-17.1.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:58f7593726437cda5bd19793027e027768450b5c4a594777bf487798a33db702", size = 24459265, upload-time = "2026-06-07T05:52:14.66Z" },
+    { url = "https://files.pythonhosted.org/packages/14/ac/ceb84b7553db21f1143d817245c560d9267168e1e58b1a8eeae2b62c4d04/av-17.1.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bbab058bd965309f39962e53caac8126987c68c0be094fc4f9427e5615b0218f", size = 34283709, upload-time = "2026-06-07T05:52:17.389Z" },
+    { url = "https://files.pythonhosted.org/packages/59/f9/4115fd84148c9a1cf365096694be6ac882fd3cd3cdb7a2f35e71fecf1631/av-17.1.0-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:9514cfda85180554c430695282faf4be3ffdf95775d8519733821244eecb58e0", size = 25397573, upload-time = "2026-06-07T05:52:20.012Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/ac/92e52d5ed0e0b84d9d93e52b4338c2713d8a44082b8696e6516fdae7c4e4/av-17.1.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e1c90f85cd7431ede95b11e8e711571a896ebea433f298849c2c0f1594c8d86e", size = 36451495, upload-time = "2026-06-07T05:52:22.581Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/f2/53a7cd34adb6a971d7e6d99663e74db286966c9db8afdca17472fdf0f98e/av-17.1.0-cp311-abi3-win_amd64.whl", hash = "sha256:5df5c1172ef1cf65a1529d612f7da7798ce2cf82c1ff7212466b538a6cc7214c", size = 28036393, upload-time = "2026-06-07T05:52:25.657Z" },
+    { url = "https://files.pythonhosted.org/packages/66/47/cd9ae0edf2206351c1251bb94b5ec58728e42c5f6ee16c03c412f3a1bb3e/av-17.1.0-cp311-abi3-win_arm64.whl", hash = "sha256:ee98534242a74da847af78624779ac5a3177dc7c69f956a4da9e6f0fdb37d7f6", size = 21174601, upload-time = "2026-06-07T05:52:28.077Z" },
+    { url = "https://files.pythonhosted.org/packages/36/90/b5668cddb3c401fcf22553bc495d5b0c6d8a01d118624b26f0db1d0b8653/av-17.1.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:5327807c1219293803ef0c5d1578ff3ae1cf638c09e5998962026e1a554ec240", size = 22699499, upload-time = "2026-06-07T05:52:30.335Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/7e/7be6bfddb823d045ff9fd5d4deb922ee3847605e162c3882e6c45b4c35ff/av-17.1.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:6c9b71fe5c0c5a8d303b1588d4d8ce9397d6b023f467cfef95000ba1f75507fa", size = 18366696, upload-time = "2026-06-07T05:52:32.645Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/23/391dcfa75c1ae1977efca44b753a11b929399b558826670c16a8808dd0e3/av-17.1.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f997e3351bdf51127c07a74e21741a2996e9230cbeb2d81c14acde761b116c9c", size = 36582649, upload-time = "2026-06-07T05:52:35.218Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/32/7312854868b318b9d1b1dcbd1bddb460aaaeac7d57f816e11efec3bef5b1/av-17.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:efe9b1397300b67b644ad220c89df4892a76f2debe70f16bae1749fa20526e63", size = 38479390, upload-time = "2026-06-07T05:52:37.968Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/72/af47f59b4458e81ca7d89f477698dbfb3d5a0cd8ae6c1e4441d01074af8a/av-17.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:fa64e1f1500d01c4a98e7a41dc1a9a35fb4dfe71f5de0389264ec1192200c76a", size = 27127432, upload-time = "2026-06-07T05:52:40.371Z" },
+    { url = "https://files.pythonhosted.org/packages/88/85/c2e6861baf0f8c7d21c4ce811d4d424fedac915e3910d3570ce4377717dc/av-17.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ffbd78d73d2c9bf31e9a007c992faec3991428b2941a3b085b84fb82e8c32d19", size = 37406592, upload-time = "2026-06-07T05:52:43.215Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/40/3cc13125aea976101c0858af99ac47257c0654411aa199b5d8e81eea7002/av-17.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:bff8896454b38fcb785a70e5ae0485d7021cb776303a5849393128a30b8f850b", size = 28336228, upload-time = "2026-06-07T05:52:46.134Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/38/c7d9c3e746209a1a695c13e3aa7d817229e84a85d0a84271f313d1befdd3/av-17.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1284addf3c0dd939887a9722dc30df2241a97471ad52c3c507e31583ae22ff02", size = 39490680, upload-time = "2026-06-07T05:52:48.887Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/25/9d42da561b7b8f7dabdfaebba07b52977bee58c5c7e4285ac991abcfaa72/av-17.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:ec630be6321b04e317862f6082e84812bbd801e55a3c2298312e3fc8a0a4af4f", size = 28355673, upload-time = "2026-06-07T05:52:51.614Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/41/562a61d5a61fba3ffb273a115e249f1d8471b9515c59fcc38b4b9deda238/av-17.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b41647e42884bf543b8e8d0a1dabd4d1b006c99183eb1a2d7afc5b01f73eeff4", size = 21324700, upload-time = "2026-06-07T05:52:53.972Z" },
+]
+
 [[package]]
 name = "av-live-data-only-viz"
 version = "0.1.0"
@@ -100,6 +128,10 @@ detrpose = [
    { name = "transformers" },
    { name = "xtcocotools" },
 ]
+iphone-usb = [
+    { name = "av" },
+    { name = "opencv-python" },
+]
 lidar = [
    { name = "open3d" },
 ]
@@ -154,6 +186,7 @@ dev = [

 [package.metadata]
 requires-dist = [
+    { name = "av", marker = "extra == 'iphone-usb'", specifier = ">=12.0" },
    { name = "cloudpickle", marker = "extra == 'detrpose'", specifier = ">=3.0" },
    { name = "coremltools", marker = "extra == 'pose'", specifier = ">=9.0" },
    { name = "einops", marker = "extra == 'multihmr'", specifier = ">=0.8" },
@@ -170,6 +203,7 @@ requires-dist = [
    { name = "omegaconf", marker = "extra == 'detrpose'", specifier = ">=2.3" },
    { name = "open3d", marker = "extra == 'lidar'", specifier = ">=0.18,<0.20" },
    { name = "opencv-python", marker = "extra == 'detrpose'", specifier = ">=4.10" },
+    { name = "opencv-python", marker = "extra == 'iphone-usb'", specifier = ">=4.10" },
    { name = "opencv-python", marker = "extra == 'multihmr'", specifier = ">=4.10" },
    { name = "opencv-python", marker = "extra == 'nlf'", specifier = ">=4.10" },
    { name = "opencv-python", marker = "extra == 'pose'", specifier = ">=4.10" },
@@ -210,7 +244,7 @@ requires-dist = [
    { name = "xtcocotools", marker = "extra == 'detrpose'", specifier = ">=1.14" },
    { name = "yacs", marker = "extra == 'smplerx'", specifier = ">=0.1.8" },
 ]
-provides-extras = ["pose", "detrpose", "lidar", "nlf", "multihmr", "smplerx"]
+provides-extras = ["pose", "detrpose", "lidar", "nlf", "multihmr", "iphone-usb", "smplerx"]

 [package.metadata.requires-dev]
 dev = [{ name = "pytest", specifier = ">=9.0.3" }]