diff --git a/data_only_viz/iphone_usb_source.py b/data_only_viz/iphone_usb_source.py index 05184d9..9bdf9ec 100644 --- a/data_only_viz/iphone_usb_source.py +++ b/data_only_viz/iphone_usb_source.py @@ -5,11 +5,15 @@ State. Substitutes for cv2.VideoCapture in multi.py under --iphone-usb.""" from __future__ import annotations import logging +import socket import struct import threading import time -import av +try: + import av +except ImportError: # pragma: no cover — av missing is caught at runtime by caller + av = None # type: ignore[assignment] import cv2 import numpy as np @@ -23,6 +27,7 @@ TAG_HANDS = 4 LOG = logging.getLogger("iphone_usb_source") _HAND_BYTES = 1 + 21 * 12 # chirality:u8 + 21 × (x,y,z) × 4 bytes BE f32 +_ARKIT_JOINTS = 91 def _make_point(x: float, y: float, z: float) -> PoseKp: @@ -67,14 +72,17 @@ def _to_annexb(data: bytes) -> bytes: class IphoneUSBSource: - def __init__(self, state=None, target_size=(640, 480)) -> None: + def __init__(self, state=None, target_size=(640, 480), + write_hands: bool = True) -> None: self.state = state self.target_w, self.target_h = target_size - self._codec = av.codec.CodecContext.create("hevc", "r") + self._write_hands = write_hands + self._codec = av.codec.CodecContext.create("hevc", "r") if av is not None else None self._lock = threading.Lock() self._frame = None # latest BGR np.ndarray self._stop = threading.Event() self._thread = None + self._sock = None # stored for shutdown() in release() self._opened = False def start(self) -> bool: @@ -82,6 +90,7 @@ class IphoneUSBSource: if sock is None: LOG.error("iphone usb: no device / connect failed") return False + self._sock = sock self._opened = True self._thread = threading.Thread( target=self._run, args=(sock,), name="iphone_usb_src", daemon=True) @@ -98,27 +107,30 @@ class IphoneUSBSource: break if tag == TAG_VIDEO and len(payload) > 1: annexb = _to_annexb(payload[1:]) - try: - for fr in self._codec.decode(av.Packet(annexb)): - img = fr.to_ndarray(format="bgr24") - img = cv2.resize(img, (self.target_w, self.target_h)) - with self._lock: - self._frame = img - except av.AVError as e: - LOG.debug("hevc decode: %s", e) + if self._codec is not None and av is not None: + try: + for fr in self._codec.decode(av.Packet(annexb)): + img = fr.to_ndarray(format="bgr24") + img = cv2.resize(img, (self.target_w, self.target_h)) + with self._lock: + self._frame = img + except Exception as e: # av.AVError is a removal-prone alias + LOG.debug("hevc decode: %s", e) elif tag == TAG_SKELETON and self.state is not None: joints = decode_skeleton(payload) if joints is not None: - arr = np.array( - [[j[0], j[1], j[2]] for j in joints], - dtype=np.float32, - ) with self.state.lock(): + arr = self.state.persons_arkit_joints.get(pid) + if arr is None or arr.shape != (_ARKIT_JOINTS, 3): + arr = np.zeros((_ARKIT_JOINTS, 3), dtype=np.float32) + for i, (x, y, z, valid) in enumerate(joints): + if valid: + arr[i] = (x, y, z) self.state.persons_arkit_joints[pid] = arr self.state.persons_arkit_last_t[pid] = time.perf_counter() - elif tag == TAG_HANDS and self.state is not None: + elif tag == TAG_HANDS: hands = _decode_hands(payload) - if hands is not None: + if hands is not None and self._write_hands and self.state is not None: with self.state.lock(): self.state.persons_hands = hands except OSError as e: @@ -132,7 +144,7 @@ class IphoneUSBSource: def read(self): with self._lock: - if self._frame is None: + if self._frame is None or not self._opened: return False, None return True, self._frame.copy() @@ -141,6 +153,11 @@ class IphoneUSBSource: def release(self) -> None: self._stop.set() + if self._sock is not None: + try: + self._sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass if self._thread is not None: self._thread.join(timeout=2.0) self._opened = False diff --git a/data_only_viz/multi.py b/data_only_viz/multi.py index dd78cb0..b103b34 100644 --- a/data_only_viz/multi.py +++ b/data_only_viz/multi.py @@ -346,9 +346,11 @@ class MultiWorker: if self.iphone_usb: from .iphone_usb_source import IphoneUSBSource # noqa: PLC0415 - cap = IphoneUSBSource(self.state) - cap.start() - LOG.info("multi — iphone USB source") + cap = IphoneUSBSource(self.state, write_hands=False) + if not cap.start(): + LOG.error("iphone USB source unavailable (app running? phone unlocked?)") + return + LOG.info("iphone USB source") else: cap = cv2.VideoCapture(self.camera_index) cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) diff --git a/data_only_viz/pyproject.toml b/data_only_viz/pyproject.toml index b21cda2..f1b811b 100644 --- a/data_only_viz/pyproject.toml +++ b/data_only_viz/pyproject.toml @@ -73,6 +73,10 @@ multihmr = [ # via git submodule third_party/SMPLer-X (fork electron-rare). # mmcv-lite suffit pour Config (le repo vendorise sa propre mmpose). # Le detector mmdet est remplace par YOLO Ultralytics (extras pose). +iphone-usb = [ + "av>=12.0", + "opencv-python>=4.10", +] smplerx = [ "torch>=2.4", "torchvision>=0.19", diff --git a/data_only_viz/tests/test_iphone_usb_source.py b/data_only_viz/tests/test_iphone_usb_source.py new file mode 100644 index 0000000..a470312 --- /dev/null +++ b/data_only_viz/tests/test_iphone_usb_source.py @@ -0,0 +1,124 @@ +"""Pure (no-device) unit tests for iphone_usb_source helpers. + +Tests _to_annexb and _decode_hands without any hardware or network connection. +""" +import struct + +import pytest + +from data_only_viz.iphone_usb_source import _to_annexb, _decode_hands, _HAND_BYTES + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + +def _avcc(nals: list[bytes]) -> bytes: + """Build an AVCC-style buffer with 4-byte big-endian length prefixes.""" + out = bytearray() + for nal in nals: + out += struct.pack(">I", len(nal)) + nal + return bytes(out) + + +def _make_hands_payload(hands: list[list[tuple[float, float, float]]]) -> bytes: + """Build a synthetic HandsPayload matching the wire layout. + + count:u8, then per hand: chirality:u8 (1=right) + 21 × (x,y,z) big-endian f32. + """ + out = bytearray() + out += bytes([len(hands)]) + for hand_kps in hands: + out += bytes([1]) # chirality = right + for x, y, z in hand_kps: + out += struct.pack(">fff", x, y, z) + return bytes(out) + + +# --------------------------------------------------------------------------- +# _to_annexb +# --------------------------------------------------------------------------- + +def test_to_annexb_two_nals(): + nal1 = b"\x67\x01\x02\x03" # SPS-like + nal2 = b"\x41\xAB\xCD" + avcc = _avcc([nal1, nal2]) + result = _to_annexb(avcc) + # First unit: Annex-B start code + nal1 + assert result[:4] == b"\x00\x00\x00\x01" + assert result[4:4 + len(nal1)] == nal1 + # Second unit: Annex-B start code + nal2 + rest = result[4 + len(nal1):] + assert rest[:4] == b"\x00\x00\x00\x01" + assert rest[4:4 + len(nal2)] == nal2 + assert len(result) == 4 + len(nal1) + 4 + len(nal2) + + +def test_to_annexb_payload_bytes_preserved(): + nal = bytes(range(32)) + result = _to_annexb(_avcc([nal])) + assert result[4:] == nal + + +def test_to_annexb_empty(): + assert _to_annexb(b"") == b"" + + +def test_to_annexb_truncated_nal_is_dropped(): + # Length prefix claims 10 bytes but only 3 are present + buf = struct.pack(">I", 10) + b"\x00" * 3 + result = _to_annexb(buf) + assert result == b"" + + +def test_to_annexb_single_nal(): + nal = b"\x65\xB8" + result = _to_annexb(_avcc([nal])) + assert result == b"\x00\x00\x00\x01" + nal + + +# --------------------------------------------------------------------------- +# _decode_hands +# --------------------------------------------------------------------------- + +def test_decode_hands_empty_payload(): + assert _decode_hands(b"") is None + + +def test_decode_hands_count_zero(): + assert _decode_hands(bytes([0])) == [] + + +def test_decode_hands_two_hands_length_and_coords(): + hand_a = [(float(i) * 0.1, float(i) * 0.2, float(i) * 0.05) for i in range(21)] + hand_b = [(1.0 - float(i) * 0.03, 0.5, float(i) * 0.01) for i in range(21)] + payload = _make_hands_payload([hand_a, hand_b]) + result = _decode_hands(payload) + assert isinstance(result, list) + assert len(result) == 2 + for hand in result: + assert len(hand) == 21 + # x/y spot-checks for hand A + assert abs(result[0][0].x - hand_a[0][0]) < 1e-5 + assert abs(result[0][5].y - hand_a[5][1]) < 1e-5 + # x/y spot-checks for hand B + assert abs(result[1][3].x - hand_b[3][0]) < 1e-5 + assert abs(result[1][20].y - hand_b[20][1]) < 1e-5 + + +def test_decode_hands_one_hand(): + kps = [(0.1 * i, 0.2 * i, 0.0) for i in range(21)] + result = _decode_hands(_make_hands_payload([kps])) + assert isinstance(result, list) + assert len(result) == 1 + assert len(result[0]) == 21 + + +def test_decode_hands_truncated_does_not_raise(): + hand_kps = [(0.1, 0.2, 0.3)] * 21 + full = _make_hands_payload([hand_kps, hand_kps]) + # Truncate to half — second hand will be incomplete + truncated = full[:len(full) // 2] + # Must not raise + result = _decode_hands(truncated) + assert isinstance(result, (list, type(None))) diff --git a/data_only_viz/uv.lock b/data_only_viz/uv.lock index 3e11e2e..e98f6c6 100644 --- a/data_only_viz/uv.lock +++ b/data_only_viz/uv.lock @@ -71,6 +71,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "av" +version = "17.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/e3/477fa20578c284abeda08d91b63ee9abaebc93445d8feeb989d3d444bae1/av-17.1.0.tar.gz", hash = "sha256:7f1e71ff621b66253333926f948e00faae11d855b2442133c65128bca64cdeb3", size = 4288546, upload-time = "2026-06-07T05:52:55.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/87/8036b5c781bc3639ea04ef42d4e26da253bd4bd4311d8705b6a1c8824047/av-17.1.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:ad7b4aa011093324b7118245f50ac6db244cfe9900d4072508a5245a2b0d3f41", size = 22460847, upload-time = "2026-06-07T05:52:04.261Z" }, + { url = "https://files.pythonhosted.org/packages/6d/af/dfdf6fc7b17814b50d0aa9e7a7e37b87be91be3890f44b0d525433cd1fd1/av-17.1.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:43ebbe977f19a7f2d2bd1a4e119675a0b15e05852cf7309846b6ab922ba7ffe9", size = 18159115, upload-time = "2026-06-07T05:52:06.64Z" }, + { url = "https://files.pythonhosted.org/packages/ad/13/64f6c466471cea225b8b2f4cdc51a571f8a286984b55a08d169b932fda5d/av-17.1.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6a20658ec7d96a70e14b1196eff00b7cdd8831ac3b99868e16b8ba8b24090847", size = 33224427, upload-time = "2026-06-07T05:52:09.165Z" }, + { url = "https://files.pythonhosted.org/packages/77/43/96b35170bf2e64e00a41748c6400ff73232dc0fc62ded283679fb07c7fe0/av-17.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f9a65d1f48b818323fb411e80358f89d77dec340b01d27c6b2dfbb9cbf4b779f", size = 35370183, upload-time = "2026-06-07T05:52:11.959Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b3/8e8b4b6498731bfbd88e8399a756543f8088f1bd33d08eab678b5aebe728/av-17.1.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:58f7593726437cda5bd19793027e027768450b5c4a594777bf487798a33db702", size = 24459265, upload-time = "2026-06-07T05:52:14.66Z" }, + { url = "https://files.pythonhosted.org/packages/14/ac/ceb84b7553db21f1143d817245c560d9267168e1e58b1a8eeae2b62c4d04/av-17.1.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bbab058bd965309f39962e53caac8126987c68c0be094fc4f9427e5615b0218f", size = 34283709, upload-time = "2026-06-07T05:52:17.389Z" }, + { url = "https://files.pythonhosted.org/packages/59/f9/4115fd84148c9a1cf365096694be6ac882fd3cd3cdb7a2f35e71fecf1631/av-17.1.0-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:9514cfda85180554c430695282faf4be3ffdf95775d8519733821244eecb58e0", size = 25397573, upload-time = "2026-06-07T05:52:20.012Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ac/92e52d5ed0e0b84d9d93e52b4338c2713d8a44082b8696e6516fdae7c4e4/av-17.1.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e1c90f85cd7431ede95b11e8e711571a896ebea433f298849c2c0f1594c8d86e", size = 36451495, upload-time = "2026-06-07T05:52:22.581Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f2/53a7cd34adb6a971d7e6d99663e74db286966c9db8afdca17472fdf0f98e/av-17.1.0-cp311-abi3-win_amd64.whl", hash = "sha256:5df5c1172ef1cf65a1529d612f7da7798ce2cf82c1ff7212466b538a6cc7214c", size = 28036393, upload-time = "2026-06-07T05:52:25.657Z" }, + { url = "https://files.pythonhosted.org/packages/66/47/cd9ae0edf2206351c1251bb94b5ec58728e42c5f6ee16c03c412f3a1bb3e/av-17.1.0-cp311-abi3-win_arm64.whl", hash = "sha256:ee98534242a74da847af78624779ac5a3177dc7c69f956a4da9e6f0fdb37d7f6", size = 21174601, upload-time = "2026-06-07T05:52:28.077Z" }, + { url = "https://files.pythonhosted.org/packages/36/90/b5668cddb3c401fcf22553bc495d5b0c6d8a01d118624b26f0db1d0b8653/av-17.1.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:5327807c1219293803ef0c5d1578ff3ae1cf638c09e5998962026e1a554ec240", size = 22699499, upload-time = "2026-06-07T05:52:30.335Z" }, + { url = "https://files.pythonhosted.org/packages/e0/7e/7be6bfddb823d045ff9fd5d4deb922ee3847605e162c3882e6c45b4c35ff/av-17.1.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:6c9b71fe5c0c5a8d303b1588d4d8ce9397d6b023f467cfef95000ba1f75507fa", size = 18366696, upload-time = "2026-06-07T05:52:32.645Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/391dcfa75c1ae1977efca44b753a11b929399b558826670c16a8808dd0e3/av-17.1.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f997e3351bdf51127c07a74e21741a2996e9230cbeb2d81c14acde761b116c9c", size = 36582649, upload-time = "2026-06-07T05:52:35.218Z" }, + { url = "https://files.pythonhosted.org/packages/fb/32/7312854868b318b9d1b1dcbd1bddb460aaaeac7d57f816e11efec3bef5b1/av-17.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:efe9b1397300b67b644ad220c89df4892a76f2debe70f16bae1749fa20526e63", size = 38479390, upload-time = "2026-06-07T05:52:37.968Z" }, + { url = "https://files.pythonhosted.org/packages/2a/72/af47f59b4458e81ca7d89f477698dbfb3d5a0cd8ae6c1e4441d01074af8a/av-17.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:fa64e1f1500d01c4a98e7a41dc1a9a35fb4dfe71f5de0389264ec1192200c76a", size = 27127432, upload-time = "2026-06-07T05:52:40.371Z" }, + { url = "https://files.pythonhosted.org/packages/88/85/c2e6861baf0f8c7d21c4ce811d4d424fedac915e3910d3570ce4377717dc/av-17.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ffbd78d73d2c9bf31e9a007c992faec3991428b2941a3b085b84fb82e8c32d19", size = 37406592, upload-time = "2026-06-07T05:52:43.215Z" }, + { url = "https://files.pythonhosted.org/packages/ba/40/3cc13125aea976101c0858af99ac47257c0654411aa199b5d8e81eea7002/av-17.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:bff8896454b38fcb785a70e5ae0485d7021cb776303a5849393128a30b8f850b", size = 28336228, upload-time = "2026-06-07T05:52:46.134Z" }, + { url = "https://files.pythonhosted.org/packages/a2/38/c7d9c3e746209a1a695c13e3aa7d817229e84a85d0a84271f313d1befdd3/av-17.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1284addf3c0dd939887a9722dc30df2241a97471ad52c3c507e31583ae22ff02", size = 39490680, upload-time = "2026-06-07T05:52:48.887Z" }, + { url = "https://files.pythonhosted.org/packages/a1/25/9d42da561b7b8f7dabdfaebba07b52977bee58c5c7e4285ac991abcfaa72/av-17.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:ec630be6321b04e317862f6082e84812bbd801e55a3c2298312e3fc8a0a4af4f", size = 28355673, upload-time = "2026-06-07T05:52:51.614Z" }, + { url = "https://files.pythonhosted.org/packages/a8/41/562a61d5a61fba3ffb273a115e249f1d8471b9515c59fcc38b4b9deda238/av-17.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b41647e42884bf543b8e8d0a1dabd4d1b006c99183eb1a2d7afc5b01f73eeff4", size = 21324700, upload-time = "2026-06-07T05:52:53.972Z" }, +] + [[package]] name = "av-live-data-only-viz" version = "0.1.0" @@ -100,6 +128,10 @@ detrpose = [ { name = "transformers" }, { name = "xtcocotools" }, ] +iphone-usb = [ + { name = "av" }, + { name = "opencv-python" }, +] lidar = [ { name = "open3d" }, ] @@ -154,6 +186,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "av", marker = "extra == 'iphone-usb'", specifier = ">=12.0" }, { name = "cloudpickle", marker = "extra == 'detrpose'", specifier = ">=3.0" }, { name = "coremltools", marker = "extra == 'pose'", specifier = ">=9.0" }, { name = "einops", marker = "extra == 'multihmr'", specifier = ">=0.8" }, @@ -170,6 +203,7 @@ requires-dist = [ { name = "omegaconf", marker = "extra == 'detrpose'", specifier = ">=2.3" }, { name = "open3d", marker = "extra == 'lidar'", specifier = ">=0.18,<0.20" }, { name = "opencv-python", marker = "extra == 'detrpose'", specifier = ">=4.10" }, + { name = "opencv-python", marker = "extra == 'iphone-usb'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'multihmr'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'nlf'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'pose'", specifier = ">=4.10" }, @@ -210,7 +244,7 @@ requires-dist = [ { name = "xtcocotools", marker = "extra == 'detrpose'", specifier = ">=1.14" }, { name = "yacs", marker = "extra == 'smplerx'", specifier = ">=0.1.8" }, ] -provides-extras = ["pose", "detrpose", "lidar", "nlf", "multihmr", "smplerx"] +provides-extras = ["pose", "detrpose", "lidar", "nlf", "multihmr", "iphone-usb", "smplerx"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.3" }]