From ef1d557eff7613addefeb7547f072fcc51196112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:36:57 +0200 Subject: [PATCH] feat(viz): rotate iphone 2d points, hide fingers Portrait mode validated live: VIDEO_ROTATE now also rotates the iPhone 2D landmarks (skeleton2D + Vision hands) at the decode layer so overlays/panels/gestures stay aligned with the rotated video. ARKit per-finger joints are masked from the wireframe (unreliable 2D, seen as false hands); the display shows the filtered Vision hands instead. --- data_only_viz/arkit_skeleton.py | 17 +++++++++ data_only_viz/iphone_usb_source.py | 37 +++++++++++++++++-- data_only_viz/renderer.py | 15 +++++++- data_only_viz/tests/test_arkit_skeleton.py | 10 +++++ data_only_viz/tests/test_iphone_usb_source.py | 15 ++++++++ 5 files changed, 90 insertions(+), 4 deletions(-) diff --git a/data_only_viz/arkit_skeleton.py b/data_only_viz/arkit_skeleton.py index 5eedc54..9dc72ba 100644 --- a/data_only_viz/arkit_skeleton.py +++ b/data_only_viz/arkit_skeleton.py @@ -50,3 +50,20 @@ def arkit_segments(arr2d, valid, parents, max_len: float = 0.5, bounds: float = continue segs.append((ax, ay, bx, by)) return segs + + +# ARKit's 91-joint body includes per-finger joints whose 2D projection is +# unreliable (the user sees them as "false hands" glued to the skeleton). +# The display uses the filtered Vision hands instead; these joints are +# masked out of the wireframe. The wrist anchors (left_hand_joint / +# right_hand_joint) are kept so the forearm bone still terminates. +_FINGER_TOKENS = ("thumb", "index", "mid", "ring", "little", "pinky") + + +def finger_joint_mask(names: list) -> list[bool]: + """True for each ARKit joint that belongs to a finger (to be hidden).""" + out = [] + for n in names: + ln = str(n).lower() + out.append("hand" in ln and any(t in ln for t in _FINGER_TOKENS)) + return out diff --git a/data_only_viz/iphone_usb_source.py b/data_only_viz/iphone_usb_source.py index 3f64fd6..0321e63 100644 --- a/data_only_viz/iphone_usb_source.py +++ b/data_only_viz/iphone_usb_source.py @@ -33,6 +33,24 @@ _HAND_BYTES = 1 + 21 * 12 # chirality:u8 + 21 × (x,y,z) × 4 bytes BE f32 _ARKIT_JOINTS = 91 +# Normalized-coordinate counterpart of multi._apply_video_rotate (np.rot90 +# conventions): the iPhone computes 2D landmarks on the UNROTATED sensor +# frame, so when VIDEO_ROTATE turns the displayed video the 2D points must +# turn the same way or overlays/panels/gestures go misaligned. ARKit 3D +# world joints are gravity-aligned and unaffected. +_ROT_XY = { + "ccw": lambda x, y: (y, 1.0 - x), + "cw": lambda x, y: (1.0 - y, x), + "180": lambda x, y: (1.0 - x, 1.0 - y), +} + + +def rotate_norm_xy(x: float, y: float, mode: str) -> "tuple[float, float]": + """Rotate a normalized (x, y) like np.rot90 rotates the frame.""" + f = _ROT_XY.get(mode) + return f(x, y) if f else (x, y) + + def _make_point(x: float, y: float, z: float, conf: float) -> PoseKp: """Wrap raw floats from the wire into a PoseKp landmark. @@ -128,7 +146,12 @@ class IphoneUSBSource: # interacts naturally (move right -> goes right; raise right arm -> # the right side responds). Env CONCERT_MIRROR=0 disables it. from .config import VizConfig as _VizConfig - self._mirror = mirror and _VizConfig.from_env().concert_mirror + _cfg = _VizConfig.from_env() + self._mirror = mirror and _cfg.concert_mirror + # VIDEO_ROTATE also applies to the iPhone 2D landmarks (skeleton2D + + # hands) so they stay aligned with the rotated video (multi.py only + # rotates the frame). + self._video_rotate = _cfg.video_rotate or "none" self._codec = av.codec.CodecContext.create("hevc", "r") if av is not None else None self._lock = threading.Lock() self._frame = None # latest BGR np.ndarray @@ -197,8 +220,11 @@ class IphoneUSBSource: elif tag == TAG_SKELETON2D and self.state is not None: pts = decode_skeleton2D(payload) if pts is not None: - arr = np.array([[x, y] for (x, y, _v) in pts], - dtype=np.float32) + _rot = self._video_rotate + arr = np.array( + [rotate_norm_xy(x, y, _rot) + for (x, y, _v) in pts], + dtype=np.float32) valid = np.array([v for (_x, _y, v) in pts], dtype=bool) with self.state.lock(): @@ -217,6 +243,11 @@ class IphoneUSBSource: result = _decode_hands(payload) if result is not None and self.state is not None: hands, chirality = result + if self._video_rotate != "none": + for hand in hands: + for p in hand: + p.x, p.y = rotate_norm_xy( + p.x, p.y, self._video_rotate) with self.state.lock(): # Always expose iPhone Vision hands (stable, # rotation-invariant) for the air-piano. diff --git a/data_only_viz/renderer.py b/data_only_viz/renderer.py index 6a6080e..1efb550 100644 --- a/data_only_viz/renderer.py +++ b/data_only_viz/renderer.py @@ -43,7 +43,7 @@ from Metal import ( # pyobjc detecte automatiquement l'implementation par signature. from MetalKit import MTKView # noqa: F401 (utilise par d'autres modules) -from .arkit_skeleton import arkit_segments +from .arkit_skeleton import arkit_segments, finger_joint_mask from .hand_display import ( HandPersistenceGate, arkit_2d_fresh, @@ -408,11 +408,24 @@ class MetalRenderer(NSObject): if use_arkit: parents = s.arkit_parents + # Hide ARKit's per-finger joints ("false hands"): the display + # shows the filtered Vision hands instead. Mask cached until the + # topology changes. + fmask = getattr(self, "_finger_mask", None) + if fmask is None or len(fmask) != len(s.arkit_joint_names): + fmask = np.asarray( + finger_joint_mask(s.arkit_joint_names), dtype=bool) + self._finger_mask = fmask for pid, arr2d in s.persons_arkit_2d.items(): ts = s.persons_arkit_2d_t.get(pid) if ts is None or (_arkit_now - ts) >= 1.0: continue # skip stale or unknown pid valid = s.persons_arkit_2d_valid.get(pid) + if len(fmask): + if valid is None: + valid = ~fmask + elif len(valid) == len(fmask): + valid = valid & ~fmask for (ax, ay, bx, by) in arkit_segments( arr2d, valid, parents, max_len=self._arkit_bone_max ): diff --git a/data_only_viz/tests/test_arkit_skeleton.py b/data_only_viz/tests/test_arkit_skeleton.py index 7a14525..354c074 100644 --- a/data_only_viz/tests/test_arkit_skeleton.py +++ b/data_only_viz/tests/test_arkit_skeleton.py @@ -93,3 +93,13 @@ def test_segments_legacy_diagonal_bone_now_filtered(): assert arkit_segments(arr2d, valid, parents) == [] # ...and the env-tunable knob restores it for close-up shots. assert len(arkit_segments(arr2d, valid, parents, max_len=0.8)) == 1 + + +def test_finger_joint_mask_hides_fingers_keeps_wrist(): + from data_only_viz.arkit_skeleton import finger_joint_mask + names = ["root", "left_hand_joint", "left_handIndexStart_joint", + "left_handThumb1_joint", "right_handMid2_joint", + "right_handPinkyEnd_joint", "head_joint", "right_hand_joint", + "left_handRing3_joint"] + mask = finger_joint_mask(names) + assert mask == [False, False, True, True, True, True, False, False, True] diff --git a/data_only_viz/tests/test_iphone_usb_source.py b/data_only_viz/tests/test_iphone_usb_source.py index 5b31a3a..da9b88c 100644 --- a/data_only_viz/tests/test_iphone_usb_source.py +++ b/data_only_viz/tests/test_iphone_usb_source.py @@ -286,3 +286,18 @@ def test_apply_skeleton_joints_does_not_mutate_prev(): # arr1 must be byte-for-byte identical to what it was before the call np.testing.assert_array_equal(arr1, arr1_copy) + + +# --------------------------------------------------------------------------- +# VIDEO_ROTATE normalized-point counterpart +# --------------------------------------------------------------------------- + +def test_rotate_norm_xy_matches_rot90(): + from data_only_viz.iphone_usb_source import rotate_norm_xy + # np.rot90 k=1 (ccw): original (x, y) lands at (y, 1-x) + assert rotate_norm_xy(0.2, 0.7, "ccw") == pytest.approx((0.7, 0.8)) + # cw (k=3): (x, y) -> (1-y, x) + assert rotate_norm_xy(0.2, 0.7, "cw") == pytest.approx((0.3, 0.2)) + assert rotate_norm_xy(0.2, 0.7, "180") == pytest.approx((0.8, 0.3)) + assert rotate_norm_xy(0.2, 0.7, "none") == (0.2, 0.7) + assert rotate_norm_xy(0.2, 0.7, "garbage") == (0.2, 0.7)