From 0a2865e136eb52e8724763b5b7af2514f14dae6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:56:08 +0200 Subject: [PATCH] feat(viz): left/right hand front-view panels Add panel_rect/panel_segments/panel_frame helpers to hand_display.py (PANEL_SIDE=0.30, PANEL_MARGIN=0.02, PANEL_INNER=0.10). Segments use uniform pixel-space scale so the hand aspect ratio is preserved regardless of view aspect. mirror=True flips X within the panel. In renderer._update_skeleton add push_panel (no mirror flip) and a panels block before the segs==0 epilogue: routes persons_hands to left/right panel via chirality list when aligned, screen-cx fallback otherwise. Draws frame (pid 7) + wireframe (pid 5 left / 6 right). Runs on every path including use_arkit. Guards: _mp_bones None, empty hands, misaligned chirality, height==0. 26 tests pass (10 new for panel helpers). --- data_only_viz/hand_display.py | 160 ++++++++++++++++++++- data_only_viz/renderer.py | 67 ++++++++- data_only_viz/tests/test_hand_display.py | 171 ++++++++++++++++++++++- 3 files changed, 395 insertions(+), 3 deletions(-) diff --git a/data_only_viz/hand_display.py b/data_only_viz/hand_display.py index 0082d8f..8a5f213 100644 --- a/data_only_viz/hand_display.py +++ b/data_only_viz/hand_display.py @@ -1,4 +1,4 @@ -"""Pure geometry helpers for hand overlay filtering. +"""Pure geometry helpers for hand overlay filtering and panel rendering. No Metal, no numpy, no pyobjc — safe to unit-test anywhere. """ @@ -8,6 +8,22 @@ import math from typing import Protocol +# --------------------------------------------------------------------------- +# Panel layout constants +# --------------------------------------------------------------------------- + +#: Panel square size in normalized HEIGHT units. A panel occupies +#: PANEL_SIDE * view_height pixels on both axes (square in pixel space). +PANEL_SIDE: float = 0.30 + +#: Gap between the screen edge and the near edge of the panel (normalized). +PANEL_MARGIN: float = 0.02 + +#: Fraction of each panel dimension removed from every side to form the +#: inner rect used as the drawing area for hand wireframes. +PANEL_INNER: float = 0.10 + + class _HasXYC(Protocol): x: float y: float @@ -64,3 +80,145 @@ def segment_ok( if bone_len > max_bone_ratio * size: return False return True + + +# --------------------------------------------------------------------------- +# Panel helpers +# --------------------------------------------------------------------------- + +def panel_rect(side: str, aspect: float) -> tuple[float, float, float, float]: + """Normalized [0,1] screen rect of the hand side panel. + + The panel is square in pixel space: + pixel_width = norm_width * view_width = (PANEL_SIDE / aspect) * aspect * view_height + = PANEL_SIDE * view_height = pixel_height. + + Args: + side: "left" or "right". + aspect: view width / view height (> 0). + + Returns: + (x0, y0, x1, y1) in normalized [0,1] coords, y down, origin top-left. + """ + norm_w = PANEL_SIDE / aspect # normalized width (square in pixel space) + norm_h = PANEL_SIDE # normalized height + y0 = 0.5 - norm_h / 2.0 + y1 = 0.5 + norm_h / 2.0 + if side == "left": + x0 = PANEL_MARGIN + x1 = PANEL_MARGIN + norm_w + else: # "right" + x1 = 1.0 - PANEL_MARGIN + x0 = x1 - norm_w + return x0, y0, x1, y1 + + +def panel_frame(side: str, aspect: float) -> list[tuple[float, float, float, float]]: + """Four border segments tracing the panel rect (clock-wise, starting top). + + Returns: + List of (ax, ay, bx, by) in normalized [0,1] screen coords, y down. + """ + x0, y0, x1, y1 = panel_rect(side, aspect) + return [ + (x0, y0, x1, y0), # top edge (left→right) + (x1, y0, x1, y1), # right edge (top→bottom) + (x1, y1, x0, y1), # bottom edge (right→left) + (x0, y1, x0, y0), # left edge (bottom→top) + ] + + +def panel_segments( + kp: list[_HasXYC], + side: str, + bones: list[tuple[int, int]], + aspect: float, + mirror: bool = True, +) -> list[tuple[float, float, float, float]]: + """Map hand landmarks into a side panel using uniform pixel-space scale. + + The hand bounding box is fit into the inner panel rect (panel rect shrunk + by PANEL_INNER fraction on each side) with a uniform pixel-space scale — + meaning the pixel aspect ratio of the hand is preserved regardless of the + view aspect ratio. + + Scale derivation: + s = min(inner_pixel_w / bbox_pixel_w, inner_pixel_h / bbox_pixel_h) + = min(iw / kw, ih / kh) + where iw/kw are normalized-coord ratios whose view dimensions cancel. + Moving Δkp.x in kp-x corresponds to Δkp.x * s in normalized screen x, and + Δkp.y in kp-y corresponds to Δkp.y * s in normalized screen y — preserving + the original pixel ratio between any two segment endpoints. + + When mirror=True, X is flipped within the panel so the zoomed hand matches + the mirrored video background (i.e. the panel always shows a "front view"). + + Args: + kp: 21+ keypoints with .x, .y (normalized image coords, y down), .c. + side: "left" or "right". + bones: List of (index_a, index_b) pairs defining which keypoints to connect. + aspect: view width / view height. + mirror: If True, flip X within the panel. + + Returns: + List of (ax, ay, bx, by) in normalized [0,1] screen coords, y down. + Returns [] if hand_plausible(kp) fails. + """ + if not hand_plausible(kp): + return [] + + x0, y0, x1, y1 = panel_rect(side, aspect) + pw = x1 - x0 + ph = y1 - y0 + + # Inner rect: shrink by PANEL_INNER fraction on every side + ix0 = x0 + PANEL_INNER * pw + ix1 = x1 - PANEL_INNER * pw + iy0 = y0 + PANEL_INNER * ph + iy1 = y1 - PANEL_INNER * ph + iw = ix1 - ix0 + ih = iy1 - iy0 + + # Keypoint bounding box in normalized image coords + kx_min = min(p.x for p in kp) + kx_max = max(p.x for p in kp) + ky_min = min(p.y for p in kp) + ky_max = max(p.y for p in kp) + kw = kx_max - kx_min + kh = ky_max - ky_min + + if kw == 0.0 and kh == 0.0: + return [] + + # Uniform pixel-space scale factor (view dimensions cancel in the ratio) + if kw > 0.0 and kh > 0.0: + s = min(iw / kw, ih / kh) + elif kw > 0.0: + s = iw / kw + else: + s = ih / kh + + # Centre the scaled hand bbox inside the inner rect + offset_x = (iw - kw * s) / 2.0 + offset_y = (ih - kh * s) / 2.0 + + def _map(p: _HasXYC) -> tuple[float, float]: + rel_x = (kx_max - p.x) if mirror else (p.x - kx_min) + nx = ix0 + offset_x + rel_x * s + ny = iy0 + offset_y + (p.y - ky_min) * s + return nx, ny + + sz = hand_size(kp) + result: list[tuple[float, float, float, float]] = [] + for a, b in bones: + if a >= len(kp) or b >= len(kp): + continue + A = kp[a] + B = kp[b] + if not segment_ok(A, B, sz): + continue + ax, ay = _map(A) + bx, by = _map(B) + result.append((ax, ay, bx, by)) + + return result diff --git a/data_only_viz/renderer.py b/data_only_viz/renderer.py index 345fddc..e11027b 100644 --- a/data_only_viz/renderer.py +++ b/data_only_viz/renderer.py @@ -44,7 +44,7 @@ from Metal import ( from MetalKit import MTKView # noqa: F401 (utilise par d'autres modules) from .arkit_skeleton import arkit_segments -from .hand_display import hand_plausible, hand_size, segment_ok +from .hand_display import hand_plausible, hand_size, segment_ok, panel_frame, panel_segments from .mesh_topology import ( BODY_TRIANGLES, FACE_TRIANGLES, @@ -363,6 +363,25 @@ class MetalRenderer(NSObject): segs += 1 return True + def push_panel(ax, ay, bx, by, conf, pid): + """Like push_seg but NEVER applies mirror. + + Panel coords are final screen coords [0,1] — they must not be + flipped again even when the video background is mirrored. + """ + nonlocal segs + if segs >= SKEL_MAX_SEGS: + return False + cax = ax * 2.0 - 1.0; cay = 1.0 - ay * 2.0 + cbx = bx * 2.0 - 1.0; cby = 1.0 - by * 2.0 + i = segs * 10 + buf[i+0] = cax; buf[i+1] = cay; buf[i+2] = 0.0 + buf[i+3] = conf; buf[i+4] = float(pid) + buf[i+5] = cbx; buf[i+6] = cby; buf[i+7] = 0.0 + buf[i+8] = conf; buf[i+9] = float(pid) + segs += 1 + return True + if use_arkit: parents = s.arkit_parents for pid, arr2d in s.persons_arkit_2d.items(): @@ -436,6 +455,52 @@ class MetalRenderer(NSObject): if A.c < 0.2 or B.c < 0.2: continue if not push(A, B, min(A.c, B.c), 0): break + # ---- SIDE PANELS: left/right hand front-view zoomed wireframe ---- + # Runs on every path (arkit + mediapipe + fallback COCO). + # Hands always come from s.persons_hands regardless of use_arkit. + if self._mp_bones is not None: + _body_b, _face_b, lhand_bones_p, _ = self._mp_bones + hands = s.persons_hands + chir = getattr(s, "persons_hands_chirality", None) or [] + asp = float(s.width) / float(s.height) if s.height else 1.0 + + # Route hands to left/right panels via chirality when available, + # otherwise by screen position (mirror-aware). + left_kp = None + right_kp = None + if chir and len(chir) == len(hands): + # chirality: 0 = left hand, 1 = right hand + for h_kp, c in zip(hands, chir): + if c == 0 and left_kp is None: + left_kp = h_kp + elif c == 1 and right_kp is None: + right_kp = h_kp + else: + # Fallback: order plausible hands by screen cx, mirror-aware. + # Screen left has low cx normally; high cx when video is mirrored. + plausible = [ + (h_kp, sum(p.x for p in h_kp) / len(h_kp)) + for h_kp in hands + if hand_plausible(h_kp) + ] + plausible.sort(key=lambda t: -t[1] if mirror else t[1]) + if plausible: + left_kp = plausible[0][0] + if len(plausible) >= 2: + right_kp = plausible[1][0] + + # Draw frame (pid 7) then wireframe (pid 5 left / 6 right) + for side_name, h_kp, pid_s in ( + ("left", left_kp, 5), + ("right", right_kp, 6), + ): + if h_kp is None: + continue + for seg in panel_frame(side_name, asp): + push_panel(*seg, 1.0, 7) + for seg in panel_segments(h_kp, side_name, lhand_bones_p, asp, mirror=mirror): + push_panel(*seg, 1.0, pid_s) + if segs == 0: return 0 data = self._skel_cpu_buf[: segs * 2 * SKEL_VERT_FLOATS].tobytes() diff --git a/data_only_viz/tests/test_hand_display.py b/data_only_viz/tests/test_hand_display.py index 113987e..a67f9f0 100644 --- a/data_only_viz/tests/test_hand_display.py +++ b/data_only_viz/tests/test_hand_display.py @@ -9,7 +9,17 @@ from dataclasses import dataclass import pytest -from data_only_viz.hand_display import hand_plausible, hand_size, segment_ok +from data_only_viz.hand_display import ( + hand_plausible, + hand_size, + segment_ok, + panel_rect, + panel_segments, + panel_frame, + PANEL_SIDE, + PANEL_MARGIN, + PANEL_INNER, +) # --------------------------------------------------------------------------- @@ -176,3 +186,162 @@ def test_mediapipe_hand_all_bones_pass(): if not segment_ok(kp[i], kp[i + 1], size): failures.append(i) assert failures == [], f"Segments failed segment_ok: {failures}" + + +# --------------------------------------------------------------------------- +# 2D hand helper: spans a proper bbox with both x and y extent +# --------------------------------------------------------------------------- + +def _make_hand_2d( + cx: float = 0.5, + cy: float = 0.5, + half_w: float = 0.05, + half_h: float = 0.05, + c: float = 1.0, +) -> list[Kp]: + """21 landmarks filling bbox [cx±half_w, cy±half_h]. + + kp[0] = wrist at (cx-half_w, cy+half_h) — bottom-left of bbox + kp[9] = MCP at (cx+half_w, cy-half_h) — top-right of bbox + All other kp are spread uniformly across the bbox via a diagonal. + hand_size = dist(kp[0], kp[9]) = 2 * sqrt(half_w² + half_h²). + """ + n = 21 + kps: list[Kp] = [] + for i in range(n): + t = i / (n - 1) + kps.append(Kp( + x=cx - half_w + t * 2 * half_w, + y=cy - half_h + t * 2 * half_h, + z=0.0, + c=c, + )) + # Override wrist and MCP so bbox spans full [cx-half_w..cx+half_w, cy-half_h..cy+half_h] + kps[0] = Kp(x=cx - half_w, y=cy + half_h, z=0.0, c=c) # bottom-left + kps[9] = Kp(x=cx + half_w, y=cy - half_h, z=0.0, c=c) # top-right + return kps + + +# Minimal bone list for panel tests: wrist(0) → middle-MCP(9) +_SIMPLE_BONES: list[tuple[int, int]] = [(0, 9)] + + +# --------------------------------------------------------------------------- +# panel_rect +# --------------------------------------------------------------------------- + +def test_panel_rect_left_x0_anchor(): + """Left panel x0 == PANEL_MARGIN.""" + x0, _y0, _x1, _y1 = panel_rect("left", aspect=1.0) + assert math.isclose(x0, PANEL_MARGIN, rel_tol=1e-6) + + +def test_panel_rect_right_x1_anchor(): + """Right panel x1 == 1 - PANEL_MARGIN.""" + _x0, _y0, x1, _y1 = panel_rect("right", aspect=1.0) + assert math.isclose(x1, 1.0 - PANEL_MARGIN, rel_tol=1e-6) + + +def test_panel_rect_vertically_centered(): + """Panel vertical center is at 0.5.""" + _x0, y0, _x1, y1 = panel_rect("left", aspect=1.0) + assert math.isclose((y0 + y1) / 2.0, 0.5, rel_tol=1e-6) + + +def test_panel_rect_square_in_pixels_landscape(): + """With 16:9 aspect ratio, normalized width × aspect == normalized height.""" + aspect = 16.0 / 9.0 + x0, y0, x1, y1 = panel_rect("left", aspect=aspect) + norm_w = x1 - x0 + norm_h = y1 - y0 + # pixel_w = norm_w * view_w = norm_w * aspect * view_h == norm_h * view_h = pixel_h + assert math.isclose(norm_w * aspect, norm_h, rel_tol=1e-6) + + +# --------------------------------------------------------------------------- +# panel_segments +# --------------------------------------------------------------------------- + +def test_panel_segments_implausible_returns_empty(): + """hand_plausible fails → panel_segments returns [].""" + kp = _make_hand(size=0.005, c=1.0) # size < size_min=0.02 → implausible + result = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False) + assert result == [] + + +def test_panel_segments_inside_inner_rect(): + """All output segment endpoints lie inside the inner panel rect.""" + kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=0.04, half_h=0.08, c=1.0) + x0, y0, x1, y1 = panel_rect("left", aspect=1.0) + pw, ph = x1 - x0, y1 - y0 + ix0 = x0 + PANEL_INNER * pw + ix1 = x1 - PANEL_INNER * pw + iy0 = y0 + PANEL_INNER * ph + iy1 = y1 - PANEL_INNER * ph + segs = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False) + assert segs, "expected at least one segment" + eps = 1e-9 + for ax, ay, bx, by in segs: + assert ix0 - eps <= ax <= ix1 + eps, f"ax={ax} outside [{ix0}, {ix1}]" + assert ix0 - eps <= bx <= ix1 + eps, f"bx={bx} outside [{ix0}, {ix1}]" + assert iy0 - eps <= ay <= iy1 + eps, f"ay={ay} outside [{iy0}, {iy1}]" + assert iy0 - eps <= by <= iy1 + eps, f"by={by} outside [{iy0}, {iy1}]" + + +def test_panel_segments_aspect_preserved(): + """A hand 2× taller than wide in pixel space maps to 2× taller than wide in panel. + + With aspect=1.0 (square screen), pixel ratio == normalized ratio. + The hand has half_h = 2*half_w, so pixel bbox is 2× taller than wide. + After uniform pixel-space scaling the output segment's |Δy|/|Δx| must also be 2. + """ + half_w, half_h = 0.04, 0.08 # kh=0.16, kw=0.08 → ratio 2 + kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=half_w, half_h=half_h, c=1.0) + segs = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False) + assert len(segs) == 1, f"expected 1 segment, got {len(segs)}" + ax, ay, bx, by = segs[0] + dx = abs(bx - ax) + dy = abs(by - ay) + assert dx > 1e-9 and dy > 1e-9, "segment must have both x and y extent" + ratio = dy / dx # pixel ratio (= normalized ratio at aspect=1) + assert math.isclose(ratio, 2.0, rel_tol=1e-3), f"expected pixel ratio 2.0, got {ratio}" + + +def test_panel_segments_mirror_flips_x(): + """mirror=True flips X within the panel relative to mirror=False.""" + kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=0.04, half_h=0.04, c=1.0) + # kp[0] at bottom-left (kx_min), kp[9] at top-right (kx_max) + segs_normal = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False) + segs_mirror = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=True) + assert len(segs_normal) == 1 + assert len(segs_mirror) == 1 + ax0, _ay0, bx0, _by0 = segs_normal[0] + ax1, _ay1, bx1, _by1 = segs_mirror[0] + # kp[0].x < kp[9].x → without mirror: ax < bx; with mirror: ax > bx + assert ax0 < bx0, "without mirror: wrist (kx_min) should project left of MCP (kx_max)" + assert ax1 > bx1, "with mirror: X is flipped so wrist projects right of MCP" + + +# --------------------------------------------------------------------------- +# panel_frame +# --------------------------------------------------------------------------- + +def test_panel_frame_returns_four_segments(): + """panel_frame returns exactly 4 segments.""" + segs = panel_frame("left", aspect=1.0) + assert len(segs) == 4 + + +def test_panel_frame_traces_rect(): + """Every endpoint of each frame segment is a corner of the panel rect.""" + aspect = 1.0 + x0, y0, x1, y1 = panel_rect("left", aspect) + corners = [(x0, y0), (x0, y1), (x1, y0), (x1, y1)] + segs = panel_frame("left", aspect) + for ax, ay, bx, by in segs: + for ex, ey in [(ax, ay), (bx, by)]: + on_corner = any( + math.isclose(ex, cx, abs_tol=1e-9) and math.isclose(ey, cy, abs_tol=1e-9) + for cx, cy in corners + ) + assert on_corner, f"Endpoint ({ex:.4f}, {ey:.4f}) not a corner of panel rect"