feat(viz): left/right hand front-view panels
CI build oscope-of / build-check (push) Has been cancelled

Add panel_rect/panel_segments/panel_frame helpers to hand_display.py
(PANEL_SIDE=0.30, PANEL_MARGIN=0.02, PANEL_INNER=0.10). Segments use
uniform pixel-space scale so the hand aspect ratio is preserved
regardless of view aspect. mirror=True flips X within the panel.

In renderer._update_skeleton add push_panel (no mirror flip) and a
panels block before the segs==0 epilogue: routes persons_hands to
left/right panel via chirality list when aligned, screen-cx fallback
otherwise. Draws frame (pid 7) + wireframe (pid 5 left / 6 right).
Runs on every path including use_arkit. Guards: _mp_bones None, empty
hands, misaligned chirality, height==0.

26 tests pass (10 new for panel helpers).
This commit is contained in:
L'électron rare
2026-07-02 09:56:08 +02:00
parent 3a3e6b113f
commit 0a2865e136
3 changed files with 395 additions and 3 deletions
+159 -1
View File
@@ -1,4 +1,4 @@
"""Pure geometry helpers for hand overlay filtering.
"""Pure geometry helpers for hand overlay filtering and panel rendering.
No Metal, no numpy, no pyobjc — safe to unit-test anywhere.
"""
@@ -8,6 +8,22 @@ import math
from typing import Protocol
# ---------------------------------------------------------------------------
# Panel layout constants
# ---------------------------------------------------------------------------
#: Panel square size in normalized HEIGHT units. A panel occupies
#: PANEL_SIDE * view_height pixels on both axes (square in pixel space).
PANEL_SIDE: float = 0.30
#: Gap between the screen edge and the near edge of the panel (normalized).
PANEL_MARGIN: float = 0.02
#: Fraction of each panel dimension removed from every side to form the
#: inner rect used as the drawing area for hand wireframes.
PANEL_INNER: float = 0.10
class _HasXYC(Protocol):
x: float
y: float
@@ -64,3 +80,145 @@ def segment_ok(
if bone_len > max_bone_ratio * size:
return False
return True
# ---------------------------------------------------------------------------
# Panel helpers
# ---------------------------------------------------------------------------
def panel_rect(side: str, aspect: float) -> tuple[float, float, float, float]:
"""Normalized [0,1] screen rect of the hand side panel.
The panel is square in pixel space:
pixel_width = norm_width * view_width = (PANEL_SIDE / aspect) * aspect * view_height
= PANEL_SIDE * view_height = pixel_height.
Args:
side: "left" or "right".
aspect: view width / view height (> 0).
Returns:
(x0, y0, x1, y1) in normalized [0,1] coords, y down, origin top-left.
"""
norm_w = PANEL_SIDE / aspect # normalized width (square in pixel space)
norm_h = PANEL_SIDE # normalized height
y0 = 0.5 - norm_h / 2.0
y1 = 0.5 + norm_h / 2.0
if side == "left":
x0 = PANEL_MARGIN
x1 = PANEL_MARGIN + norm_w
else: # "right"
x1 = 1.0 - PANEL_MARGIN
x0 = x1 - norm_w
return x0, y0, x1, y1
def panel_frame(side: str, aspect: float) -> list[tuple[float, float, float, float]]:
"""Four border segments tracing the panel rect (clock-wise, starting top).
Returns:
List of (ax, ay, bx, by) in normalized [0,1] screen coords, y down.
"""
x0, y0, x1, y1 = panel_rect(side, aspect)
return [
(x0, y0, x1, y0), # top edge (left→right)
(x1, y0, x1, y1), # right edge (top→bottom)
(x1, y1, x0, y1), # bottom edge (right→left)
(x0, y1, x0, y0), # left edge (bottom→top)
]
def panel_segments(
kp: list[_HasXYC],
side: str,
bones: list[tuple[int, int]],
aspect: float,
mirror: bool = True,
) -> list[tuple[float, float, float, float]]:
"""Map hand landmarks into a side panel using uniform pixel-space scale.
The hand bounding box is fit into the inner panel rect (panel rect shrunk
by PANEL_INNER fraction on each side) with a uniform pixel-space scale —
meaning the pixel aspect ratio of the hand is preserved regardless of the
view aspect ratio.
Scale derivation:
s = min(inner_pixel_w / bbox_pixel_w, inner_pixel_h / bbox_pixel_h)
= min(iw / kw, ih / kh)
where iw/kw are normalized-coord ratios whose view dimensions cancel.
Moving Δkp.x in kp-x corresponds to Δkp.x * s in normalized screen x, and
Δkp.y in kp-y corresponds to Δkp.y * s in normalized screen y — preserving
the original pixel ratio between any two segment endpoints.
When mirror=True, X is flipped within the panel so the zoomed hand matches
the mirrored video background (i.e. the panel always shows a "front view").
Args:
kp: 21+ keypoints with .x, .y (normalized image coords, y down), .c.
side: "left" or "right".
bones: List of (index_a, index_b) pairs defining which keypoints to connect.
aspect: view width / view height.
mirror: If True, flip X within the panel.
Returns:
List of (ax, ay, bx, by) in normalized [0,1] screen coords, y down.
Returns [] if hand_plausible(kp) fails.
"""
if not hand_plausible(kp):
return []
x0, y0, x1, y1 = panel_rect(side, aspect)
pw = x1 - x0
ph = y1 - y0
# Inner rect: shrink by PANEL_INNER fraction on every side
ix0 = x0 + PANEL_INNER * pw
ix1 = x1 - PANEL_INNER * pw
iy0 = y0 + PANEL_INNER * ph
iy1 = y1 - PANEL_INNER * ph
iw = ix1 - ix0
ih = iy1 - iy0
# Keypoint bounding box in normalized image coords
kx_min = min(p.x for p in kp)
kx_max = max(p.x for p in kp)
ky_min = min(p.y for p in kp)
ky_max = max(p.y for p in kp)
kw = kx_max - kx_min
kh = ky_max - ky_min
if kw == 0.0 and kh == 0.0:
return []
# Uniform pixel-space scale factor (view dimensions cancel in the ratio)
if kw > 0.0 and kh > 0.0:
s = min(iw / kw, ih / kh)
elif kw > 0.0:
s = iw / kw
else:
s = ih / kh
# Centre the scaled hand bbox inside the inner rect
offset_x = (iw - kw * s) / 2.0
offset_y = (ih - kh * s) / 2.0
def _map(p: _HasXYC) -> tuple[float, float]:
rel_x = (kx_max - p.x) if mirror else (p.x - kx_min)
nx = ix0 + offset_x + rel_x * s
ny = iy0 + offset_y + (p.y - ky_min) * s
return nx, ny
sz = hand_size(kp)
result: list[tuple[float, float, float, float]] = []
for a, b in bones:
if a >= len(kp) or b >= len(kp):
continue
A = kp[a]
B = kp[b]
if not segment_ok(A, B, sz):
continue
ax, ay = _map(A)
bx, by = _map(B)
result.append((ax, ay, bx, by))
return result
+66 -1
View File
@@ -44,7 +44,7 @@ from Metal import (
from MetalKit import MTKView # noqa: F401 (utilise par d'autres modules)
from .arkit_skeleton import arkit_segments
from .hand_display import hand_plausible, hand_size, segment_ok
from .hand_display import hand_plausible, hand_size, segment_ok, panel_frame, panel_segments
from .mesh_topology import (
BODY_TRIANGLES,
FACE_TRIANGLES,
@@ -363,6 +363,25 @@ class MetalRenderer(NSObject):
segs += 1
return True
def push_panel(ax, ay, bx, by, conf, pid):
"""Like push_seg but NEVER applies mirror.
Panel coords are final screen coords [0,1] — they must not be
flipped again even when the video background is mirrored.
"""
nonlocal segs
if segs >= SKEL_MAX_SEGS:
return False
cax = ax * 2.0 - 1.0; cay = 1.0 - ay * 2.0
cbx = bx * 2.0 - 1.0; cby = 1.0 - by * 2.0
i = segs * 10
buf[i+0] = cax; buf[i+1] = cay; buf[i+2] = 0.0
buf[i+3] = conf; buf[i+4] = float(pid)
buf[i+5] = cbx; buf[i+6] = cby; buf[i+7] = 0.0
buf[i+8] = conf; buf[i+9] = float(pid)
segs += 1
return True
if use_arkit:
parents = s.arkit_parents
for pid, arr2d in s.persons_arkit_2d.items():
@@ -436,6 +455,52 @@ class MetalRenderer(NSObject):
if A.c < 0.2 or B.c < 0.2: continue
if not push(A, B, min(A.c, B.c), 0): break
# ---- SIDE PANELS: left/right hand front-view zoomed wireframe ----
# Runs on every path (arkit + mediapipe + fallback COCO).
# Hands always come from s.persons_hands regardless of use_arkit.
if self._mp_bones is not None:
_body_b, _face_b, lhand_bones_p, _ = self._mp_bones
hands = s.persons_hands
chir = getattr(s, "persons_hands_chirality", None) or []
asp = float(s.width) / float(s.height) if s.height else 1.0
# Route hands to left/right panels via chirality when available,
# otherwise by screen position (mirror-aware).
left_kp = None
right_kp = None
if chir and len(chir) == len(hands):
# chirality: 0 = left hand, 1 = right hand
for h_kp, c in zip(hands, chir):
if c == 0 and left_kp is None:
left_kp = h_kp
elif c == 1 and right_kp is None:
right_kp = h_kp
else:
# Fallback: order plausible hands by screen cx, mirror-aware.
# Screen left has low cx normally; high cx when video is mirrored.
plausible = [
(h_kp, sum(p.x for p in h_kp) / len(h_kp))
for h_kp in hands
if hand_plausible(h_kp)
]
plausible.sort(key=lambda t: -t[1] if mirror else t[1])
if plausible:
left_kp = plausible[0][0]
if len(plausible) >= 2:
right_kp = plausible[1][0]
# Draw frame (pid 7) then wireframe (pid 5 left / 6 right)
for side_name, h_kp, pid_s in (
("left", left_kp, 5),
("right", right_kp, 6),
):
if h_kp is None:
continue
for seg in panel_frame(side_name, asp):
push_panel(*seg, 1.0, 7)
for seg in panel_segments(h_kp, side_name, lhand_bones_p, asp, mirror=mirror):
push_panel(*seg, 1.0, pid_s)
if segs == 0:
return 0
data = self._skel_cpu_buf[: segs * 2 * SKEL_VERT_FLOATS].tobytes()
+170 -1
View File
@@ -9,7 +9,17 @@ from dataclasses import dataclass
import pytest
from data_only_viz.hand_display import hand_plausible, hand_size, segment_ok
from data_only_viz.hand_display import (
hand_plausible,
hand_size,
segment_ok,
panel_rect,
panel_segments,
panel_frame,
PANEL_SIDE,
PANEL_MARGIN,
PANEL_INNER,
)
# ---------------------------------------------------------------------------
@@ -176,3 +186,162 @@ def test_mediapipe_hand_all_bones_pass():
if not segment_ok(kp[i], kp[i + 1], size):
failures.append(i)
assert failures == [], f"Segments failed segment_ok: {failures}"
# ---------------------------------------------------------------------------
# 2D hand helper: spans a proper bbox with both x and y extent
# ---------------------------------------------------------------------------
def _make_hand_2d(
cx: float = 0.5,
cy: float = 0.5,
half_w: float = 0.05,
half_h: float = 0.05,
c: float = 1.0,
) -> list[Kp]:
"""21 landmarks filling bbox [cx±half_w, cy±half_h].
kp[0] = wrist at (cx-half_w, cy+half_h) — bottom-left of bbox
kp[9] = MCP at (cx+half_w, cy-half_h) — top-right of bbox
All other kp are spread uniformly across the bbox via a diagonal.
hand_size = dist(kp[0], kp[9]) = 2 * sqrt(half_w² + half_h²).
"""
n = 21
kps: list[Kp] = []
for i in range(n):
t = i / (n - 1)
kps.append(Kp(
x=cx - half_w + t * 2 * half_w,
y=cy - half_h + t * 2 * half_h,
z=0.0,
c=c,
))
# Override wrist and MCP so bbox spans full [cx-half_w..cx+half_w, cy-half_h..cy+half_h]
kps[0] = Kp(x=cx - half_w, y=cy + half_h, z=0.0, c=c) # bottom-left
kps[9] = Kp(x=cx + half_w, y=cy - half_h, z=0.0, c=c) # top-right
return kps
# Minimal bone list for panel tests: wrist(0) → middle-MCP(9)
_SIMPLE_BONES: list[tuple[int, int]] = [(0, 9)]
# ---------------------------------------------------------------------------
# panel_rect
# ---------------------------------------------------------------------------
def test_panel_rect_left_x0_anchor():
"""Left panel x0 == PANEL_MARGIN."""
x0, _y0, _x1, _y1 = panel_rect("left", aspect=1.0)
assert math.isclose(x0, PANEL_MARGIN, rel_tol=1e-6)
def test_panel_rect_right_x1_anchor():
"""Right panel x1 == 1 - PANEL_MARGIN."""
_x0, _y0, x1, _y1 = panel_rect("right", aspect=1.0)
assert math.isclose(x1, 1.0 - PANEL_MARGIN, rel_tol=1e-6)
def test_panel_rect_vertically_centered():
"""Panel vertical center is at 0.5."""
_x0, y0, _x1, y1 = panel_rect("left", aspect=1.0)
assert math.isclose((y0 + y1) / 2.0, 0.5, rel_tol=1e-6)
def test_panel_rect_square_in_pixels_landscape():
"""With 16:9 aspect ratio, normalized width × aspect == normalized height."""
aspect = 16.0 / 9.0
x0, y0, x1, y1 = panel_rect("left", aspect=aspect)
norm_w = x1 - x0
norm_h = y1 - y0
# pixel_w = norm_w * view_w = norm_w * aspect * view_h == norm_h * view_h = pixel_h
assert math.isclose(norm_w * aspect, norm_h, rel_tol=1e-6)
# ---------------------------------------------------------------------------
# panel_segments
# ---------------------------------------------------------------------------
def test_panel_segments_implausible_returns_empty():
"""hand_plausible fails → panel_segments returns []."""
kp = _make_hand(size=0.005, c=1.0) # size < size_min=0.02 → implausible
result = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
assert result == []
def test_panel_segments_inside_inner_rect():
"""All output segment endpoints lie inside the inner panel rect."""
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=0.04, half_h=0.08, c=1.0)
x0, y0, x1, y1 = panel_rect("left", aspect=1.0)
pw, ph = x1 - x0, y1 - y0
ix0 = x0 + PANEL_INNER * pw
ix1 = x1 - PANEL_INNER * pw
iy0 = y0 + PANEL_INNER * ph
iy1 = y1 - PANEL_INNER * ph
segs = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
assert segs, "expected at least one segment"
eps = 1e-9
for ax, ay, bx, by in segs:
assert ix0 - eps <= ax <= ix1 + eps, f"ax={ax} outside [{ix0}, {ix1}]"
assert ix0 - eps <= bx <= ix1 + eps, f"bx={bx} outside [{ix0}, {ix1}]"
assert iy0 - eps <= ay <= iy1 + eps, f"ay={ay} outside [{iy0}, {iy1}]"
assert iy0 - eps <= by <= iy1 + eps, f"by={by} outside [{iy0}, {iy1}]"
def test_panel_segments_aspect_preserved():
"""A hand 2× taller than wide in pixel space maps to 2× taller than wide in panel.
With aspect=1.0 (square screen), pixel ratio == normalized ratio.
The hand has half_h = 2*half_w, so pixel bbox is 2× taller than wide.
After uniform pixel-space scaling the output segment's |Δy|/|Δx| must also be 2.
"""
half_w, half_h = 0.04, 0.08 # kh=0.16, kw=0.08 → ratio 2
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=half_w, half_h=half_h, c=1.0)
segs = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
assert len(segs) == 1, f"expected 1 segment, got {len(segs)}"
ax, ay, bx, by = segs[0]
dx = abs(bx - ax)
dy = abs(by - ay)
assert dx > 1e-9 and dy > 1e-9, "segment must have both x and y extent"
ratio = dy / dx # pixel ratio (= normalized ratio at aspect=1)
assert math.isclose(ratio, 2.0, rel_tol=1e-3), f"expected pixel ratio 2.0, got {ratio}"
def test_panel_segments_mirror_flips_x():
"""mirror=True flips X within the panel relative to mirror=False."""
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=0.04, half_h=0.04, c=1.0)
# kp[0] at bottom-left (kx_min), kp[9] at top-right (kx_max)
segs_normal = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
segs_mirror = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=True)
assert len(segs_normal) == 1
assert len(segs_mirror) == 1
ax0, _ay0, bx0, _by0 = segs_normal[0]
ax1, _ay1, bx1, _by1 = segs_mirror[0]
# kp[0].x < kp[9].x → without mirror: ax < bx; with mirror: ax > bx
assert ax0 < bx0, "without mirror: wrist (kx_min) should project left of MCP (kx_max)"
assert ax1 > bx1, "with mirror: X is flipped so wrist projects right of MCP"
# ---------------------------------------------------------------------------
# panel_frame
# ---------------------------------------------------------------------------
def test_panel_frame_returns_four_segments():
"""panel_frame returns exactly 4 segments."""
segs = panel_frame("left", aspect=1.0)
assert len(segs) == 4
def test_panel_frame_traces_rect():
"""Every endpoint of each frame segment is a corner of the panel rect."""
aspect = 1.0
x0, y0, x1, y1 = panel_rect("left", aspect)
corners = [(x0, y0), (x0, y1), (x1, y0), (x1, y1)]
segs = panel_frame("left", aspect)
for ax, ay, bx, by in segs:
for ex, ey in [(ax, ay), (bx, by)]:
on_corner = any(
math.isclose(ex, cx, abs_tol=1e-9) and math.isclose(ey, cy, abs_tol=1e-9)
for cx, cy in corners
)
assert on_corner, f"Endpoint ({ex:.4f}, {ey:.4f}) not a corner of panel rect"