feat(viz): left/right hand front-view panels
CI build oscope-of / build-check (push) Has been cancelled
CI build oscope-of / build-check (push) Has been cancelled
Add panel_rect/panel_segments/panel_frame helpers to hand_display.py (PANEL_SIDE=0.30, PANEL_MARGIN=0.02, PANEL_INNER=0.10). Segments use uniform pixel-space scale so the hand aspect ratio is preserved regardless of view aspect. mirror=True flips X within the panel. In renderer._update_skeleton add push_panel (no mirror flip) and a panels block before the segs==0 epilogue: routes persons_hands to left/right panel via chirality list when aligned, screen-cx fallback otherwise. Draws frame (pid 7) + wireframe (pid 5 left / 6 right). Runs on every path including use_arkit. Guards: _mp_bones None, empty hands, misaligned chirality, height==0. 26 tests pass (10 new for panel helpers).
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
"""Pure geometry helpers for hand overlay filtering.
|
||||
"""Pure geometry helpers for hand overlay filtering and panel rendering.
|
||||
|
||||
No Metal, no numpy, no pyobjc — safe to unit-test anywhere.
|
||||
"""
|
||||
@@ -8,6 +8,22 @@ import math
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Panel layout constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Panel square size in normalized HEIGHT units. A panel occupies
|
||||
#: PANEL_SIDE * view_height pixels on both axes (square in pixel space).
|
||||
PANEL_SIDE: float = 0.30
|
||||
|
||||
#: Gap between the screen edge and the near edge of the panel (normalized).
|
||||
PANEL_MARGIN: float = 0.02
|
||||
|
||||
#: Fraction of each panel dimension removed from every side to form the
|
||||
#: inner rect used as the drawing area for hand wireframes.
|
||||
PANEL_INNER: float = 0.10
|
||||
|
||||
|
||||
class _HasXYC(Protocol):
|
||||
x: float
|
||||
y: float
|
||||
@@ -64,3 +80,145 @@ def segment_ok(
|
||||
if bone_len > max_bone_ratio * size:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Panel helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def panel_rect(side: str, aspect: float) -> tuple[float, float, float, float]:
|
||||
"""Normalized [0,1] screen rect of the hand side panel.
|
||||
|
||||
The panel is square in pixel space:
|
||||
pixel_width = norm_width * view_width = (PANEL_SIDE / aspect) * aspect * view_height
|
||||
= PANEL_SIDE * view_height = pixel_height.
|
||||
|
||||
Args:
|
||||
side: "left" or "right".
|
||||
aspect: view width / view height (> 0).
|
||||
|
||||
Returns:
|
||||
(x0, y0, x1, y1) in normalized [0,1] coords, y down, origin top-left.
|
||||
"""
|
||||
norm_w = PANEL_SIDE / aspect # normalized width (square in pixel space)
|
||||
norm_h = PANEL_SIDE # normalized height
|
||||
y0 = 0.5 - norm_h / 2.0
|
||||
y1 = 0.5 + norm_h / 2.0
|
||||
if side == "left":
|
||||
x0 = PANEL_MARGIN
|
||||
x1 = PANEL_MARGIN + norm_w
|
||||
else: # "right"
|
||||
x1 = 1.0 - PANEL_MARGIN
|
||||
x0 = x1 - norm_w
|
||||
return x0, y0, x1, y1
|
||||
|
||||
|
||||
def panel_frame(side: str, aspect: float) -> list[tuple[float, float, float, float]]:
|
||||
"""Four border segments tracing the panel rect (clock-wise, starting top).
|
||||
|
||||
Returns:
|
||||
List of (ax, ay, bx, by) in normalized [0,1] screen coords, y down.
|
||||
"""
|
||||
x0, y0, x1, y1 = panel_rect(side, aspect)
|
||||
return [
|
||||
(x0, y0, x1, y0), # top edge (left→right)
|
||||
(x1, y0, x1, y1), # right edge (top→bottom)
|
||||
(x1, y1, x0, y1), # bottom edge (right→left)
|
||||
(x0, y1, x0, y0), # left edge (bottom→top)
|
||||
]
|
||||
|
||||
|
||||
def panel_segments(
|
||||
kp: list[_HasXYC],
|
||||
side: str,
|
||||
bones: list[tuple[int, int]],
|
||||
aspect: float,
|
||||
mirror: bool = True,
|
||||
) -> list[tuple[float, float, float, float]]:
|
||||
"""Map hand landmarks into a side panel using uniform pixel-space scale.
|
||||
|
||||
The hand bounding box is fit into the inner panel rect (panel rect shrunk
|
||||
by PANEL_INNER fraction on each side) with a uniform pixel-space scale —
|
||||
meaning the pixel aspect ratio of the hand is preserved regardless of the
|
||||
view aspect ratio.
|
||||
|
||||
Scale derivation:
|
||||
s = min(inner_pixel_w / bbox_pixel_w, inner_pixel_h / bbox_pixel_h)
|
||||
= min(iw / kw, ih / kh)
|
||||
where iw/kw are normalized-coord ratios whose view dimensions cancel.
|
||||
Moving Δkp.x in kp-x corresponds to Δkp.x * s in normalized screen x, and
|
||||
Δkp.y in kp-y corresponds to Δkp.y * s in normalized screen y — preserving
|
||||
the original pixel ratio between any two segment endpoints.
|
||||
|
||||
When mirror=True, X is flipped within the panel so the zoomed hand matches
|
||||
the mirrored video background (i.e. the panel always shows a "front view").
|
||||
|
||||
Args:
|
||||
kp: 21+ keypoints with .x, .y (normalized image coords, y down), .c.
|
||||
side: "left" or "right".
|
||||
bones: List of (index_a, index_b) pairs defining which keypoints to connect.
|
||||
aspect: view width / view height.
|
||||
mirror: If True, flip X within the panel.
|
||||
|
||||
Returns:
|
||||
List of (ax, ay, bx, by) in normalized [0,1] screen coords, y down.
|
||||
Returns [] if hand_plausible(kp) fails.
|
||||
"""
|
||||
if not hand_plausible(kp):
|
||||
return []
|
||||
|
||||
x0, y0, x1, y1 = panel_rect(side, aspect)
|
||||
pw = x1 - x0
|
||||
ph = y1 - y0
|
||||
|
||||
# Inner rect: shrink by PANEL_INNER fraction on every side
|
||||
ix0 = x0 + PANEL_INNER * pw
|
||||
ix1 = x1 - PANEL_INNER * pw
|
||||
iy0 = y0 + PANEL_INNER * ph
|
||||
iy1 = y1 - PANEL_INNER * ph
|
||||
iw = ix1 - ix0
|
||||
ih = iy1 - iy0
|
||||
|
||||
# Keypoint bounding box in normalized image coords
|
||||
kx_min = min(p.x for p in kp)
|
||||
kx_max = max(p.x for p in kp)
|
||||
ky_min = min(p.y for p in kp)
|
||||
ky_max = max(p.y for p in kp)
|
||||
kw = kx_max - kx_min
|
||||
kh = ky_max - ky_min
|
||||
|
||||
if kw == 0.0 and kh == 0.0:
|
||||
return []
|
||||
|
||||
# Uniform pixel-space scale factor (view dimensions cancel in the ratio)
|
||||
if kw > 0.0 and kh > 0.0:
|
||||
s = min(iw / kw, ih / kh)
|
||||
elif kw > 0.0:
|
||||
s = iw / kw
|
||||
else:
|
||||
s = ih / kh
|
||||
|
||||
# Centre the scaled hand bbox inside the inner rect
|
||||
offset_x = (iw - kw * s) / 2.0
|
||||
offset_y = (ih - kh * s) / 2.0
|
||||
|
||||
def _map(p: _HasXYC) -> tuple[float, float]:
|
||||
rel_x = (kx_max - p.x) if mirror else (p.x - kx_min)
|
||||
nx = ix0 + offset_x + rel_x * s
|
||||
ny = iy0 + offset_y + (p.y - ky_min) * s
|
||||
return nx, ny
|
||||
|
||||
sz = hand_size(kp)
|
||||
result: list[tuple[float, float, float, float]] = []
|
||||
for a, b in bones:
|
||||
if a >= len(kp) or b >= len(kp):
|
||||
continue
|
||||
A = kp[a]
|
||||
B = kp[b]
|
||||
if not segment_ok(A, B, sz):
|
||||
continue
|
||||
ax, ay = _map(A)
|
||||
bx, by = _map(B)
|
||||
result.append((ax, ay, bx, by))
|
||||
|
||||
return result
|
||||
|
||||
@@ -44,7 +44,7 @@ from Metal import (
|
||||
from MetalKit import MTKView # noqa: F401 (utilise par d'autres modules)
|
||||
|
||||
from .arkit_skeleton import arkit_segments
|
||||
from .hand_display import hand_plausible, hand_size, segment_ok
|
||||
from .hand_display import hand_plausible, hand_size, segment_ok, panel_frame, panel_segments
|
||||
from .mesh_topology import (
|
||||
BODY_TRIANGLES,
|
||||
FACE_TRIANGLES,
|
||||
@@ -363,6 +363,25 @@ class MetalRenderer(NSObject):
|
||||
segs += 1
|
||||
return True
|
||||
|
||||
def push_panel(ax, ay, bx, by, conf, pid):
|
||||
"""Like push_seg but NEVER applies mirror.
|
||||
|
||||
Panel coords are final screen coords [0,1] — they must not be
|
||||
flipped again even when the video background is mirrored.
|
||||
"""
|
||||
nonlocal segs
|
||||
if segs >= SKEL_MAX_SEGS:
|
||||
return False
|
||||
cax = ax * 2.0 - 1.0; cay = 1.0 - ay * 2.0
|
||||
cbx = bx * 2.0 - 1.0; cby = 1.0 - by * 2.0
|
||||
i = segs * 10
|
||||
buf[i+0] = cax; buf[i+1] = cay; buf[i+2] = 0.0
|
||||
buf[i+3] = conf; buf[i+4] = float(pid)
|
||||
buf[i+5] = cbx; buf[i+6] = cby; buf[i+7] = 0.0
|
||||
buf[i+8] = conf; buf[i+9] = float(pid)
|
||||
segs += 1
|
||||
return True
|
||||
|
||||
if use_arkit:
|
||||
parents = s.arkit_parents
|
||||
for pid, arr2d in s.persons_arkit_2d.items():
|
||||
@@ -436,6 +455,52 @@ class MetalRenderer(NSObject):
|
||||
if A.c < 0.2 or B.c < 0.2: continue
|
||||
if not push(A, B, min(A.c, B.c), 0): break
|
||||
|
||||
# ---- SIDE PANELS: left/right hand front-view zoomed wireframe ----
|
||||
# Runs on every path (arkit + mediapipe + fallback COCO).
|
||||
# Hands always come from s.persons_hands regardless of use_arkit.
|
||||
if self._mp_bones is not None:
|
||||
_body_b, _face_b, lhand_bones_p, _ = self._mp_bones
|
||||
hands = s.persons_hands
|
||||
chir = getattr(s, "persons_hands_chirality", None) or []
|
||||
asp = float(s.width) / float(s.height) if s.height else 1.0
|
||||
|
||||
# Route hands to left/right panels via chirality when available,
|
||||
# otherwise by screen position (mirror-aware).
|
||||
left_kp = None
|
||||
right_kp = None
|
||||
if chir and len(chir) == len(hands):
|
||||
# chirality: 0 = left hand, 1 = right hand
|
||||
for h_kp, c in zip(hands, chir):
|
||||
if c == 0 and left_kp is None:
|
||||
left_kp = h_kp
|
||||
elif c == 1 and right_kp is None:
|
||||
right_kp = h_kp
|
||||
else:
|
||||
# Fallback: order plausible hands by screen cx, mirror-aware.
|
||||
# Screen left has low cx normally; high cx when video is mirrored.
|
||||
plausible = [
|
||||
(h_kp, sum(p.x for p in h_kp) / len(h_kp))
|
||||
for h_kp in hands
|
||||
if hand_plausible(h_kp)
|
||||
]
|
||||
plausible.sort(key=lambda t: -t[1] if mirror else t[1])
|
||||
if plausible:
|
||||
left_kp = plausible[0][0]
|
||||
if len(plausible) >= 2:
|
||||
right_kp = plausible[1][0]
|
||||
|
||||
# Draw frame (pid 7) then wireframe (pid 5 left / 6 right)
|
||||
for side_name, h_kp, pid_s in (
|
||||
("left", left_kp, 5),
|
||||
("right", right_kp, 6),
|
||||
):
|
||||
if h_kp is None:
|
||||
continue
|
||||
for seg in panel_frame(side_name, asp):
|
||||
push_panel(*seg, 1.0, 7)
|
||||
for seg in panel_segments(h_kp, side_name, lhand_bones_p, asp, mirror=mirror):
|
||||
push_panel(*seg, 1.0, pid_s)
|
||||
|
||||
if segs == 0:
|
||||
return 0
|
||||
data = self._skel_cpu_buf[: segs * 2 * SKEL_VERT_FLOATS].tobytes()
|
||||
|
||||
@@ -9,7 +9,17 @@ from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
|
||||
from data_only_viz.hand_display import hand_plausible, hand_size, segment_ok
|
||||
from data_only_viz.hand_display import (
|
||||
hand_plausible,
|
||||
hand_size,
|
||||
segment_ok,
|
||||
panel_rect,
|
||||
panel_segments,
|
||||
panel_frame,
|
||||
PANEL_SIDE,
|
||||
PANEL_MARGIN,
|
||||
PANEL_INNER,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -176,3 +186,162 @@ def test_mediapipe_hand_all_bones_pass():
|
||||
if not segment_ok(kp[i], kp[i + 1], size):
|
||||
failures.append(i)
|
||||
assert failures == [], f"Segments failed segment_ok: {failures}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2D hand helper: spans a proper bbox with both x and y extent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_hand_2d(
|
||||
cx: float = 0.5,
|
||||
cy: float = 0.5,
|
||||
half_w: float = 0.05,
|
||||
half_h: float = 0.05,
|
||||
c: float = 1.0,
|
||||
) -> list[Kp]:
|
||||
"""21 landmarks filling bbox [cx±half_w, cy±half_h].
|
||||
|
||||
kp[0] = wrist at (cx-half_w, cy+half_h) — bottom-left of bbox
|
||||
kp[9] = MCP at (cx+half_w, cy-half_h) — top-right of bbox
|
||||
All other kp are spread uniformly across the bbox via a diagonal.
|
||||
hand_size = dist(kp[0], kp[9]) = 2 * sqrt(half_w² + half_h²).
|
||||
"""
|
||||
n = 21
|
||||
kps: list[Kp] = []
|
||||
for i in range(n):
|
||||
t = i / (n - 1)
|
||||
kps.append(Kp(
|
||||
x=cx - half_w + t * 2 * half_w,
|
||||
y=cy - half_h + t * 2 * half_h,
|
||||
z=0.0,
|
||||
c=c,
|
||||
))
|
||||
# Override wrist and MCP so bbox spans full [cx-half_w..cx+half_w, cy-half_h..cy+half_h]
|
||||
kps[0] = Kp(x=cx - half_w, y=cy + half_h, z=0.0, c=c) # bottom-left
|
||||
kps[9] = Kp(x=cx + half_w, y=cy - half_h, z=0.0, c=c) # top-right
|
||||
return kps
|
||||
|
||||
|
||||
# Minimal bone list for panel tests: wrist(0) → middle-MCP(9)
|
||||
_SIMPLE_BONES: list[tuple[int, int]] = [(0, 9)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# panel_rect
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_panel_rect_left_x0_anchor():
|
||||
"""Left panel x0 == PANEL_MARGIN."""
|
||||
x0, _y0, _x1, _y1 = panel_rect("left", aspect=1.0)
|
||||
assert math.isclose(x0, PANEL_MARGIN, rel_tol=1e-6)
|
||||
|
||||
|
||||
def test_panel_rect_right_x1_anchor():
|
||||
"""Right panel x1 == 1 - PANEL_MARGIN."""
|
||||
_x0, _y0, x1, _y1 = panel_rect("right", aspect=1.0)
|
||||
assert math.isclose(x1, 1.0 - PANEL_MARGIN, rel_tol=1e-6)
|
||||
|
||||
|
||||
def test_panel_rect_vertically_centered():
|
||||
"""Panel vertical center is at 0.5."""
|
||||
_x0, y0, _x1, y1 = panel_rect("left", aspect=1.0)
|
||||
assert math.isclose((y0 + y1) / 2.0, 0.5, rel_tol=1e-6)
|
||||
|
||||
|
||||
def test_panel_rect_square_in_pixels_landscape():
|
||||
"""With 16:9 aspect ratio, normalized width × aspect == normalized height."""
|
||||
aspect = 16.0 / 9.0
|
||||
x0, y0, x1, y1 = panel_rect("left", aspect=aspect)
|
||||
norm_w = x1 - x0
|
||||
norm_h = y1 - y0
|
||||
# pixel_w = norm_w * view_w = norm_w * aspect * view_h == norm_h * view_h = pixel_h
|
||||
assert math.isclose(norm_w * aspect, norm_h, rel_tol=1e-6)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# panel_segments
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_panel_segments_implausible_returns_empty():
|
||||
"""hand_plausible fails → panel_segments returns []."""
|
||||
kp = _make_hand(size=0.005, c=1.0) # size < size_min=0.02 → implausible
|
||||
result = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_panel_segments_inside_inner_rect():
|
||||
"""All output segment endpoints lie inside the inner panel rect."""
|
||||
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=0.04, half_h=0.08, c=1.0)
|
||||
x0, y0, x1, y1 = panel_rect("left", aspect=1.0)
|
||||
pw, ph = x1 - x0, y1 - y0
|
||||
ix0 = x0 + PANEL_INNER * pw
|
||||
ix1 = x1 - PANEL_INNER * pw
|
||||
iy0 = y0 + PANEL_INNER * ph
|
||||
iy1 = y1 - PANEL_INNER * ph
|
||||
segs = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
|
||||
assert segs, "expected at least one segment"
|
||||
eps = 1e-9
|
||||
for ax, ay, bx, by in segs:
|
||||
assert ix0 - eps <= ax <= ix1 + eps, f"ax={ax} outside [{ix0}, {ix1}]"
|
||||
assert ix0 - eps <= bx <= ix1 + eps, f"bx={bx} outside [{ix0}, {ix1}]"
|
||||
assert iy0 - eps <= ay <= iy1 + eps, f"ay={ay} outside [{iy0}, {iy1}]"
|
||||
assert iy0 - eps <= by <= iy1 + eps, f"by={by} outside [{iy0}, {iy1}]"
|
||||
|
||||
|
||||
def test_panel_segments_aspect_preserved():
|
||||
"""A hand 2× taller than wide in pixel space maps to 2× taller than wide in panel.
|
||||
|
||||
With aspect=1.0 (square screen), pixel ratio == normalized ratio.
|
||||
The hand has half_h = 2*half_w, so pixel bbox is 2× taller than wide.
|
||||
After uniform pixel-space scaling the output segment's |Δy|/|Δx| must also be 2.
|
||||
"""
|
||||
half_w, half_h = 0.04, 0.08 # kh=0.16, kw=0.08 → ratio 2
|
||||
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=half_w, half_h=half_h, c=1.0)
|
||||
segs = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
|
||||
assert len(segs) == 1, f"expected 1 segment, got {len(segs)}"
|
||||
ax, ay, bx, by = segs[0]
|
||||
dx = abs(bx - ax)
|
||||
dy = abs(by - ay)
|
||||
assert dx > 1e-9 and dy > 1e-9, "segment must have both x and y extent"
|
||||
ratio = dy / dx # pixel ratio (= normalized ratio at aspect=1)
|
||||
assert math.isclose(ratio, 2.0, rel_tol=1e-3), f"expected pixel ratio 2.0, got {ratio}"
|
||||
|
||||
|
||||
def test_panel_segments_mirror_flips_x():
|
||||
"""mirror=True flips X within the panel relative to mirror=False."""
|
||||
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=0.04, half_h=0.04, c=1.0)
|
||||
# kp[0] at bottom-left (kx_min), kp[9] at top-right (kx_max)
|
||||
segs_normal = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
|
||||
segs_mirror = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=True)
|
||||
assert len(segs_normal) == 1
|
||||
assert len(segs_mirror) == 1
|
||||
ax0, _ay0, bx0, _by0 = segs_normal[0]
|
||||
ax1, _ay1, bx1, _by1 = segs_mirror[0]
|
||||
# kp[0].x < kp[9].x → without mirror: ax < bx; with mirror: ax > bx
|
||||
assert ax0 < bx0, "without mirror: wrist (kx_min) should project left of MCP (kx_max)"
|
||||
assert ax1 > bx1, "with mirror: X is flipped so wrist projects right of MCP"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# panel_frame
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_panel_frame_returns_four_segments():
|
||||
"""panel_frame returns exactly 4 segments."""
|
||||
segs = panel_frame("left", aspect=1.0)
|
||||
assert len(segs) == 4
|
||||
|
||||
|
||||
def test_panel_frame_traces_rect():
|
||||
"""Every endpoint of each frame segment is a corner of the panel rect."""
|
||||
aspect = 1.0
|
||||
x0, y0, x1, y1 = panel_rect("left", aspect)
|
||||
corners = [(x0, y0), (x0, y1), (x1, y0), (x1, y1)]
|
||||
segs = panel_frame("left", aspect)
|
||||
for ax, ay, bx, by in segs:
|
||||
for ex, ey in [(ax, ay), (bx, by)]:
|
||||
on_corner = any(
|
||||
math.isclose(ex, cx, abs_tol=1e-9) and math.isclose(ey, cy, abs_tol=1e-9)
|
||||
for cx, cy in corners
|
||||
)
|
||||
assert on_corner, f"Endpoint ({ex:.4f}, {ey:.4f}) not a corner of panel rect"
|
||||
|
||||
Reference in New Issue
Block a user