diff --git a/data_only_viz/action_head_pub.py b/data_only_viz/action_head_pub.py index f53dd5b..de9369b 100644 --- a/data_only_viz/action_head_pub.py +++ b/data_only_viz/action_head_pub.py @@ -24,7 +24,7 @@ from data_only_viz.action_head import ( J3D_FINGERS_PER_HAND, LABELS, ) -from data_only_viz.hand_display import hand_plausible, HandPersistenceGate +from data_only_viz.hand_display import gesture_quality, hand_plausible, HandPersistenceGate from data_only_viz.hand_slots import route_hands LOG = logging.getLogger("action_head_pub") @@ -350,10 +350,11 @@ class ActionHeadPublisher(threading.Thread): self.bridge.send_pinch(ev) def _update_gesture_slot_status(self) -> None: - """Compute per-slot gesture status and write it to state. + """Compute per-slot gesture status and quality, write both to state. Status values: 0=absent, 1=detected(plausible+established, not armed), 2=armed(near+facing), 3=pinch engaged. + Quality ∈ [0, 1]: continuous intensity for frame brightness + thickness. Called once per tick after all emitters have run. """ stab = getattr(self, "_stab", None) @@ -364,6 +365,9 @@ class ActionHeadPublisher(threading.Thread): active = stab.active_flags() engaged = pinch_det.engaged_slots() if pinch_det is not None else (False, False) status = [0, 0] + quality = [0.0, 0.0] + face_min = stab._face_min + near_on = stab._near_on for slot in range(2): if engaged[slot]: status[slot] = 3 @@ -371,8 +375,15 @@ class ActionHeadPublisher(threading.Thread): status[slot] = 2 elif raw[slot] is not None: status[slot] = 1 + quality[slot] = gesture_quality( + raw[slot], + face_min=face_min, + near_on=near_on, + engaged=bool(engaged[slot]), + ) with self.state.lock(): self.state.gesture_slot_status = status + self.state.gesture_slot_quality = quality def _read_sources(self) -> tuple[ list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] | None, diff --git a/data_only_viz/hand_display.py b/data_only_viz/hand_display.py index 58229e5..c44b643 100644 --- a/data_only_viz/hand_display.py +++ b/data_only_viz/hand_display.py @@ -2,11 +2,19 @@ No Metal, no numpy, no pyobjc — safe to unit-test anywhere. -Panel frame status indicator color legend (used in renderer.py): - pid 7 conf 0.3 → status 0: absent (dim frame) - pid 7 conf 1.0 → status 1: detected (plausible+established, not armed) - pid 8 conf 1.0 → status 2: armed (near + facing camera) - pid 9 conf 1.0 → status 3: pinch engaged (double-stroke bold) +Panel frame legend (renderer.py): + pid 7 conf=0.25+0.75*q → status 0: absent (q=0); status 1: detected (q≥0.30) + pid 8 conf=0.25+0.75*q → status 2: armed (near+facing camera, q≥0.30) + pid 9 conf=1.0 (q forced)→ status 3: pinch engaged + Thickness: 1 stroke always; +1 at q≥0.50; +1 at q≥0.85 or status==3. + See gesture_quality() for the continuous quality formula. + +Position gauges (renderer.py, drawn around each panel): + X gauge: horizontal rail BELOW panel (pid 7, conf 0.4 or 0.25 if absent); + vertical notch marker at cx position (pid 5/6, conf 1.0). + Y gauge: vertical rail on outer side (pid 7, conf 0.4 or 0.25 if absent); + horizontal notch marker at cy position (pid 5/6, conf 1.0). + See gauge_segments() — marker x is mirror-flipped if mirror=True. """ from __future__ import annotations @@ -366,3 +374,149 @@ class HandPersistenceGate: self._tracks = new_tracks return result + + +# --------------------------------------------------------------------------- +# Gauge layout constants +# --------------------------------------------------------------------------- + +#: Normalised gap between a panel edge and its adjacent gauge rail. +GAUGE_GAP: float = 0.012 + +#: Half the tick length (total = 2 * GAUGE_TICK_HALF), in normalised coords. +GAUGE_TICK_HALF: float = 0.004 + +#: Parallel offset for the second stroke of a bold-notch marker. +GAUGE_BOLD: float = 0.001 + + +# --------------------------------------------------------------------------- +# Continuous quality score +# --------------------------------------------------------------------------- + +def gesture_quality( + hand: object, + *, + face_min: float, + near_on: float, + engaged: bool = False, +) -> float: + """Continuous quality score ∈ [0, 1] for one gesture slot. + + Drives panel frame brightness and stroke thickness in renderer.py. + + Args: + hand: 21-kp hand object (attribute-style .x/.y) or None. + ``None`` → 0.0 (slot absent or not yet validated by gates). + face_min: palm-spread ratio at which the slot is considered fully + armed (from GestureSlotStabilizer; default env 0.5). + near_on: wrist-to-middle-MCP distance (normalised) at which the slot + is fully near (from GestureSlotStabilizer; default 0.10). + engaged: True when a pinch is held for this slot → forces 1.0. + + Returns: + quality ∈ [0, 1]: + * 0.0 when ``hand`` is None. + * 1.0 when ``engaged`` is True. + * else: 0.30·established + 0.35·facing_norm + 0.35·near_norm + where established = 1.0 (hand passed plausibility + temporal gates), + facing_norm = clamp((hand_facing(hand) − 0.25) / (face_min − 0.25), 0, 1), + near_norm = clamp((hand_size(hand) − 0.5·near_on) / (0.5·near_on), 0, 1). + """ + if hand is None: + return 0.0 + if engaged: + return 1.0 + # facing_norm: 0 at facing = 0.25, 1 at facing = face_min. + denom_f = face_min - 0.25 + if denom_f > 1e-9: + facing_norm = min(1.0, max(0.0, (hand_facing(hand) - 0.25) / denom_f)) + else: + # face_min ≤ 0.25 — any spread qualifies. + facing_norm = 1.0 + # near_norm: 0 at hand_size = 0.5·near_on, 1 at hand_size = near_on. + half_on = 0.5 * near_on + if half_on > 1e-9: + near_norm = min(1.0, max(0.0, (hand_size(hand) - half_on) / half_on)) + else: + near_norm = 1.0 + return 0.30 + 0.35 * facing_norm + 0.35 * near_norm + + +# --------------------------------------------------------------------------- +# Position gauges +# --------------------------------------------------------------------------- + +def gauge_segments( + cx: float | None, + cy: float | None, + side: str, + aspect: float, + mirror: bool = False, + *, + content_pid: int = 5, + rail_pid: int = 7, +) -> list[tuple[float, float, float, float, float, int]]: + """Rail + marker segments for the X and Y position gauges around a panel. + + X gauge: a horizontal rail placed GAUGE_GAP below the panel bottom edge, + spanning the full panel width. A bold vertical notch marks the + hand's cx position on the rail. + + Y gauge: a vertical rail placed GAUGE_GAP outside the panel's outer edge + (left of the left panel, right of the right panel), spanning the + full panel height. A bold horizontal notch marks cy. + + Args: + cx: Normalised hand centre-x ∈ [0, 1], or None when absent. + cy: Normalised hand centre-y ∈ [0, 1], or None when absent. + side: "left" or "right". + aspect: View width / view height (> 0). + mirror: If True, flip the cx marker position (1 − cx) so it moves + in the same direction as the mirrored video. cy is never + flipped. + content_pid: pid for marker segments (5 = left slot, 6 = right slot). + rail_pid: pid for rail segments (default 7). + + Returns: + List of (ax, ay, bx, by, conf, pid) tuples ready to feed + ``push_panel`` in the renderer. When cx/cy is None the rails are + drawn dim (conf 0.25) with no marker; otherwise rails use conf 0.4 + and markers use conf 1.0. + """ + x0, y0, x1, y1 = panel_rect(side, aspect) + has_data = cx is not None and cy is not None + rail_conf = 0.4 if has_data else 0.25 + out: list[tuple[float, float, float, float, float, int]] = [] + + # ---- X gauge: horizontal rail below panel ---- + xg_y = y1 + GAUGE_GAP + out.append((x0, xg_y, x1, xg_y, rail_conf, rail_pid)) + if has_data: + assert cx is not None and cy is not None # narrowing for type checkers + cx_eff = 1.0 - cx if mirror else cx + mx = x0 + cx_eff * (x1 - x0) + # Bold vertical notch: two parallel vertical ticks + out.append((mx, xg_y - GAUGE_TICK_HALF, mx, xg_y + GAUGE_TICK_HALF, + 1.0, content_pid)) + out.append((mx + GAUGE_BOLD, xg_y - GAUGE_TICK_HALF, + mx + GAUGE_BOLD, xg_y + GAUGE_TICK_HALF, + 1.0, content_pid)) + + # ---- Y gauge: vertical rail on outer side ---- + if side == "left": + yg_x = x0 - GAUGE_GAP + else: + yg_x = x1 + GAUGE_GAP + out.append((yg_x, y0, yg_x, y1, rail_conf, rail_pid)) + if has_data: + assert cx is not None and cy is not None + my = y0 + cy * (y1 - y0) + # Bold horizontal notch: two parallel horizontal ticks + out.append((yg_x - GAUGE_TICK_HALF, my, yg_x + GAUGE_TICK_HALF, my, + 1.0, content_pid)) + out.append((yg_x - GAUGE_TICK_HALF, my + GAUGE_BOLD, + yg_x + GAUGE_TICK_HALF, my + GAUGE_BOLD, + 1.0, content_pid)) + + return out diff --git a/data_only_viz/renderer.py b/data_only_viz/renderer.py index 8d7771f..86235f4 100644 --- a/data_only_viz/renderer.py +++ b/data_only_viz/renderer.py @@ -47,6 +47,7 @@ from .arkit_skeleton import arkit_segments, finger_joint_mask from .hand_display import ( HandPersistenceGate, arkit_2d_fresh, + gauge_segments, hand_plausible, hand_size, segment_ok, @@ -528,33 +529,47 @@ class MetalRenderer(NSObject): ) left_kp, right_kp = slotted - # Panel frame colored by gesture status (read from state, already under lock): - # pid 7 conf 0.3 = status 0 (absent, dim) - # pid 7 conf 1.0 = status 1 (detected, normal) - # pid 8 conf 1.0 = status 2 (armed: near+facing) - # pid 9 conf 1.0 = status 3 (pinch engaged, double-stroke) + # Panel frame: status drives hue (pid 7/8/9), quality drives + # brightness (0.25+0.75*q) and thickness (1..3 stroke passes). + # pid 7 conf=0.25+0.75*q → status 0: absent (q=0) + # → status 1: detected (q≥0.30) + # pid 8 conf=0.25+0.75*q → status 2: armed (near+facing) + # pid 9 conf=1.0 (q=1.0) → status 3: pinch engaged + # Thickness: pass 1 always; pass 2 at q≥0.50; pass 3 at q≥0.85 or status==3. _slot_status = getattr(s, "gesture_slot_status", [0, 0]) + _slot_quality = getattr(s, "gesture_slot_quality", [0.0, 0.0]) + _hf_all = getattr(s, "hand_feats", None) or {} for _si, (side_name, h_kp, pid_s) in enumerate(( ("left", left_kp, 5), ("right", right_kp, 6), )): _st = _slot_status[_si] if _si < len(_slot_status) else 0 + _q = float(_slot_quality[_si]) if _si < len(_slot_quality) else 0.0 + _fconf = 0.25 + 0.75 * _q + # Status drives hue (pid selection) + _fpid = 9 if _st == 3 else (8 if _st == 2 else 7) _frame_segs = panel_frame(side_name, asp) - if _st == 0: - for seg in _frame_segs: - push_panel(*seg, 0.3, 7) # absent: dim - elif _st == 1: - for seg in _frame_segs: - push_panel(*seg, 1.0, 7) # detected: normal - elif _st == 2: - for seg in _frame_segs: - push_panel(*seg, 1.0, 8) # armed: distinct palette color - else: # 3: pinch engaged - _off = 0.001 # tiny offset for bold double-stroke - for seg in _frame_segs: - push_panel(*seg, 1.0, 9) - ax, ay, bx, by = seg - push_panel(ax + _off, ay, bx + _off, by, 1.0, 9) + # Pass 1: always + for seg in _frame_segs: + push_panel(*seg, _fconf, _fpid) + # Pass 2: thicker when quality ≥ 0.50 + if _q >= 0.5: + for ax, ay, bx, by in _frame_segs: + push_panel(ax + 0.001, ay, bx + 0.001, by, _fconf, _fpid) + # Pass 3: boldest when quality ≥ 0.85 or pinch engaged + if _q >= 0.85 or _st == 3: + for ax, ay, bx, by in _frame_segs: + push_panel(ax + 0.002, ay, bx + 0.002, by, _fconf, _fpid) + # Position gauges (X below / Y outer-side) + _slot_key = "L" if side_name == "left" else "R" + _slot_hf = _hf_all.get(_slot_key) + _g_cx = _slot_hf["cx"] if _slot_hf else None + _g_cy = _slot_hf["cy"] if _slot_hf else None + for ax, ay, bx, by, gconf, gpid in gauge_segments( + _g_cx, _g_cy, side_name, asp, mirror, + content_pid=pid_s, + ): + push_panel(ax, ay, bx, by, gconf, gpid) if h_kp is not None: for seg in panel_segments( h_kp, side_name, lhand_bones_p, asp, diff --git a/data_only_viz/state.py b/data_only_viz/state.py index 96a33de..19fe45d 100644 --- a/data_only_viz/state.py +++ b/data_only_viz/state.py @@ -179,6 +179,9 @@ class State: # Gesture slot status per hand slot (written by action_head_pub, read by renderer): # 0=absent, 1=detected(plausible+established not armed), 2=armed(near+facing), 3=pinch engaged gesture_slot_status: list[int] = field(default_factory=lambda: [0, 0]) + # Continuous quality score per slot ∈ [0, 1] (written alongside gesture_slot_status). + # Drives panel frame brightness (0.25+0.75*q) and stroke thickness. + gesture_slot_quality: list[float] = field(default_factory=lambda: [0.0, 0.0]) # Renderer width: int = 1280 diff --git a/data_only_viz/tests/test_hand_display.py b/data_only_viz/tests/test_hand_display.py index 6d1f409..d5d3e1f 100644 --- a/data_only_viz/tests/test_hand_display.py +++ b/data_only_viz/tests/test_hand_display.py @@ -12,6 +12,8 @@ import pytest from data_only_viz.hand_display import ( HandPersistenceGate, arkit_2d_fresh, + gauge_segments, + gesture_quality, hand_facing, hand_plausible, hand_size, @@ -19,6 +21,8 @@ from data_only_viz.hand_display import ( panel_rect, panel_segments, panel_frame, + GAUGE_GAP, + GAUGE_TICK_HALF, PANEL_SIDE, PANEL_MARGIN, PANEL_INNER, @@ -616,3 +620,153 @@ def test_hand_facing_list_format(): f = hand_facing(lm) # facing = 0.14 / 0.15 ~ 0.93 assert f >= 0.7 + + +# --------------------------------------------------------------------------- +# gesture_quality +# --------------------------------------------------------------------------- + +def _make_quality_hand(size: float = 0.10, facing: float = 0.7) -> list[Kp]: + """21-kp hand with precise hand_size and hand_facing values. + + hand_size = wrist(kp[0]) to middle-MCP(kp[9]) distance = size + hand_facing = dist(kp[5], kp[17]) / hand_size = facing + """ + kp = _make_hand(wrist_x=0.5, wrist_y=0.5, size=size, c=1.0) + spread = facing * size + mid_y = 0.5 - size / 2 + kp[5] = Kp(x=0.5 - spread / 2, y=mid_y) + kp[17] = Kp(x=0.5 + spread / 2, y=mid_y) + return kp + + +def test_quality_absent_hand_returns_zero(): + """hand=None → quality 0.0 regardless of other params.""" + assert gesture_quality(None, face_min=0.5, near_on=0.10) == 0.0 + + +def test_quality_engaged_forces_one(): + """engaged=True → quality 1.0 regardless of hand geometry.""" + hand = _make_quality_hand(size=0.05, facing=0.1) # far and side-on + assert gesture_quality(hand, face_min=0.5, near_on=0.10, engaged=True) == 1.0 + + +def test_quality_far_but_fully_facing(): + """Hand at near-threshold (size=0.5*near_on) but fully facing → 0.65. + + near_norm = 0.0 (at lower bound), facing_norm = 1.0 → 0.30 + 0.35 = 0.65. + """ + face_min, near_on = 0.5, 0.10 + # size = 0.5 * near_on = 0.05 → near_norm = 0 + # facing = face_min = 0.5 → facing_norm = (0.5-0.25)/(0.5-0.25) = 1.0 + hand = _make_quality_hand(size=0.05, facing=face_min) + q = gesture_quality(hand, face_min=face_min, near_on=near_on) + assert math.isclose(q, 0.65, rel_tol=1e-6), f"expected 0.65, got {q}" + + +def test_quality_near_but_side_on(): + """Hand fully near (size=near_on) but side-on (facing=0.25) → 0.65. + + facing_norm = 0.0 (at lower bound), near_norm = 1.0 → 0.30 + 0.35 = 0.65. + """ + face_min, near_on = 0.5, 0.10 + # size = near_on = 0.10 → near_norm = (0.10-0.05)/0.05 = 1.0 + # facing = 0.25 → facing_norm = (0.25-0.25)/0.25 = 0.0 + hand = _make_quality_hand(size=near_on, facing=0.25) + q = gesture_quality(hand, face_min=face_min, near_on=near_on) + assert math.isclose(q, 0.65, rel_tol=1e-6), f"expected 0.65, got {q}" + + +def test_quality_full_caps_at_one(): + """Hand well above near_on and facing threshold → quality = 1.0.""" + face_min, near_on = 0.5, 0.10 + hand = _make_quality_hand(size=0.20, facing=0.9) # near_norm=1, facing_norm=1 + q = gesture_quality(hand, face_min=face_min, near_on=near_on) + assert math.isclose(q, 1.0, rel_tol=1e-6), f"expected 1.0, got {q}" + + +def test_quality_monotonic_in_size(): + """For fixed full-facing, quality increases strictly with hand size.""" + face_min, near_on = 0.5, 0.10 + facing = face_min # facing_norm = 1.0 at exactly face_min + sizes = [0.5 * near_on, 0.75 * near_on, near_on] + qs = [gesture_quality(_make_quality_hand(size=s, facing=facing), + face_min=face_min, near_on=near_on) + for s in sizes] + assert qs[0] < qs[1] < qs[2], f"not monotone: {qs}" + + +# --------------------------------------------------------------------------- +# gauge_segments +# --------------------------------------------------------------------------- + +def test_gauge_segments_no_data_returns_two_dim_rails(): + """cx/cy=None → 2 rail segments (dim, no markers).""" + segs = gauge_segments(None, None, "left", aspect=1.0) + # Exactly 2 rails: one horizontal (X gauge), one vertical (Y gauge) + assert len(segs) == 2, f"expected 2 rail segs, got {len(segs)}" + confs = [s[4] for s in segs] + assert all(c == 0.25 for c in confs), f"dim conf expected 0.25, got {confs}" + pids = [s[5] for s in segs] + assert all(p == 7 for p in pids), f"rail pid should be 7, got {pids}" + + +def test_gauge_segments_with_data_returns_rails_and_markers(): + """cx/cy provided → 2 rails + 2 X-marker ticks + 2 Y-marker ticks = 6 total.""" + segs = gauge_segments(0.5, 0.5, "left", aspect=1.0) + assert len(segs) == 6, f"expected 6 segments (2 rail + 4 marker), got {len(segs)}" + rail_segs = [s for s in segs if s[5] == 7] + marker_segs = [s for s in segs if s[5] != 7] + assert len(rail_segs) == 2 and len(marker_segs) == 4 + + +def test_gauge_x_marker_centered_on_rail(): + """cx=0.5 → primary X marker tick is at x = (x0+x1)/2 of the panel.""" + x0, _y0, x1, _y1 = panel_rect("left", aspect=1.0) + center_x = (x0 + x1) / 2.0 + segs = gauge_segments(0.5, 0.5, "left", aspect=1.0, content_pid=5) + # X marker ticks are vertical segments (same x on both endpoints), pid != 7 + x_marker_segs = [s for s in segs + if s[5] != 7 and math.isclose(s[0], s[2], abs_tol=1e-9)] + assert x_marker_segs, "no vertical X-marker ticks found" + # The primary tick (minimum x) should land at center_x; the bold-offset + # tick is at center_x + GAUGE_BOLD (a tiny offset for visual thickness). + primary_x = min(s[0] for s in x_marker_segs) + assert math.isclose(primary_x, center_x, rel_tol=1e-6), \ + f"primary X marker at {primary_x}, expected {center_x}" + + +def test_gauge_mirror_flips_x_marker(): + """mirror=True → X marker at 1-cx (mirrored position), cx unchanged for Y.""" + x0, _y0, x1, _y1 = panel_rect("left", aspect=1.0) + cx = 0.3 + # Without mirror: marker at x0 + 0.3*(x1-x0) + segs_norm = gauge_segments(cx, 0.5, "left", 1.0, mirror=False, content_pid=5) + # With mirror: marker at x0 + 0.7*(x1-x0) + segs_mirr = gauge_segments(cx, 0.5, "left", 1.0, mirror=True, content_pid=5) + # Extract first X-marker tick (vertical segment, pid=5) + def _x_tick_x(segs): + for ax, ay, bx, by, c, p in segs: + if p == 5 and math.isclose(ax, bx, rel_tol=1e-9): + return ax + return None + xn = _x_tick_x(segs_norm) + xm = _x_tick_x(segs_mirr) + assert xn is not None and xm is not None + expected_n = x0 + cx * (x1 - x0) + expected_m = x0 + (1.0 - cx) * (x1 - x0) + assert math.isclose(xn, expected_n, rel_tol=1e-6), f"normal: {xn} != {expected_n}" + assert math.isclose(xm, expected_m, rel_tol=1e-6), f"mirror: {xm} != {expected_m}" + + +def test_gauge_y_rail_below_panel_x_rail(): + """X gauge rail y-coordinate is below the panel bottom edge.""" + _x0, _y0, _x1, y1 = panel_rect("left", aspect=1.0) + segs = gauge_segments(0.5, 0.5, "left", 1.0) + # Horizontal rail: ay == by (same y, spanning x) + h_rail = [s for s in segs if math.isclose(s[1], s[3], rel_tol=1e-9) + and s[5] == 7] + assert h_rail, "no horizontal rail segment" + rail_y = h_rail[0][1] + assert rail_y > y1, f"X gauge rail y={rail_y} should be below panel y1={y1}" + assert math.isclose(rail_y, y1 + GAUGE_GAP, rel_tol=1e-6)