feat(viz): continuous frame quality intensity
CI build oscope-of / build-check (push) Has been cancelled

Add gesture_quality() pure helper (hand_display.py) that maps a slot's
plausibility, facing, and proximity to a score in [0,1]. Renderer uses
it to modulate panel frame brightness (0.25+0.75*q) and stroke count
(1..3 passes at q≥0, ≥0.5, ≥0.85 / status==3). Status still drives
hue (pid 7/8/9). New gesture_slot_quality field in State written
alongside gesture_slot_status each tick.

Add gauge_segments() for X/Y position gauges around each panel: a
horizontal rail below and a vertical rail on the outer side, each with
a bold two-tick notch at the hand's normalised cx/cy (mirror-aware for
X). Dim rails (conf 0.25) when slot absent. Renderer reads hand_feats
L/R directly from state.

11 new tests (6 gesture_quality + 5 gauge_segments); suite 430 passed.
This commit is contained in:
L'électron rare
2026-07-02 16:34:08 +02:00
parent 32a722e281
commit 6fe98c6b00
5 changed files with 364 additions and 27 deletions
+13 -2
View File
@@ -24,7 +24,7 @@ from data_only_viz.action_head import (
J3D_FINGERS_PER_HAND,
LABELS,
)
from data_only_viz.hand_display import hand_plausible, HandPersistenceGate
from data_only_viz.hand_display import gesture_quality, hand_plausible, HandPersistenceGate
from data_only_viz.hand_slots import route_hands
LOG = logging.getLogger("action_head_pub")
@@ -350,10 +350,11 @@ class ActionHeadPublisher(threading.Thread):
self.bridge.send_pinch(ev)
def _update_gesture_slot_status(self) -> None:
"""Compute per-slot gesture status and write it to state.
"""Compute per-slot gesture status and quality, write both to state.
Status values: 0=absent, 1=detected(plausible+established, not armed),
2=armed(near+facing), 3=pinch engaged.
Quality ∈ [0, 1]: continuous intensity for frame brightness + thickness.
Called once per tick after all emitters have run.
"""
stab = getattr(self, "_stab", None)
@@ -364,6 +365,9 @@ class ActionHeadPublisher(threading.Thread):
active = stab.active_flags()
engaged = pinch_det.engaged_slots() if pinch_det is not None else (False, False)
status = [0, 0]
quality = [0.0, 0.0]
face_min = stab._face_min
near_on = stab._near_on
for slot in range(2):
if engaged[slot]:
status[slot] = 3
@@ -371,8 +375,15 @@ class ActionHeadPublisher(threading.Thread):
status[slot] = 2
elif raw[slot] is not None:
status[slot] = 1
quality[slot] = gesture_quality(
raw[slot],
face_min=face_min,
near_on=near_on,
engaged=bool(engaged[slot]),
)
with self.state.lock():
self.state.gesture_slot_status = status
self.state.gesture_slot_quality = quality
def _read_sources(self) -> tuple[
list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] | None,
+159 -5
View File
@@ -2,11 +2,19 @@
No Metal, no numpy, no pyobjc — safe to unit-test anywhere.
Panel frame status indicator color legend (used in renderer.py):
pid 7 conf 0.3 → status 0: absent (dim frame)
pid 7 conf 1.0 → status 1: detected (plausible+established, not armed)
pid 8 conf 1.0 → status 2: armed (near + facing camera)
pid 9 conf 1.0 → status 3: pinch engaged (double-stroke bold)
Panel frame legend (renderer.py):
pid 7 conf=0.25+0.75*q → status 0: absent (q=0); status 1: detected (q≥0.30)
pid 8 conf=0.25+0.75*q → status 2: armed (near+facing camera, q≥0.30)
pid 9 conf=1.0 (q forced)→ status 3: pinch engaged
Thickness: 1 stroke always; +1 at q≥0.50; +1 at q≥0.85 or status==3.
See gesture_quality() for the continuous quality formula.
Position gauges (renderer.py, drawn around each panel):
X gauge: horizontal rail BELOW panel (pid 7, conf 0.4 or 0.25 if absent);
vertical notch marker at cx position (pid 5/6, conf 1.0).
Y gauge: vertical rail on outer side (pid 7, conf 0.4 or 0.25 if absent);
horizontal notch marker at cy position (pid 5/6, conf 1.0).
See gauge_segments() — marker x is mirror-flipped if mirror=True.
"""
from __future__ import annotations
@@ -366,3 +374,149 @@ class HandPersistenceGate:
self._tracks = new_tracks
return result
# ---------------------------------------------------------------------------
# Gauge layout constants
# ---------------------------------------------------------------------------
#: Normalised gap between a panel edge and its adjacent gauge rail.
GAUGE_GAP: float = 0.012
#: Half the tick length (total = 2 * GAUGE_TICK_HALF), in normalised coords.
GAUGE_TICK_HALF: float = 0.004
#: Parallel offset for the second stroke of a bold-notch marker.
GAUGE_BOLD: float = 0.001
# ---------------------------------------------------------------------------
# Continuous quality score
# ---------------------------------------------------------------------------
def gesture_quality(
hand: object,
*,
face_min: float,
near_on: float,
engaged: bool = False,
) -> float:
"""Continuous quality score ∈ [0, 1] for one gesture slot.
Drives panel frame brightness and stroke thickness in renderer.py.
Args:
hand: 21-kp hand object (attribute-style .x/.y) or None.
``None`` → 0.0 (slot absent or not yet validated by gates).
face_min: palm-spread ratio at which the slot is considered fully
armed (from GestureSlotStabilizer; default env 0.5).
near_on: wrist-to-middle-MCP distance (normalised) at which the slot
is fully near (from GestureSlotStabilizer; default 0.10).
engaged: True when a pinch is held for this slot → forces 1.0.
Returns:
quality ∈ [0, 1]:
* 0.0 when ``hand`` is None.
* 1.0 when ``engaged`` is True.
* else: 0.30·established + 0.35·facing_norm + 0.35·near_norm
where established = 1.0 (hand passed plausibility + temporal gates),
facing_norm = clamp((hand_facing(hand) 0.25) / (face_min 0.25), 0, 1),
near_norm = clamp((hand_size(hand) 0.5·near_on) / (0.5·near_on), 0, 1).
"""
if hand is None:
return 0.0
if engaged:
return 1.0
# facing_norm: 0 at facing = 0.25, 1 at facing = face_min.
denom_f = face_min - 0.25
if denom_f > 1e-9:
facing_norm = min(1.0, max(0.0, (hand_facing(hand) - 0.25) / denom_f))
else:
# face_min ≤ 0.25 — any spread qualifies.
facing_norm = 1.0
# near_norm: 0 at hand_size = 0.5·near_on, 1 at hand_size = near_on.
half_on = 0.5 * near_on
if half_on > 1e-9:
near_norm = min(1.0, max(0.0, (hand_size(hand) - half_on) / half_on))
else:
near_norm = 1.0
return 0.30 + 0.35 * facing_norm + 0.35 * near_norm
# ---------------------------------------------------------------------------
# Position gauges
# ---------------------------------------------------------------------------
def gauge_segments(
cx: float | None,
cy: float | None,
side: str,
aspect: float,
mirror: bool = False,
*,
content_pid: int = 5,
rail_pid: int = 7,
) -> list[tuple[float, float, float, float, float, int]]:
"""Rail + marker segments for the X and Y position gauges around a panel.
X gauge: a horizontal rail placed GAUGE_GAP below the panel bottom edge,
spanning the full panel width. A bold vertical notch marks the
hand's cx position on the rail.
Y gauge: a vertical rail placed GAUGE_GAP outside the panel's outer edge
(left of the left panel, right of the right panel), spanning the
full panel height. A bold horizontal notch marks cy.
Args:
cx: Normalised hand centre-x ∈ [0, 1], or None when absent.
cy: Normalised hand centre-y ∈ [0, 1], or None when absent.
side: "left" or "right".
aspect: View width / view height (> 0).
mirror: If True, flip the cx marker position (1 cx) so it moves
in the same direction as the mirrored video. cy is never
flipped.
content_pid: pid for marker segments (5 = left slot, 6 = right slot).
rail_pid: pid for rail segments (default 7).
Returns:
List of (ax, ay, bx, by, conf, pid) tuples ready to feed
``push_panel`` in the renderer. When cx/cy is None the rails are
drawn dim (conf 0.25) with no marker; otherwise rails use conf 0.4
and markers use conf 1.0.
"""
x0, y0, x1, y1 = panel_rect(side, aspect)
has_data = cx is not None and cy is not None
rail_conf = 0.4 if has_data else 0.25
out: list[tuple[float, float, float, float, float, int]] = []
# ---- X gauge: horizontal rail below panel ----
xg_y = y1 + GAUGE_GAP
out.append((x0, xg_y, x1, xg_y, rail_conf, rail_pid))
if has_data:
assert cx is not None and cy is not None # narrowing for type checkers
cx_eff = 1.0 - cx if mirror else cx
mx = x0 + cx_eff * (x1 - x0)
# Bold vertical notch: two parallel vertical ticks
out.append((mx, xg_y - GAUGE_TICK_HALF, mx, xg_y + GAUGE_TICK_HALF,
1.0, content_pid))
out.append((mx + GAUGE_BOLD, xg_y - GAUGE_TICK_HALF,
mx + GAUGE_BOLD, xg_y + GAUGE_TICK_HALF,
1.0, content_pid))
# ---- Y gauge: vertical rail on outer side ----
if side == "left":
yg_x = x0 - GAUGE_GAP
else:
yg_x = x1 + GAUGE_GAP
out.append((yg_x, y0, yg_x, y1, rail_conf, rail_pid))
if has_data:
assert cx is not None and cy is not None
my = y0 + cy * (y1 - y0)
# Bold horizontal notch: two parallel horizontal ticks
out.append((yg_x - GAUGE_TICK_HALF, my, yg_x + GAUGE_TICK_HALF, my,
1.0, content_pid))
out.append((yg_x - GAUGE_TICK_HALF, my + GAUGE_BOLD,
yg_x + GAUGE_TICK_HALF, my + GAUGE_BOLD,
1.0, content_pid))
return out
+35 -20
View File
@@ -47,6 +47,7 @@ from .arkit_skeleton import arkit_segments, finger_joint_mask
from .hand_display import (
HandPersistenceGate,
arkit_2d_fresh,
gauge_segments,
hand_plausible,
hand_size,
segment_ok,
@@ -528,33 +529,47 @@ class MetalRenderer(NSObject):
)
left_kp, right_kp = slotted
# Panel frame colored by gesture status (read from state, already under lock):
# pid 7 conf 0.3 = status 0 (absent, dim)
# pid 7 conf 1.0 = status 1 (detected, normal)
# pid 8 conf 1.0 = status 2 (armed: near+facing)
# pid 9 conf 1.0 = status 3 (pinch engaged, double-stroke)
# Panel frame: status drives hue (pid 7/8/9), quality drives
# brightness (0.25+0.75*q) and thickness (1..3 stroke passes).
# pid 7 conf=0.25+0.75*q → status 0: absent (q=0)
# → status 1: detected (q≥0.30)
# pid 8 conf=0.25+0.75*q → status 2: armed (near+facing)
# pid 9 conf=1.0 (q=1.0) → status 3: pinch engaged
# Thickness: pass 1 always; pass 2 at q≥0.50; pass 3 at q≥0.85 or status==3.
_slot_status = getattr(s, "gesture_slot_status", [0, 0])
_slot_quality = getattr(s, "gesture_slot_quality", [0.0, 0.0])
_hf_all = getattr(s, "hand_feats", None) or {}
for _si, (side_name, h_kp, pid_s) in enumerate((
("left", left_kp, 5),
("right", right_kp, 6),
)):
_st = _slot_status[_si] if _si < len(_slot_status) else 0
_q = float(_slot_quality[_si]) if _si < len(_slot_quality) else 0.0
_fconf = 0.25 + 0.75 * _q
# Status drives hue (pid selection)
_fpid = 9 if _st == 3 else (8 if _st == 2 else 7)
_frame_segs = panel_frame(side_name, asp)
if _st == 0:
for seg in _frame_segs:
push_panel(*seg, 0.3, 7) # absent: dim
elif _st == 1:
for seg in _frame_segs:
push_panel(*seg, 1.0, 7) # detected: normal
elif _st == 2:
for seg in _frame_segs:
push_panel(*seg, 1.0, 8) # armed: distinct palette color
else: # 3: pinch engaged
_off = 0.001 # tiny offset for bold double-stroke
for seg in _frame_segs:
push_panel(*seg, 1.0, 9)
ax, ay, bx, by = seg
push_panel(ax + _off, ay, bx + _off, by, 1.0, 9)
# Pass 1: always
for seg in _frame_segs:
push_panel(*seg, _fconf, _fpid)
# Pass 2: thicker when quality ≥ 0.50
if _q >= 0.5:
for ax, ay, bx, by in _frame_segs:
push_panel(ax + 0.001, ay, bx + 0.001, by, _fconf, _fpid)
# Pass 3: boldest when quality ≥ 0.85 or pinch engaged
if _q >= 0.85 or _st == 3:
for ax, ay, bx, by in _frame_segs:
push_panel(ax + 0.002, ay, bx + 0.002, by, _fconf, _fpid)
# Position gauges (X below / Y outer-side)
_slot_key = "L" if side_name == "left" else "R"
_slot_hf = _hf_all.get(_slot_key)
_g_cx = _slot_hf["cx"] if _slot_hf else None
_g_cy = _slot_hf["cy"] if _slot_hf else None
for ax, ay, bx, by, gconf, gpid in gauge_segments(
_g_cx, _g_cy, side_name, asp, mirror,
content_pid=pid_s,
):
push_panel(ax, ay, bx, by, gconf, gpid)
if h_kp is not None:
for seg in panel_segments(
h_kp, side_name, lhand_bones_p, asp,
+3
View File
@@ -179,6 +179,9 @@ class State:
# Gesture slot status per hand slot (written by action_head_pub, read by renderer):
# 0=absent, 1=detected(plausible+established not armed), 2=armed(near+facing), 3=pinch engaged
gesture_slot_status: list[int] = field(default_factory=lambda: [0, 0])
# Continuous quality score per slot ∈ [0, 1] (written alongside gesture_slot_status).
# Drives panel frame brightness (0.25+0.75*q) and stroke thickness.
gesture_slot_quality: list[float] = field(default_factory=lambda: [0.0, 0.0])
# Renderer
width: int = 1280
+154
View File
@@ -12,6 +12,8 @@ import pytest
from data_only_viz.hand_display import (
HandPersistenceGate,
arkit_2d_fresh,
gauge_segments,
gesture_quality,
hand_facing,
hand_plausible,
hand_size,
@@ -19,6 +21,8 @@ from data_only_viz.hand_display import (
panel_rect,
panel_segments,
panel_frame,
GAUGE_GAP,
GAUGE_TICK_HALF,
PANEL_SIDE,
PANEL_MARGIN,
PANEL_INNER,
@@ -616,3 +620,153 @@ def test_hand_facing_list_format():
f = hand_facing(lm)
# facing = 0.14 / 0.15 ~ 0.93
assert f >= 0.7
# ---------------------------------------------------------------------------
# gesture_quality
# ---------------------------------------------------------------------------
def _make_quality_hand(size: float = 0.10, facing: float = 0.7) -> list[Kp]:
"""21-kp hand with precise hand_size and hand_facing values.
hand_size = wrist(kp[0]) to middle-MCP(kp[9]) distance = size
hand_facing = dist(kp[5], kp[17]) / hand_size = facing
"""
kp = _make_hand(wrist_x=0.5, wrist_y=0.5, size=size, c=1.0)
spread = facing * size
mid_y = 0.5 - size / 2
kp[5] = Kp(x=0.5 - spread / 2, y=mid_y)
kp[17] = Kp(x=0.5 + spread / 2, y=mid_y)
return kp
def test_quality_absent_hand_returns_zero():
"""hand=None → quality 0.0 regardless of other params."""
assert gesture_quality(None, face_min=0.5, near_on=0.10) == 0.0
def test_quality_engaged_forces_one():
"""engaged=True → quality 1.0 regardless of hand geometry."""
hand = _make_quality_hand(size=0.05, facing=0.1) # far and side-on
assert gesture_quality(hand, face_min=0.5, near_on=0.10, engaged=True) == 1.0
def test_quality_far_but_fully_facing():
"""Hand at near-threshold (size=0.5*near_on) but fully facing → 0.65.
near_norm = 0.0 (at lower bound), facing_norm = 1.0 → 0.30 + 0.35 = 0.65.
"""
face_min, near_on = 0.5, 0.10
# size = 0.5 * near_on = 0.05 → near_norm = 0
# facing = face_min = 0.5 → facing_norm = (0.5-0.25)/(0.5-0.25) = 1.0
hand = _make_quality_hand(size=0.05, facing=face_min)
q = gesture_quality(hand, face_min=face_min, near_on=near_on)
assert math.isclose(q, 0.65, rel_tol=1e-6), f"expected 0.65, got {q}"
def test_quality_near_but_side_on():
"""Hand fully near (size=near_on) but side-on (facing=0.25) → 0.65.
facing_norm = 0.0 (at lower bound), near_norm = 1.0 → 0.30 + 0.35 = 0.65.
"""
face_min, near_on = 0.5, 0.10
# size = near_on = 0.10 → near_norm = (0.10-0.05)/0.05 = 1.0
# facing = 0.25 → facing_norm = (0.25-0.25)/0.25 = 0.0
hand = _make_quality_hand(size=near_on, facing=0.25)
q = gesture_quality(hand, face_min=face_min, near_on=near_on)
assert math.isclose(q, 0.65, rel_tol=1e-6), f"expected 0.65, got {q}"
def test_quality_full_caps_at_one():
"""Hand well above near_on and facing threshold → quality = 1.0."""
face_min, near_on = 0.5, 0.10
hand = _make_quality_hand(size=0.20, facing=0.9) # near_norm=1, facing_norm=1
q = gesture_quality(hand, face_min=face_min, near_on=near_on)
assert math.isclose(q, 1.0, rel_tol=1e-6), f"expected 1.0, got {q}"
def test_quality_monotonic_in_size():
"""For fixed full-facing, quality increases strictly with hand size."""
face_min, near_on = 0.5, 0.10
facing = face_min # facing_norm = 1.0 at exactly face_min
sizes = [0.5 * near_on, 0.75 * near_on, near_on]
qs = [gesture_quality(_make_quality_hand(size=s, facing=facing),
face_min=face_min, near_on=near_on)
for s in sizes]
assert qs[0] < qs[1] < qs[2], f"not monotone: {qs}"
# ---------------------------------------------------------------------------
# gauge_segments
# ---------------------------------------------------------------------------
def test_gauge_segments_no_data_returns_two_dim_rails():
"""cx/cy=None → 2 rail segments (dim, no markers)."""
segs = gauge_segments(None, None, "left", aspect=1.0)
# Exactly 2 rails: one horizontal (X gauge), one vertical (Y gauge)
assert len(segs) == 2, f"expected 2 rail segs, got {len(segs)}"
confs = [s[4] for s in segs]
assert all(c == 0.25 for c in confs), f"dim conf expected 0.25, got {confs}"
pids = [s[5] for s in segs]
assert all(p == 7 for p in pids), f"rail pid should be 7, got {pids}"
def test_gauge_segments_with_data_returns_rails_and_markers():
"""cx/cy provided → 2 rails + 2 X-marker ticks + 2 Y-marker ticks = 6 total."""
segs = gauge_segments(0.5, 0.5, "left", aspect=1.0)
assert len(segs) == 6, f"expected 6 segments (2 rail + 4 marker), got {len(segs)}"
rail_segs = [s for s in segs if s[5] == 7]
marker_segs = [s for s in segs if s[5] != 7]
assert len(rail_segs) == 2 and len(marker_segs) == 4
def test_gauge_x_marker_centered_on_rail():
"""cx=0.5 → primary X marker tick is at x = (x0+x1)/2 of the panel."""
x0, _y0, x1, _y1 = panel_rect("left", aspect=1.0)
center_x = (x0 + x1) / 2.0
segs = gauge_segments(0.5, 0.5, "left", aspect=1.0, content_pid=5)
# X marker ticks are vertical segments (same x on both endpoints), pid != 7
x_marker_segs = [s for s in segs
if s[5] != 7 and math.isclose(s[0], s[2], abs_tol=1e-9)]
assert x_marker_segs, "no vertical X-marker ticks found"
# The primary tick (minimum x) should land at center_x; the bold-offset
# tick is at center_x + GAUGE_BOLD (a tiny offset for visual thickness).
primary_x = min(s[0] for s in x_marker_segs)
assert math.isclose(primary_x, center_x, rel_tol=1e-6), \
f"primary X marker at {primary_x}, expected {center_x}"
def test_gauge_mirror_flips_x_marker():
"""mirror=True → X marker at 1-cx (mirrored position), cx unchanged for Y."""
x0, _y0, x1, _y1 = panel_rect("left", aspect=1.0)
cx = 0.3
# Without mirror: marker at x0 + 0.3*(x1-x0)
segs_norm = gauge_segments(cx, 0.5, "left", 1.0, mirror=False, content_pid=5)
# With mirror: marker at x0 + 0.7*(x1-x0)
segs_mirr = gauge_segments(cx, 0.5, "left", 1.0, mirror=True, content_pid=5)
# Extract first X-marker tick (vertical segment, pid=5)
def _x_tick_x(segs):
for ax, ay, bx, by, c, p in segs:
if p == 5 and math.isclose(ax, bx, rel_tol=1e-9):
return ax
return None
xn = _x_tick_x(segs_norm)
xm = _x_tick_x(segs_mirr)
assert xn is not None and xm is not None
expected_n = x0 + cx * (x1 - x0)
expected_m = x0 + (1.0 - cx) * (x1 - x0)
assert math.isclose(xn, expected_n, rel_tol=1e-6), f"normal: {xn} != {expected_n}"
assert math.isclose(xm, expected_m, rel_tol=1e-6), f"mirror: {xm} != {expected_m}"
def test_gauge_y_rail_below_panel_x_rail():
"""X gauge rail y-coordinate is below the panel bottom edge."""
_x0, _y0, _x1, y1 = panel_rect("left", aspect=1.0)
segs = gauge_segments(0.5, 0.5, "left", 1.0)
# Horizontal rail: ay == by (same y, spanning x)
h_rail = [s for s in segs if math.isclose(s[1], s[3], rel_tol=1e-9)
and s[5] == 7]
assert h_rail, "no horizontal rail segment"
rail_y = h_rail[0][1]
assert rail_y > y1, f"X gauge rail y={rail_y} should be below panel y1={y1}"
assert math.isclose(rail_y, y1 + GAUGE_GAP, rel_tol=1e-6)