L'électron rare
32a722e281
CI build oscope-of / build-check (push) Has been cancelled
Add gesture_slot_status [0..3] to State (absent/detected/armed/engaged). PinchDetector.engaged_slots() exposes per-slot pinch state. Publisher writes status after each tick via _update_gesture_slot_status(). Renderer draws panel frame always: dim (pid7 conf=0.3) when absent, normal (pid7) detected, pid8 armed, pid9 double-stroke engaged.
236 lines
10 KiB
Python
236 lines
10 KiB
Python
"""Air-piano finger strike detection from pre-routed hand slots.
|
|
|
|
A "strike" is a fast downward motion of a fingertip RELATIVE to its base
|
|
knuckle, so translating the whole hand does not fire all fingers. Output feeds
|
|
the OSC /pose/finger route consumed by SuperCollider.
|
|
|
|
Consumers call route_hands() (hand_slots) once per tick and pass the resulting
|
|
2-element [left|None, right|None] list to FingerStrikeDetector.step() and
|
|
PinchDetector.step(). slot 0 = left hand (matrix "MG"), slot 1 = right.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from dataclasses import dataclass
|
|
|
|
from data_only_viz.hand_features import _clamp, _coord, _finite
|
|
|
|
# MediaPipe 21-kp hand: fingertip and base-knuckle indices per finger
|
|
# (thumb, index, middle, ring, pinky). Thumb base = ThumbMP (2).
|
|
FINGERTIPS: tuple[int, ...] = (4, 8, 12, 16, 20)
|
|
FINGER_BASES: tuple[int, ...] = (2, 5, 9, 13, 17)
|
|
|
|
# Pinch: thumb tip vs the 4 opposable finger tips (index..pinky).
|
|
THUMB_TIP: int = 4
|
|
WRIST: int = 0
|
|
MIDDLE_MCP: int = 9
|
|
PINCH_TIPS: tuple[int, ...] = (8, 12, 16, 20)
|
|
|
|
|
|
@dataclass
|
|
class StrikeEvent:
|
|
hand: int # 0 = left slot, 1 = right slot
|
|
finger: int # 0..4 = thumb, index, middle, ring, pinky
|
|
strike_speed: float
|
|
z: float
|
|
tipx: float
|
|
tipy: float
|
|
|
|
|
|
class _FingerState:
|
|
__slots__ = ("prev_rel", "armed", "last_t")
|
|
|
|
def __init__(self) -> None:
|
|
self.prev_rel: float | None = None
|
|
self.armed: bool = True
|
|
self.last_t: float = -1e9
|
|
|
|
|
|
class FingerStrikeDetector:
|
|
def __init__(self, vel_thresh: float = 0.02, refractory_ms: float = 120.0,
|
|
speed_scale: float = 0.10, history_slots: int = 2) -> None:
|
|
self.vel_thresh = vel_thresh
|
|
self.refractory_s = refractory_ms / 1000.0
|
|
self.speed_scale = max(1e-6, speed_scale)
|
|
# state[slot][finger]
|
|
self._state = [[_FingerState() for _ in range(5)]
|
|
for _ in range(history_slots)]
|
|
|
|
def reset_slot(self, slot: int) -> None:
|
|
"""Clear per-finger prev_rel and re-arm for the given slot.
|
|
|
|
Call this BEFORE step() when GestureSlotStabilizer.resumed_flags()
|
|
reports a held -> real transition for that slot. During a Vision hole
|
|
the stabilizer replays the last hand (frozen coords, vel=0), but when
|
|
the real hand returns within the hold window the first delta compresses
|
|
several frames of motion into one large velocity spike, causing a
|
|
phantom strike. Clearing prev_rel makes the first real frame a
|
|
no-op prime that restarts tracking from the new position.
|
|
"""
|
|
for f in range(5):
|
|
self._state[slot][f].prev_rel = None
|
|
self._state[slot][f].armed = True
|
|
|
|
def step(self, slotted: list, t_now: float) -> list[StrikeEvent]:
|
|
"""Process a pre-routed 2-slot hand list [left|None, right|None].
|
|
|
|
Absent slots (None) reset their finger state so re-entry does not
|
|
produce spurious velocity spikes.
|
|
"""
|
|
events: list[StrikeEvent] = []
|
|
for slot, lm in enumerate(slotted):
|
|
if lm is None:
|
|
for f in range(5):
|
|
self._state[slot][f].prev_rel = None
|
|
self._state[slot][f].armed = True
|
|
continue
|
|
for f in range(5):
|
|
tip = lm[FINGERTIPS[f]]
|
|
base = lm[FINGER_BASES[f]]
|
|
tip_y = _finite(_coord(tip, "y", 1), 0.5)
|
|
base_y = _finite(_coord(base, "y", 1), 0.5)
|
|
rel = tip_y - base_y # +down (image y grows downward)
|
|
st = self._state[slot][f]
|
|
if st.prev_rel is None:
|
|
st.prev_rel = rel
|
|
continue
|
|
vel = rel - st.prev_rel # +down velocity per frame
|
|
st.prev_rel = rel
|
|
if vel < 0.0: # lifting -> rearm
|
|
st.armed = True
|
|
if (vel > self.vel_thresh and st.armed
|
|
and (t_now - st.last_t) >= self.refractory_s):
|
|
st.armed = False
|
|
st.last_t = t_now
|
|
events.append(StrikeEvent(
|
|
hand=slot, finger=f,
|
|
strike_speed=_clamp(vel / self.speed_scale, 0.0, 1.0),
|
|
z=_finite(_coord(tip, "z", 2, 0.0), 0.0),
|
|
tipx=_finite(_coord(tip, "x", 0), 0.5),
|
|
tipy=tip_y,
|
|
))
|
|
return events
|
|
|
|
|
|
@dataclass
|
|
class PinchEvent:
|
|
hand: int # 0 = left slot, 1 = right slot
|
|
finger: int # 1..4 = index, middle, ring, pinky (thumb is trigger)
|
|
state: int = 1 # 1 = engage edge, 0 = release edge
|
|
|
|
|
|
class _PinchState:
|
|
__slots__ = ("engaged", "last_t", "qual")
|
|
|
|
def __init__(self) -> None:
|
|
self.engaged: bool = False
|
|
self.last_t: float = -1e9
|
|
self.qual: int = 0 # consecutive qualifying frames (debounce)
|
|
|
|
|
|
class PinchDetector:
|
|
"""Edge-triggered thumb-to-finger pinch with hysteresis.
|
|
|
|
Fires one PinchEvent when thumb tip contacts a finger tip (distance,
|
|
normalized by hand size, drops below ratio_on). Re-arms only after the
|
|
distance rises back above ratio_off, so one pinch = one event.
|
|
"""
|
|
|
|
def __init__(self, ratio_on: float = 0.45, ratio_off: float = 0.65,
|
|
refractory_ms: float = 250.0, history_slots: int = 2,
|
|
margin: float = 0.20, ext_ratio: float = 1.35,
|
|
ext_min: int = 0, debounce_frames: int = 1) -> None:
|
|
self.ratio_on = ratio_on
|
|
self.ratio_off = ratio_off
|
|
self.refractory_s = refractory_ms / 1000.0
|
|
# Only the single nearest fingertip may engage, and only if it is at
|
|
# least `margin` (in size-normalized units) nearer than the runner-up.
|
|
# Rejects the adjacent-finger ambiguity when fingers curl together.
|
|
self.margin = margin
|
|
# Open-hand gate: the winner may engage only when at least ext_min
|
|
# of the 3 non-pinching fingers are extended (tip-to-wrist distance
|
|
# above ext_ratio hand-sizes). Rejects relaxed-hand/fist false
|
|
# pinches during full-body play. ext_min=0 disables the gate; the
|
|
# live defaults come from the PINCH_EXT_* env vars (action_head_pub),
|
|
# constructor defaults preserve legacy behavior.
|
|
self.ext_ratio = ext_ratio
|
|
self.ext_min = int(ext_min)
|
|
# Engage fires only after debounce_frames consecutive qualifying
|
|
# frames; release stays immediate. 1 = no debounce.
|
|
self.debounce_frames = max(1, int(debounce_frames))
|
|
self._state = [[_PinchState() for _ in range(4)]
|
|
for _ in range(history_slots)]
|
|
|
|
def engaged_slots(self) -> tuple[bool, bool]:
|
|
"""Return (slot0_has_engaged_pinch, slot1_has_engaged_pinch)."""
|
|
return (
|
|
any(self._state[0][i].engaged for i in range(4)),
|
|
any(self._state[1][i].engaged for i in range(4)),
|
|
)
|
|
|
|
def step(self, slotted: list, t_now: float) -> list[PinchEvent]:
|
|
"""Process a pre-routed 2-slot hand list [left|None, right|None].
|
|
|
|
Absent slots synthesise release edges for any engaged fingers and
|
|
reset debounce state.
|
|
"""
|
|
events: list[PinchEvent] = []
|
|
for slot, lm in enumerate(slotted):
|
|
if lm is None:
|
|
for i in range(4):
|
|
st = self._state[slot][i]
|
|
st.qual = 0
|
|
if st.engaged:
|
|
st.engaged = False
|
|
events.append(PinchEvent(hand=slot, finger=i + 1, state=0))
|
|
continue
|
|
tx = _finite(_coord(lm[THUMB_TIP], "x", 0), 0.5)
|
|
ty = _finite(_coord(lm[THUMB_TIP], "y", 1), 0.5)
|
|
wx = _finite(_coord(lm[WRIST], "x", 0), 0.5)
|
|
wy = _finite(_coord(lm[WRIST], "y", 1), 0.5)
|
|
mx = _finite(_coord(lm[MIDDLE_MCP], "x", 0), 0.5)
|
|
my = _finite(_coord(lm[MIDDLE_MCP], "y", 1), 0.5)
|
|
size = math.hypot(mx - wx, my - wy)
|
|
size = size if size > 1e-4 else 1e-4
|
|
ratios = []
|
|
exts = []
|
|
for tip_idx in PINCH_TIPS:
|
|
fx = _finite(_coord(lm[tip_idx], "x", 0), 0.5)
|
|
fy = _finite(_coord(lm[tip_idx], "y", 1), 0.5)
|
|
ratios.append(math.hypot(fx - tx, fy - ty) / size)
|
|
exts.append(math.hypot(fx - wx, fy - wy) / size)
|
|
# closest-wins + margin: pick the single nearest fingertip, and treat
|
|
# it as a pinch only if it is clearly nearer than the runner-up.
|
|
order = sorted(range(4), key=lambda j: ratios[j])
|
|
nearest, runner = order[0], order[1]
|
|
# open-hand gate: a deliberate pinch keeps the other fingers
|
|
# extended; a fist/relaxed hand has them curled at the wrist.
|
|
open_ok = self.ext_min <= 0 or sum(
|
|
1 for j in range(4)
|
|
if j != nearest and exts[j] >= self.ext_ratio
|
|
) >= self.ext_min
|
|
winner = nearest if (
|
|
ratios[nearest] < self.ratio_on
|
|
and (ratios[runner] - ratios[nearest]) >= self.margin
|
|
and open_ok
|
|
) else -1
|
|
for i in range(4):
|
|
st = self._state[slot][i]
|
|
if st.engaged:
|
|
# release when this finger opens, or another finger took over.
|
|
if i != winner or ratios[i] > self.ratio_off:
|
|
st.engaged = False
|
|
events.append(PinchEvent(hand=slot, finger=i + 1, state=0))
|
|
elif i == winner and (t_now - st.last_t) >= self.refractory_s:
|
|
# engage only after debounce_frames consecutive qualifying
|
|
# frames; release below stays edge-immediate.
|
|
st.qual += 1
|
|
if st.qual >= self.debounce_frames:
|
|
st.engaged = True
|
|
st.last_t = t_now
|
|
st.qual = 0
|
|
events.append(PinchEvent(hand=slot, finger=i + 1, state=1))
|
|
else:
|
|
st.qual = 0
|
|
return events
|