electron-rare/AV-Live
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

chore(main): sync working tree to feat/action-head

Browse Source 
Apres cherry-pick des 14 commits action-head, le working tree main
divergeait encore de feat/action-head sur 21 fichiers a cause des
resolutions modify/delete qui avaient pris la mauvaise version
(c52271e botched merge avait drop ces fichiers, le cherry-pick
voyait alors "modify in feat / deleted in main" et conservait la
mauvaise version dans certains cas).

Cette commit synchronise le working tree main vers feat/action-head
en checkout-ant tous les fichiers depuis feat. Equivalent fonctionnel
a un fast-forward merge sans toucher l'historique.

Files restored : Skeleton3DRenderer.swift, FaceHandOverlay.swift,
test_multihmr_coreml.py, test_pose_bridge_*.py (4 test files).
Files updated : action_head.py, action_head_pub.py, multi.py,
pose_bridge.py, state.py, extract_j3d_offline.py, training/*.py,
SettingsPanel.swift, data_feeds.scd, et plus.

Result : main == feat/action-head sur le working tree, historique
granulaire preserve via les cherry-picks precedents.
This commit is contained in:
L'électron rare
2026-05-14 00:01:03 +02:00
parent 4fd80311e8
commit 531363dea3
22 changed files with 1173 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_only_viz/action_head.py
-84
View File
@@ -32,10 +32,6 @@ LABELS: tuple[str, str, str] = ("debout", "assise", "danse")
EXPR_DIM: int = 10
EXTRA_SCALARS: int = 4 # hip_y, knee_angle, sym_score, mouth_open
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# NEW v3 : MediaPipe Hands keypoints block.
HANDS_KP_PER_HAND: int = 21
HANDS_KP_TOTAL: int = 2 * HANDS_KP_PER_HAND # 42
@@ -44,30 +40,12 @@ HANDS_KP_FLAT: int = HANDS_KP_TOTAL * HANDS_KP_DIMS # 126
# Layout per step (v3) :
# [0 : 96] j3d (32, 3)
<<<<<<< HEAD
# [96 : 192] vel (32, 3)
# [192 : 288] accel (32, 3)
# [288 : 414] hands_kp (42, 3) zero-padded if absent
# [414 : 424] expression (10,)
# [424 : 428] scalars (hip_y, knee_angle, sym, mouth_open)
FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + HANDS_KP_FLAT + EXPR_DIM + EXTRA_SCALARS # 428
=======
# Layout per step:
# [0 : 96 ] j3d (32, 3)
# [96 : 192] vel (32, 3)
# [192 : 288] accel (32, 3)
# [288 : 298] expression (10,)
# [298 : 302] scalars (hip_y, knee_angle, sym, mouth_open)
FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + EXPR_DIM + EXTRA_SCALARS # 302
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
# [96 : 192] vel (32, 3)
# [192 : 288] accel (32, 3)
# [288 : 414] hands_kp (42, 3) zero-padded if absent
# [414 : 424] expression (10,)
# [424 : 428] scalars (hip_y, knee_angle, sym, mouth_open)
FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + HANDS_KP_FLAT + EXPR_DIM + EXTRA_SCALARS # 428
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# Body joint indices (unchanged from v1, indices 0..21).
HIP_LEFT: int = 1
@@ -89,8 +67,6 @@ FINGERTIP_RIGHT_BASE: int = 27
class FeatureExtractor:
"""Stateless feature builder over a list of recent j3d frames.
<<<<<<< HEAD
<<<<<<< HEAD
Vector layout (FEATURE_DIM = 428, v3):
[0 : 96] j3d current frame, flattened (32 joints x 3 dims)
[96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3)
@@ -98,39 +74,13 @@ class FeatureExtractor:
[288 : 414] hands_kp (42, 3) MediaPipe Hands, zero-padded if absent
[414 : 424] expression PCA coefficients (10,)
[424 : 428] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open)
=======
Vector layout (FEATURE_DIM = 302):
[0 : 96 ] j3d current frame, flattened (32 joints x 3 dims)
[96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3)
[192 : 288] acceleration vel[t] - vel[t-1] (32 x 3)
[288 : 298] expression PCA coefficients (10,)
[298 : 302] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open)
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
Vector layout (FEATURE_DIM = 428, v3):
[0 : 96] j3d current frame, flattened (32 joints x 3 dims)
[96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3)
[192 : 288] acceleration vel[t] - vel[t-1] (32 x 3)
[288 : 414] hands_kp (42, 3) MediaPipe Hands, zero-padded if absent
[414 : 424] expression PCA coefficients (10,)
[424 : 428] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open)
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
"""
@staticmethod
def from_buffer(frames: list[np.ndarray],
expr: np.ndarray | None = None,
<<<<<<< HEAD
<<<<<<< HEAD
mouth_open: float = 0.0,
hands_kp: np.ndarray | None = None) -> np.ndarray:
=======
mouth_open: float = 0.0) -> np.ndarray:
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
mouth_open: float = 0.0,
hands_kp: np.ndarray | None = None) -> np.ndarray:
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
if not frames:
return np.zeros(FEATURE_DIM, dtype=np.float32)
cur = frames[-1]
@@ -142,10 +92,6 @@ class FeatureExtractor:
hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5)
knee_angle = FeatureExtractor._mean_knee_angle(cur)
sym = FeatureExtractor._symmetry_score(vel)
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# hands block (42, 3) -> 126
hands_flat = np.zeros(HANDS_KP_FLAT, dtype=np.float32)
if hands_kp is not None:
@@ -153,11 +99,6 @@ class FeatureExtractor:
if hk.shape == (HANDS_KP_TOTAL, HANDS_KP_DIMS):
hands_flat = hk.reshape(-1).astype(np.float32, copy=False)
# expression
<<<<<<< HEAD
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
if expr is None:
expr_vec = np.zeros(EXPR_DIM, dtype=np.float32)
else:
@@ -168,14 +109,7 @@ class FeatureExtractor:
cur.reshape(-1),
vel.reshape(-1),
accel.reshape(-1),
<<<<<<< HEAD
<<<<<<< HEAD
hands_flat,
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
hands_flat,
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
expr_vec,
np.array([hip_y, knee_angle, sym, float(mouth_open)], dtype=np.float32),
]).astype(np.float32, copy=False)
@@ -310,17 +244,8 @@ class ActionHead:
def step(self, pid: int, j3d: np.ndarray,
expr: np.ndarray | None = None,
<<<<<<< HEAD
<<<<<<< HEAD
mouth_open: float = 0.0,
hands_kp: np.ndarray | None = None) -> tuple[str, np.ndarray, np.ndarray]:
=======
mouth_open: float = 0.0) -> tuple[str, np.ndarray, np.ndarray]:
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
mouth_open: float = 0.0,
hands_kp: np.ndarray | None = None) -> tuple[str, np.ndarray, np.ndarray]:
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
if np.isnan(j3d).any():
streak = self._nan_streak.get(pid, 0) + 1
self._nan_streak[pid] = streak
@@ -334,17 +259,8 @@ class ActionHead:
if len(frames) < WARMUP_FRAMES:
probs = np.array([1.0, 0.0, 0.0], dtype=np.float32)
return LABELS[0], probs, np.zeros(3, dtype=np.float32)
<<<<<<< HEAD
<<<<<<< HEAD
feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open,
hands_kp=hands_kp)
=======
feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open)
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open,
hands_kp=hands_kp)
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
kin = FeatureExtractor.kinetics(frames)
h = self._hidden.get(pid)
if h is None:
data_only_viz/action_head_pub.py
-136
View File
@@ -17,18 +17,9 @@ import numpy as np
from data_only_viz.action_head import (
ActionHead,
EXPR_DIM,
<<<<<<< HEAD
<<<<<<< HEAD
HANDS_KP_DIMS,
HANDS_KP_PER_HAND,
HANDS_KP_TOTAL,
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
HANDS_KP_DIMS,
HANDS_KP_PER_HAND,
HANDS_KP_TOTAL,
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
J3D_FINGERS,
J3D_FINGERS_PER_HAND,
LABELS,
@@ -40,30 +31,12 @@ DEFAULT_CKPT = (
Path.home() / ".cache" / "av-live-action" / "checkpoints" / "action_head.pt"
)
<<<<<<< HEAD
<<<<<<< HEAD
# Canonical SMPL-X fingertip vertex IDs from smplx.vertex_ids.SMPLX_VERTEX_IDS.
# Order : L thumb, L index, L middle, L ring, L pinky,
# R thumb, R index, R middle, R ring, R pinky.
SMPLX_FINGERTIP_VERTS: tuple[int, ...] = (
5361, 4933, 5058, 5169, 5286, # L : lthumb, lindex, lmiddle, lring, lpinky
8079, 7669, 7794, 7905, 8022, # R : rthumb, rindex, rmiddle, rring, rpinky
=======
# Approximate fingertip vertex indices on SMPL-X 10475-vert mesh.
# Order: L thumb, L index, L middle, L ring, L pinky,
# R thumb, R index, R middle, R ring, R pinky.
SMPLX_FINGERTIP_VERTS: tuple[int, ...] = (
7174, 7397, 7670, 7942, 8214, # L
4631, 4854, 5127, 5399, 5671, # R
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
# Canonical SMPL-X fingertip vertex IDs from smplx.vertex_ids.SMPLX_VERTEX_IDS.
# Order : L thumb, L index, L middle, L ring, L pinky,
# R thumb, R index, R middle, R ring, R pinky.
SMPLX_FINGERTIP_VERTS: tuple[int, ...] = (
5361, 4933, 5058, 5169, 5286, # L : lthumb, lindex, lmiddle, lring, lpinky
8079, 7669, 7794, 7905, 8022, # R : rthumb, rindex, rmiddle, rring, rpinky
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
)
# 32 vertex indices on the 10475-vertex SMPL-X mesh:
@@ -86,20 +59,11 @@ assert len(SMPLX_JOINT_ANCHOR_VERTS) == 32
SMPLX_UPPER_LIP_VERT: int = 8970
SMPLX_LOWER_LIP_VERT: int = 8855
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# MediaPipe FaceMesh inner-mouth landmark indices.
# 13 = upper inner mid, 14 = lower inner mid.
MEDIAPIPE_LIP_UPPER_INNER: int = 13
MEDIAPIPE_LIP_LOWER_INNER: int = 14
<<<<<<< HEAD
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# MediaPipe HAND fingertip indices (21-kp hand model).
MEDIAPIPE_HAND_FINGERTIPS: tuple[int, ...] = (4, 8, 12, 16, 20)
@@ -164,27 +128,11 @@ class ActionHeadPublisher(threading.Thread):
self._last_body_t = source_t
current_pids: set[int] = set()
if persons32:
<<<<<<< HEAD
<<<<<<< HEAD
for pid, j3d, expr_np, mouth, hands_kp42 in persons32:
current_pids.add(pid)
label, probs, kin = self.head.step(pid, j3d, expr=expr_np,
mouth_open=mouth,
hands_kp=hands_kp42)
=======
for pid, j3d, expr, mouth in persons32:
current_pids.add(pid)
label, probs, kin = self.head.step(pid, j3d,
expr=expr,
mouth_open=mouth)
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
for pid, j3d, expr_np, mouth, hands_kp42 in persons32:
current_pids.add(pid)
label, probs, kin = self.head.step(pid, j3d, expr=expr_np,
mouth_open=mouth,
hands_kp=hands_kp42)
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
idx = LABELS.index(label)
self.bridge.send_action(pid, idx, probs, t_now, force=True)
self.bridge.send_kin(pid, kin, t_now, force=True)
@@ -195,32 +143,13 @@ class ActionHeadPublisher(threading.Thread):
self.bridge.send_leave(pid=gone)
self._last_pids = current_pids
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
def _read_sources(self) -> tuple[
list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] | None,
float, str, bool,
]:
<<<<<<< HEAD
"""Return (persons32, source_t, source_tag, is_new).
Each person entry is (pid, j3d32, expr10, mouth_open, hands_kp42x3).
=======
def _read_sources(
self,
) -> tuple[list[tuple[int, np.ndarray, np.ndarray, float]] | None,
float, str, bool]:
"""Return (persons32, source_t, source_tag, is_new).
Each person entry is (pid, j3d32, expr10, mouth_open).
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
"""Return (persons32, source_t, source_tag, is_new).
Each person entry is (pid, j3d32, expr10, mouth_open, hands_kp42x3).
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
is_new is True when the timestamp advanced (even if person list
is empty), so _tick can still run the purge loop.
"""
@@ -234,10 +163,6 @@ class ActionHeadPublisher(threading.Thread):
persons_hands = getattr(self.state, "persons_hands", None)
ids_hands = getattr(self.state, "persons_hands_ids", None)
t_body = getattr(self.state, "pose_last_t", 0.0)
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# Build pid -> hands_kp(42, 3) map from MediaPipe persons_hands.
hands_by_pid: dict[int, np.ndarray] = self._build_hands_map(
@@ -249,20 +174,8 @@ class ActionHeadPublisher(threading.Thread):
)
# SMPL-X path (preferred)
<<<<<<< HEAD
if t_smplx > self._last_smplx_t:
out: list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] = []
=======
hands_ids = list(getattr(self.state, "persons_hands_ids", None) or [])
hands_lists = list(getattr(self.state, "persons_hands", None) or [])
# Prefer smplx when its timestamp advanced.
if t_smplx > self._last_smplx_t:
out: list[tuple[int, np.ndarray, np.ndarray, float]] = []
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
if t_smplx > self._last_smplx_t:
out: list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] = []
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
for i, p in enumerate(persons_smplx or []):
pid = int(p.get("pid", i))
v3d = p.get("v3d")
@@ -284,40 +197,19 @@ class ActionHeadPublisher(threading.Thread):
expr_np = np.asarray(expr, dtype=np.float32).flatten()
else:
expr_np = np.zeros(EXPR_DIM, dtype=np.float32)
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# mouth_open: prefer MediaPipe face lips, fallback SMPL-X v3d.
if pid in face_mouth_by_pid:
mouth = face_mouth_by_pid[pid]
elif v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT):
<<<<<<< HEAD
=======
# mouth_open
if v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT):
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
mouth = float(np.linalg.norm(
v3d_np[SMPLX_UPPER_LIP_VERT] - v3d_np[SMPLX_LOWER_LIP_VERT]
))
else:
mouth = 0.0
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
hands_kp42 = hands_by_pid.get(
pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32)
)
out.append((pid, j3d32, expr_np, mouth, hands_kp42))
<<<<<<< HEAD
=======
out.append((pid, j3d32, expr_np, mouth))
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
return out or None, t_smplx, "smplx", True
# MediaPipe body3d fallback
@@ -330,8 +222,6 @@ class ActionHeadPublisher(threading.Thread):
if arr is None or arr.shape[0] < 33:
continue
body22 = arr[list(MEDIAPIPE_TO_22)].astype(np.float32)
<<<<<<< HEAD
<<<<<<< HEAD
# fingertips from persons_hands if available
tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32)
hands_kp42 = hands_by_pid.get(
@@ -348,32 +238,6 @@ class ActionHeadPublisher(threading.Thread):
mouth = face_mouth_by_pid.get(pid, 0.0)
expr_np = np.zeros(EXPR_DIM, dtype=np.float32)
out.append((pid, j3d32, expr_np, mouth, hands_kp42))
=======
# fingertips from hands if available
=======
# fingertips from persons_hands if available
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32)
hands_kp42 = hands_by_pid.get(
pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32)
)
# extract fingertips from hands_kp42 (idx 4,8,12,16,20 each side)
for side_idx in (0, 1):
base = side_idx * HANDS_KP_PER_HAND
for k, mp_tip in enumerate(MEDIAPIPE_HAND_FINGERTIPS):
if base + mp_tip < hands_kp42.shape[0]:
tips[side_idx * J3D_FINGERS_PER_HAND + k] = \
hands_kp42[base + mp_tip]
j3d32 = np.concatenate([body22, tips], axis=0)
mouth = face_mouth_by_pid.get(pid, 0.0)
expr_np = np.zeros(EXPR_DIM, dtype=np.float32)
<<<<<<< HEAD
mouth = 0.0
out.append((pid, j3d32, expr_np, mouth))
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
out.append((pid, j3d32, expr_np, mouth, hands_kp42))
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
return out or None, t_body, "body3d", True
return None, 0.0, "", False
data_only_viz/apple_vision_pose.py
-12
View File
@@ -557,20 +557,8 @@ class AppleVisionPoseWorker:
# API d'acces simple. Face parsing depuis Apple Vision est
# actuellement bloque ; on garde MediaPipe (CPU XNNPACK) pour
# face/hand fin tandis que Vision sert body 2D sur ANE.
<<<<<<< HEAD
<<<<<<< HEAD
# Skip pour eviter le spam ObjCPointerWarning a 30 fps.
return
=======
n = 0; n_written = 0
if n_written > 0 and not hasattr(self, "_logged_face_write_" + region_name):
LOG.info("face: %s wrote %d points", region_name, n_written)
setattr(self, "_logged_face_write_" + region_name, True)
>>>>>>> 28d562b (feat(av-live): wire Apple Vision body pose on ANE)
=======
# Skip pour eviter le spam ObjCPointerWarning a 30 fps.
return
>>>>>>> 623c479 (perf(multi-hmr): autocast opt-in)
# faceContour
fill("faceContour", *FACE_OFFSETS["contour"])
data_only_viz/multi.py
+25 -2
View File
@@ -19,9 +19,10 @@ import time
import urllib.request
from pathlib import Path
from .action_head_pub import ActionHeadPublisher
from .euro_filter import SkeletonFilter
from .pose_bridge import PoseSoundBridge
from .state import PoseKp, State
from .state import Kp3D, PoseKp, State
from .tracker import IoUTracker
LOG = logging.getLogger("multi")
@@ -93,6 +94,8 @@ class MultiWorker:
self._smooth_hand = SkeletonFilter(min_cutoff=2.0, beta=0.10)
# Pont OSC pose -> sclang
self._sound_bridge = PoseSoundBridge(throttle_hz=30.0)
self._action_pub = ActionHeadPublisher(state=self.state, bridge=self._sound_bridge)
self._action_pub.start()
def start(self) -> None:
self._thread = threading.Thread(
@@ -198,6 +201,21 @@ class MultiWorker:
x=float(lm.x), y=float(lm.y), z=z, c=float(v)))
bodies.append(kp_list)
# pose_world_landmarks : xyz metric, relative to hip-center.
# Aligned 1:1 with pose_landmarks order. Empty fallback if
# the MediaPipe build doesn't populate it.
bodies3d: list[list[Kp3D]] = []
world_list = getattr(pose_res, "pose_world_landmarks", None) or []
for landmarks_list in world_list:
kp3_list: list[Kp3D] = []
for lm in landmarks_list[:33]:
v = lm.visibility if lm.visibility is not None else 1.0
kp3_list.append(Kp3D(
x=float(lm.x), y=float(lm.y),
z=float(lm.z if lm.z is not None else 0.0),
c=float(v)))
bodies3d.append(kp3_list)
faces = []
for landmarks_list in (face_res.face_landmarks or []):
kp_list = []
@@ -230,16 +248,21 @@ class MultiWorker:
for i, kps in enumerate(hands)]
# Pont sonore : envoi OSC /pose/* a sclang (body + face + hands)
# 3D world landmarks share ids with bodies (same MediaPipe
# detection, just a different coordinate space).
ids_body3d = ids_body[:len(bodies3d)] if bodies3d else []
self._sound_bridge.send(
bodies, ids_body, t_now,
persons_face=faces, persons_face_ids=ids_face,
persons_hands=hands, persons_hands_ids=ids_hand)
persons_hands=hands, persons_hands_ids=ids_hand,
persons_body3d=bodies3d, persons_body3d_ids=ids_body3d)
with self.state.lock():
self.state.persons_body = bodies
self.state.persons_face = faces
self.state.persons_hands = hands
self.state.persons_body_ids = ids_body
self.state.persons_body3d = bodies3d
self.state.persons_face_ids = ids_face
self.state.persons_hands_ids = ids_hand
# Compat single-person (1ere personne)
data_only_viz/pose_bridge.py
+221 -18
View File
@@ -10,13 +10,21 @@ Routes emises :
/pose/head <pid> <x> <y> <c> position du nez (visage)
/pose/sho_span <pid> <dx> ecart epaules (estime distance camera)
/pose/limb_span <pid> <span> envergure brassse (poignet a poignet)
/face/count <n> nombre de visages detectes
/face/kp <pid> <idx> <x> <y> <z> <c> 68-pt subset (dlib mapping)
/hand/count <n_left> <n_right> nombre de mains gauche / droite
/hand/kp <pid> <side[0=L|1=R]> <idx> <x> <y> <z> <c> 21 landmarks
/pose3d/count <n> nombre de squelettes 3D
/pose3d/kp <pid> <idx> <x> <y> <z> <c> 33 MediaPipe world landmarks (metres)
Mapping pose -> son est defini cote SC dans sound_algo/data_only/scenes.scd
(scene `live_pose`).
(scene `live_pose`). Face / hand keypoints are consumed by the Swift
launcher (AVLiveBody) on 127.0.0.1:57126 for skeleton overlay rendering.
"""
from __future__ import annotations
import logging
from typing import Any, Iterable, Sequence
from pythonosc.udp_client import SimpleUDPClient
@@ -33,6 +41,42 @@ LEFT_HIP = 23
RIGHT_HIP = 24
# MediaPipe FaceMesh (468 landmarks) -> 68-point dlib-style subset.
# Mapping inspired by community references (Google MediaPipe ->
# iBUG 68 facial landmarks). Order matches dlib 68-point convention :
# [0..16] jaw contour (left to right)
# [17..21] right brow
# [22..26] left brow
# [27..30] nose bridge (top to tip)
# [31..35] nostril base (right to left)
# [36..41] right eye (CCW from outer corner)
# [42..47] left eye (CCW from inner corner)
# [48..59] outer lip (CCW from right corner)
# [60..67] inner lip (CCW from right corner)
FACE_68_FROM_MP: tuple[int, ...] = (
# Jaw (17)
127, 234, 132, 172, 150, 176, 148, 152,
377, 400, 365, 397, 361, 401, 366, 447, 356,
# Right brow (5) — mediapipe perspective is mirrored vs subject
70, 63, 105, 66, 107,
# Left brow (5)
336, 296, 334, 293, 300,
# Nose bridge (4)
168, 6, 197, 195,
# Nostril base (5)
98, 97, 2, 326, 327,
# Right eye (6)
33, 160, 158, 133, 153, 144,
# Left eye (6)
362, 385, 387, 263, 373, 380,
# Outer lip (12)
61, 39, 37, 0, 267, 269, 291, 405, 314, 17, 84, 181,
# Inner lip (8)
78, 81, 13, 311, 308, 402, 14, 178,
)
assert len(FACE_68_FROM_MP) == 68
class PoseSoundBridge:
"""Envoie les keypoints en OSC vers sclang. Throttle a 30 Hz max."""
@@ -45,9 +89,17 @@ class PoseSoundBridge:
self._period = 1.0 / max(1.0, throttle_hz)
self._last_t = 0.0
def send(self, persons_body: list, persons_body_ids: list, t_now: float) -> None:
def send(self, persons_body: list, persons_body_ids: list, t_now: float,
*,
persons_face: Sequence[Sequence[Any]] | None = None,
persons_face_ids: Sequence[int] | None = None,
persons_hands: Sequence[Sequence[Any]] | None = None,
persons_hands_ids: Sequence[int] | None = None,
persons_body3d: Sequence[Sequence[Any]] | None = None,
persons_body3d_ids: Sequence[int] | None = None) -> None:
"""Envoie les keypoints de toutes les personnes detectees.
Throttle automatiquement."""
Throttle automatiquement. Face / hand sont optionnels et envoyes
sur le meme socket :57126 vers AVLiveBody."""
if t_now - self._last_t < self._period:
return
self._last_t = t_now
@@ -59,22 +111,20 @@ class PoseSoundBridge:
except OSError: pass
except OSError:
return # SC pas la, on continue silencieusement
if n == 0:
return
for i, body in enumerate(persons_body):
pid = persons_body_ids[i] if i < len(persons_body_ids) else i
self._emit_person(int(pid), body)
# Skeleton complet pour AV-Live-Body overlay : 33 kp body.
# On envoie en flat list pour minimiser le surcout OSC.
flat = []
for kp in body:
flat.extend([float(kp.x), float(kp.y), float(kp.c)])
try:
self._avbody.send_message(
"/pose/skel", [int(pid)] + flat)
except OSError:
pass
if n > 0:
for i, body in enumerate(persons_body):
pid = persons_body_ids[i] if i < len(persons_body_ids) else i
self._emit_person(int(pid), body)
# Face / hand : independant de la presence de body kp (utile en
# mode face-only ou hand-only).
if persons_face is not None:
self._send_face(persons_face, persons_face_ids or [])
if persons_hands is not None:
self._send_hand(persons_hands, persons_hands_ids or [])
if persons_body3d is not None:
self._send_body3d(persons_body3d, persons_body3d_ids or [])
# ------------------------------------------------------------------
def _emit_person(self, pid: int, body: list) -> None:
@@ -123,3 +173,156 @@ class PoseSoundBridge:
cli.send_message("/pose/limb_span", [pid, float(span)])
try: self._avbody.send_message("/pose/limb_span", [pid, float(span)])
except OSError: pass
# ------------------------------------------------------------------
def send_face(self, persons_face: Sequence[Sequence[Any]],
persons_face_ids: Sequence[int], t_now: float,
force: bool = False) -> None:
"""Public throttled entry point for face keypoints.
Emits a 68-point dlib-style subset of the 468 MediaPipe FaceMesh
landmarks per person on /face/count + /face/kp routes.
"""
if not force and (t_now - self._last_t) < self._period:
return
self._send_face(persons_face, persons_face_ids)
def send_hand(self, persons_hands: Sequence[Sequence[Any]],
persons_hands_ids: Sequence[int], t_now: float,
force: bool = False) -> None:
"""Public throttled entry point for hand keypoints.
Emits the full 21-landmark hand skeleton per detected hand on
/hand/count + /hand/kp routes. Side is inferred from id parity
(MediaPipe Hand task does not flag left/right reliably) : we
treat odd ids as right, even as left, which matches the
convention used by the smoother / tracker upstream.
"""
if not force and (t_now - self._last_t) < self._period:
return
self._send_hand(persons_hands, persons_hands_ids)
def _send_face(self, persons_face: Sequence[Sequence[Any]],
persons_face_ids: Sequence[int]) -> None:
n = len(persons_face)
try:
self._avbody.send_message("/face/count", [int(n)])
except OSError:
return
for i, face in enumerate(persons_face):
if not face:
continue
pid = persons_face_ids[i] if i < len(persons_face_ids) else i
n_lm = len(face)
for slot, mp_idx in enumerate(FACE_68_FROM_MP):
if mp_idx >= n_lm:
continue
kp = face[mp_idx]
try:
self._avbody.send_message("/face/kp", [
int(pid), int(slot),
float(kp.x), float(kp.y),
float(getattr(kp, "z", 0.0)),
float(getattr(kp, "c", 1.0)),
])
except OSError:
return
def _send_hand(self, persons_hands: Sequence[Sequence[Any]],
persons_hands_ids: Sequence[int]) -> None:
n_left = 0
n_right = 0
for i in range(len(persons_hands)):
pid = persons_hands_ids[i] if i < len(persons_hands_ids) else i
if int(pid) % 2 == 0:
n_left += 1
else:
n_right += 1
try:
self._avbody.send_message("/hand/count", [int(n_left), int(n_right)])
except OSError:
return
for i, hand in enumerate(persons_hands):
if not hand:
continue
pid = persons_hands_ids[i] if i < len(persons_hands_ids) else i
side = 1 if int(pid) % 2 else 0
for idx, kp in enumerate(hand[:21]):
try:
self._avbody.send_message("/hand/kp", [
int(pid), int(side), int(idx),
float(kp.x), float(kp.y),
float(getattr(kp, "z", 0.0)),
float(getattr(kp, "c", 1.0)),
])
except OSError:
return
def send_body3d(self, persons_body3d: Sequence[Sequence[Any]],
persons_body3d_ids: Sequence[int], t_now: float,
force: bool = False) -> None:
"""Public throttled entry point for 3D body world landmarks.
Emits 33 MediaPipe pose_world_landmarks per person on
/pose3d/count + /pose3d/kp routes. Coordinates are in meters,
relative to the hip-center (MediaPipe convention: x=right,
y=down, z=forward from the camera).
"""
if not force and (t_now - self._last_t) < self._period:
return
self._send_body3d(persons_body3d, persons_body3d_ids)
def _send_body3d(self, persons_body3d: Sequence[Sequence[Any]],
persons_body3d_ids: Sequence[int]) -> None:
n = len(persons_body3d)
try:
self._avbody.send_message("/pose3d/count", [int(n)])
except OSError:
return
for i, body in enumerate(persons_body3d):
if not body:
continue
pid = persons_body3d_ids[i] if i < len(persons_body3d_ids) else i
for idx, kp in enumerate(body[:33]):
try:
self._avbody.send_message("/pose3d/kp", [
int(pid), int(idx),
float(kp.x), float(kp.y),
float(getattr(kp, "z", 0.0)),
float(getattr(kp, "c", 1.0)),
])
except OSError:
return
# ------------------------------------------------------------------
def send_action(self, pid: int, label_idx: int,
probs, t_now: float, force: bool = False) -> None:
"""Send action classification result via /pose/action OSC route.
Sends: [pid (int), label_idx (int), prob_0 (float), prob_1 (float), prob_2 (float)]
"""
if not force and (t_now - self._last_t) < self._period:
return
p = [float(probs[0]), float(probs[1]), float(probs[2])]
self._client.send_message("/pose/action", [int(pid), int(label_idx), *p])
def send_kin(self, pid: int, kin,
t_now: float, force: bool = False) -> None:
"""Send kinematic angles via /pose/kin OSC route.
Sends: [pid (int), kin_0 (float), kin_1 (float), kin_2 (float)]
"""
if not force and (t_now - self._last_t) < self._period:
return
self._client.send_message(
"/pose/kin",
[int(pid), float(kin[0]), float(kin[1]), float(kin[2])],
)
def send_enter(self, pid: int) -> None:
"""Send lifecycle event when person enters frame."""
self._client.send_message("/pose/enter", [int(pid)])
def send_leave(self, pid: int) -> None:
"""Send lifecycle event when person leaves frame."""
self._client.send_message("/pose/leave", [int(pid)])
data_only_viz/scripts/convert_dinov2.py
+167
View File
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""Convert DINOv2 ViT-S/14 to a CoreML .mlpackage for ANE-friendly inference.
The wrapped module takes (1, 3, 224, 224) RGB float32 in [0, 1], applies
ImageNet normalization internally, runs the ViT, and returns the CLS
embedding (1, 384) L2-normalised. We trace + convert with
``coremltools.convert(... compute_units=ComputeUnit.ALL, compute_precision=FP16)``.
Run with the Python 3.12 venv that has coremltools and torch::
/tmp/coreml312/bin/python -m data_only_viz.scripts.convert_dinov2 [--force]
Output:
~/.cache/av-live-multihmr/dinov2_vits14.mlpackage
"""
from __future__ import annotations
import argparse
import logging
import sys
import time
from pathlib import Path
import numpy as np
LOG = logging.getLogger("convert_dinov2")
OUT_DIR = Path.home() / ".cache" / "av-live-multihmr"
OUT_PATH = OUT_DIR / "dinov2_vits14.mlpackage"
# ImageNet stats (DINOv2 expects these).
_IMAGENET_MEAN = (0.485, 0.456, 0.406)
_IMAGENET_STD = (0.229, 0.224, 0.225)
def _build_wrapper():
import torch
import torch.nn as nn
# Load DINOv2 ViT-S/14 from local torch.hub cache (offline ok).
backbone = torch.hub.load(
"facebookresearch/dinov2",
"dinov2_vits14",
source="github",
trust_repo=True,
)
backbone.eval()
# Monkey-patch interpolate_pos_encoding so we don't hit the bicubic
# upsample op (unsupported by coremltools). At fixed 224x224 input
# with patch=14 the layout is exactly 16x16 = 256 patches, which
# matches the pretrained pos_embed grid -> no interpolation needed.
def _no_interp_pos_encoding(self, x, w, h):
# Just return the stored pos_embed (cast to current dtype).
return self.pos_embed.to(x.dtype)
import types
backbone.interpolate_pos_encoding = types.MethodType(
_no_interp_pos_encoding, backbone)
mean = torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1)
std = torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1)
class DinoV2Wrapper(nn.Module):
def __init__(self):
super().__init__()
self.backbone = backbone
self.register_buffer("mean", mean)
self.register_buffer("std", std)
def forward(self, x: "torch.Tensor") -> "torch.Tensor":
# x: (1, 3, 224, 224) in [0, 1]. Normalise then forward.
x = (x - self.mean) / self.std
cls = self.backbone(x) # (1, 384) - CLS token by default
# L2 normalise so cosine sim = dot product.
cls = cls / (cls.norm(dim=-1, keepdim=True) + 1e-8)
return cls
wrap = DinoV2Wrapper().eval()
return wrap
def convert(force: bool = False) -> Path:
import torch
import coremltools as ct
OUT_DIR.mkdir(parents=True, exist_ok=True)
if OUT_PATH.exists() and not force:
LOG.info("already converted: %s", OUT_PATH)
return OUT_PATH
LOG.info("loading DINOv2 ViT-S/14 from torch.hub ...")
wrap = _build_wrapper()
example = torch.rand(1, 3, 224, 224, dtype=torch.float32)
with torch.no_grad():
ref_out = wrap(example)
LOG.info("torch out shape=%s norm=%.4f", tuple(ref_out.shape),
float(ref_out.norm(dim=-1).mean()))
LOG.info("tracing ...")
with torch.no_grad():
traced = torch.jit.trace(wrap, example, strict=False)
LOG.info("ct.convert (mlprogram FP16, computeUnits=ALL) ...")
mlmodel = ct.convert(
traced,
source="pytorch",
convert_to="mlprogram",
inputs=[ct.TensorType(name="image", shape=example.shape,
dtype=np.float32)],
outputs=[ct.TensorType(name="embedding", dtype=np.float32)],
compute_precision=ct.precision.FLOAT16,
compute_units=ct.ComputeUnit.ALL,
minimum_deployment_target=ct.target.macOS14,
)
mlmodel.short_description = "DINOv2 ViT-S/14 person re-id (384-D, L2)"
mlmodel.save(str(OUT_PATH))
LOG.info("saved %s", OUT_PATH)
# Sanity numerical check
pred = mlmodel.predict({"image": example.numpy().astype(np.float32)})
coreml_out = list(pred.values())[0].reshape(-1)
ref_np = ref_out.numpy().reshape(-1)
cos = float(np.dot(coreml_out, ref_np) /
(np.linalg.norm(coreml_out) * np.linalg.norm(ref_np) + 1e-8))
LOG.info("CoreML vs Torch cosine on random input: %.4f", cos)
return OUT_PATH
def bench(n_iter: int = 30) -> None:
import coremltools as ct
LOG.info("bench: load mlpackage ...")
m = ct.models.MLModel(str(OUT_PATH),
compute_units=ct.ComputeUnit.ALL)
crop = np.random.rand(1, 3, 224, 224).astype(np.float32)
# warmup
for _ in range(3):
m.predict({"image": crop})
times = []
for _ in range(n_iter):
t0 = time.perf_counter()
m.predict({"image": crop})
times.append((time.perf_counter() - t0) * 1e3)
times.sort()
p50 = times[len(times) // 2]
p95 = times[int(len(times) * 0.95)]
LOG.info("bench %d iter: p50=%.2f ms p95=%.2f ms mean=%.2f ms (~%.1f fps)",
n_iter, p50, p95, sum(times) / len(times), 1000.0 / p50)
def main() -> int:
logging.basicConfig(level=logging.INFO,
format="%(asctime)s %(name)s %(message)s")
ap = argparse.ArgumentParser()
ap.add_argument("--force", action="store_true")
ap.add_argument("--bench-only", action="store_true")
ap.add_argument("--n-iter", type=int, default=30)
args = ap.parse_args()
if not args.bench_only:
convert(force=args.force)
bench(n_iter=args.n_iter)
return 0
if __name__ == "__main__":
sys.exit(main())
data_only_viz/scripts/extract_j3d_offline.py
-69
View File
@@ -1,12 +1,4 @@
<<<<<<< HEAD
<<<<<<< HEAD
"""Extract j3d (32 SMPL-X joint anchors) from a recorded MP4 using the
=======
"""Extract j3d (22 SMPL-X joint anchors) from a recorded MP4 using the
>>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes)
=======
"""Extract j3d (32 SMPL-X joint anchors) from a recorded MP4 using the
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
Multi-HMR CoreML backend, write per-frame per-person jsonl rows.
Usage:
@@ -25,27 +17,12 @@ from pathlib import Path
import cv2
import numpy as np
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
from data_only_viz.action_head import EXPR_DIM, HANDS_KP_DIMS, HANDS_KP_TOTAL
=======
from data_only_viz.action_head import EXPR_DIM
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
from data_only_viz.action_head import EXPR_DIM, HANDS_KP_DIMS, HANDS_KP_TOTAL
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
from data_only_viz.action_head_pub import (
SMPLX_JOINT_ANCHOR_VERTS,
SMPLX_UPPER_LIP_VERT,
SMPLX_LOWER_LIP_VERT,
)
<<<<<<< HEAD
=======
from data_only_viz.action_head_pub import SMPLX_JOINT_ANCHOR_VERTS
>>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes)
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend
LOG = logging.getLogger("extract_j3d_offline")
@@ -75,21 +52,11 @@ def _frame_to_chw(frame_bgr: np.ndarray, size: int = IMG_SIZE) -> np.ndarray:
return rgb.transpose(2, 0, 1) # CHW
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
def _person_to_j3d32(
person: dict,
anchors: tuple[int, ...],
) -> tuple[np.ndarray, np.ndarray, float] | None:
"""Return (j3d32, expression, mouth_open) or None if v3d absent/too small."""
<<<<<<< HEAD
=======
def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | None:
>>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes)
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
v3d = person.get("v3d")
if v3d is None:
return None
@@ -99,10 +66,6 @@ def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | Non
v3d_np = np.asarray(v3d, dtype=np.float32)
if v3d_np.shape[0] < max(anchors) + 1:
return None
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
j3d32 = v3d_np[list(anchors)].astype(np.float32)
# expression
expr = person.get("expression")
@@ -120,12 +83,6 @@ def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | Non
else:
mouth = 0.0
return j3d32, expr_np, mouth
<<<<<<< HEAD
=======
return v3d_np[list(anchors)].astype(np.float32)
>>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes)
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
def extract(session: str, video: Path, out: Path,
@@ -157,46 +114,20 @@ def extract(session: str, video: Path, out: Path,
continue
ts = n_frames / fps
for i, person in enumerate(persons):
<<<<<<< HEAD
<<<<<<< HEAD
result = _person_to_j3d32(person, anchors)
if result is None:
continue
j3d32, expr_np, mouth = result
=======
j3d = _person_to_j3d22(person, anchors)
if j3d is None:
continue
>>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes)
=======
result = _person_to_j3d32(person, anchors)
if result is None:
continue
j3d32, expr_np, mouth = result
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
f.write(json.dumps({
"ts": ts,
"session": session,
"pid": int(person.get("pid", i)),
<<<<<<< HEAD
<<<<<<< HEAD
"j3d": j3d32.tolist(),
"expression": expr_np.tolist(),
"mouth_open": mouth,
"hands_kp": np.zeros(
(HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32
).tolist(),
<<<<<<< HEAD
=======
"j3d": j3d.tolist(),
>>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes)
=======
"j3d": j3d32.tolist(),
"expression": expr_np.tolist(),
"mouth_open": mouth,
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
}) + "\n")
n_rows += 1
n_frames += 1
data_only_viz/state.py
+14
View File
@@ -21,6 +21,16 @@ class PoseKp:
c: float = 0.0
@dataclass
class Kp3D:
"""3D keypoint in metric coordinates relative to hip-center.
Used for MediaPipe pose_world_landmarks (xyz in meters)."""
x: float = 0.0
y: float = 0.0
z: float = 0.0
c: float = 0.0
@dataclass
class SMPLXPerson:
"""Resultats Multi-HMR pour une personne : params SMPL-X + vertices
@@ -92,6 +102,10 @@ class State:
persons_body: list[list[PoseKp]] = field(default_factory=list)
persons_face: list[list[PoseKp]] = field(default_factory=list)
persons_hands: list[list[PoseKp]] = field(default_factory=list)
# MediaPipe pose_world_landmarks per person : 33 keypoints in meters,
# relative to the hip-center. Optional companion of persons_body
# (image-space xy). Empty if no detection or backend doesn't emit it.
persons_body3d: list[list[Kp3D]] = field(default_factory=list)
# IDs persistants entre frames (ByteTrack-like via Hungarian IoU).
# Couleur du skeleton dans le shader Metal = ID % palette_size.
persons_body_ids: list[int] = field(default_factory=list)
data_only_viz/tests/test_action_head_features.py
-18
View File
@@ -12,19 +12,9 @@ def test_module_imports() -> None:
assert hasattr(action_head, "ActionHead")
assert action_head.WINDOW_LEN == 16
assert action_head.J3D_JOINTS == 32
<<<<<<< HEAD
<<<<<<< HEAD
assert action_head.FEATURE_DIM == 428
assert action_head.HANDS_KP_TOTAL == 42
assert action_head.HANDS_KP_FLAT == 126
=======
assert action_head.FEATURE_DIM == 302
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
assert action_head.FEATURE_DIM == 428
assert action_head.HANDS_KP_TOTAL == 42
assert action_head.HANDS_KP_FLAT == 126
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
assert action_head.NUM_CLASSES == 3
assert action_head.LABELS == ("debout", "assise", "danse")
@@ -79,15 +69,7 @@ def test_feature_extractor_shape_full_buffer() -> None:
from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, FEATURE_DIM
frames = [_rand_j3d(i) for i in range(WINDOW_LEN)]
feat = FeatureExtractor.from_buffer(frames)
<<<<<<< HEAD
<<<<<<< HEAD
assert feat.shape == (428,)
=======
assert feat.shape == (302,)
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
assert feat.shape == (428,)
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
assert feat.dtype == np.float32
assert not np.isnan(feat).any()
data_only_viz/tests/test_action_head_model.py
-16
View File
@@ -24,27 +24,11 @@ def test_model_forward_shape() -> None:
assert h_new.shape == h.shape
<<<<<<< HEAD
<<<<<<< HEAD
def test_model_param_count_under_100k() -> None:
from data_only_viz.action_head import ActionHeadModel
model = ActionHeadModel()
n = sum(p.numel() for p in model.parameters())
assert n < 100_000, f"too many params: {n}"
=======
def test_model_param_count_under_80k() -> None:
from data_only_viz.action_head import ActionHeadModel
model = ActionHeadModel()
n = sum(p.numel() for p in model.parameters())
assert n < 80_000, f"too many params: {n}"
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
def test_model_param_count_under_100k() -> None:
from data_only_viz.action_head import ActionHeadModel
model = ActionHeadModel()
n = sum(p.numel() for p in model.parameters())
assert n < 100_000, f"too many params: {n}"
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
def test_action_head_step_warmup_returns_debout() -> None:
data_only_viz/tests/test_action_head_pub.py
-8
View File
@@ -19,16 +19,8 @@ class _FakeState:
self.pose_last_t = 0.0
self.persons_hands = []
self.persons_hands_ids = []
<<<<<<< HEAD
<<<<<<< HEAD
self.persons_face = []
self.persons_face_ids = []
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
self.persons_face = []
self.persons_face_ids = []
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
self._lock = threading.RLock()
def lock(self):
data_only_viz/tests/test_multihmr_coreml.py
+96
View File
@@ -0,0 +1,96 @@
"""Tests for the Multi-HMR CoreML backend.
Skipped unless the .mlpackage exists at the standard cache path.
"""
from __future__ import annotations
import time
from pathlib import Path
import numpy as np
import pytest
MLPACKAGE = (
Path.home() / ".cache" / "av-live-multihmr"
/ "multihmr_full_672_s.mlpackage"
)
pytestmark = pytest.mark.skipif(
not MLPACKAGE.exists(),
reason=f"mlpackage missing at {MLPACKAGE}",
)
def _make_K() -> np.ndarray:
f = 672.0
return np.array([[f, 0.0, 336.0],
[0.0, f, 336.0],
[0.0, 0.0, 1.0]], dtype=np.float32)
def test_is_available_true():
from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend
assert MultiHMRCoreMLBackend.is_available(MLPACKAGE) is True
def test_load_model():
from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend
backend = MultiHMRCoreMLBackend(MLPACKAGE)
assert backend._model is not None
def test_infer_random_image_shapes():
from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend
backend = MultiHMRCoreMLBackend(MLPACKAGE)
rng = np.random.default_rng(0)
img = rng.random((3, 672, 672), dtype=np.float32)
K = _make_K()
# threshold = -inf so we get all K=4 humans back
humans = backend.infer(img, K, det_thresh=-1.0)
assert len(humans) == 4
for h in humans:
v = h["v3d"].detach().cpu().numpy()
assert v.shape == (10475, 3)
assert v.dtype == np.float32
t = h["transl_pelvis"].detach().cpu().numpy()
assert t.shape == (1, 3)
s = float(h["scores"].item())
assert isinstance(s, float)
beta = h["shape"].detach().cpu().numpy()
assert beta.shape == (10,)
expr = h["expression"].detach().cpu().numpy()
assert expr.shape == (10,)
def test_infer_latency_under_target():
from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend
backend = MultiHMRCoreMLBackend(MLPACKAGE)
K = _make_K()
rng = np.random.default_rng(42)
img = rng.random((3, 672, 672), dtype=np.float32)
# warmup
backend.infer(img, K, det_thresh=-1.0)
# measure
n = 5
times = []
for _ in range(n):
t0 = time.monotonic()
backend.infer(img, K, det_thresh=-1.0)
times.append((time.monotonic() - t0) * 1e3)
times.sort()
median_ms = times[n // 2]
print(f"median latency: {median_ms:.1f} ms (n={n})")
# Target 50ms = 20fps. M5 bench shows ~29ms. Generous margin.
assert median_ms < 80.0, f"median {median_ms:.1f}ms > 80ms target"
def test_filter_threshold():
from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend
backend = MultiHMRCoreMLBackend(MLPACKAGE)
rng = np.random.default_rng(0)
img = rng.random((3, 672, 672), dtype=np.float32)
K = _make_K()
high = backend.infer(img, K, det_thresh=999.0)
assert high == [] # nothing passes
low = backend.infer(img, K, det_thresh=-1.0)
assert len(low) == 4
data_only_viz/tests/test_pose_bridge_action.py
+49
View File
@@ -0,0 +1,49 @@
"""Tests for /pose/action and /pose/kin OSC routes."""
from __future__ import annotations
from unittest.mock import MagicMock
import numpy as np
def test_send_action_formats_5_args() -> None:
from data_only_viz.pose_bridge import PoseSoundBridge
b = PoseSoundBridge()
b._client = MagicMock()
b.send_action(pid=7, label_idx=2,
probs=np.array([0.1, 0.2, 0.7], dtype=np.float32),
t_now=0.0, force=True)
b._client.send_message.assert_called_once()
address, args = b._client.send_message.call_args.args
assert address == "/pose/action"
assert args[0] == 7
assert args[1] == 2
assert all(isinstance(v, float) for v in args[2:5])
assert abs(sum(args[2:5]) - 1.0) < 1e-5
def test_send_kin_formats_4_args() -> None:
from data_only_viz.pose_bridge import PoseSoundBridge
b = PoseSoundBridge()
b._client = MagicMock()
b.send_kin(pid=3, kin=np.array([0.5, 1.2, -0.3], dtype=np.float32),
t_now=0.0, force=True)
b._client.send_message.assert_called_once()
address, args = b._client.send_message.call_args.args
assert address == "/pose/kin"
assert args[0] == 3
assert len(args) == 4
assert abs(args[1] - 0.5) < 1e-6
assert abs(args[2] - 1.2) < 1e-6
assert abs(args[3] - (-0.3)) < 1e-6
def test_send_lifecycle_enter_leave() -> None:
from data_only_viz.pose_bridge import PoseSoundBridge
b = PoseSoundBridge()
b._client = MagicMock()
b.send_enter(pid=4)
b.send_leave(pid=4)
calls = [c.args[0] for c in b._client.send_message.call_args_list]
assert "/pose/enter" in calls
assert "/pose/leave" in calls
data_only_viz/tests/test_pose_bridge_body3d.py
+79
View File
@@ -0,0 +1,79 @@
"""Tests for /pose3d/* OSC routes emitted to AVLiveBody."""
from __future__ import annotations
from dataclasses import dataclass
from unittest.mock import MagicMock
@dataclass
class _Kp3D:
x: float = 0.0
y: float = 0.0
z: float = 0.0
c: float = 1.0
def _make_bridge():
from data_only_viz.pose_bridge import PoseSoundBridge
b = PoseSoundBridge()
b._client = MagicMock()
b._avbody = MagicMock()
return b
def test_send_body3d_emits_count_and_33_kp() -> None:
b = _make_bridge()
# One person, 33 keypoints with deterministic xyz.
body = [_Kp3D(x=0.01 * i, y=-0.02 * i, z=0.03 * i, c=1.0)
for i in range(33)]
b.send_body3d([body], [7], t_now=0.0, force=True)
calls = b._avbody.send_message.call_args_list
assert calls[0].args == ("/pose3d/count", [1])
kp_calls = [c for c in calls if c.args[0] == "/pose3d/kp"]
assert len(kp_calls) == 33
# Format : [pid, idx, x, y, z, c]
first = kp_calls[0].args[1]
assert first[0] == 7
assert first[1] == 0
assert abs(first[2] - 0.0) < 1e-6
assert abs(first[4] - 0.0) < 1e-6
# idx ordering strictly 0..32
idxs = [c.args[1][1] for c in kp_calls]
assert idxs == list(range(33))
# Last kp z should be 0.03 * 32
last = kp_calls[-1].args[1]
assert abs(last[4] - 0.03 * 32) < 1e-6
def test_send_body3d_empty_emits_count_zero() -> None:
b = _make_bridge()
b.send_body3d([], [], t_now=0.0, force=True)
calls = b._avbody.send_message.call_args_list
assert len(calls) == 1
assert calls[0].args == ("/pose3d/count", [0])
def test_send_body3d_multi_person() -> None:
b = _make_bridge()
body_a = [_Kp3D(x=1.0) for _ in range(33)]
body_b = [_Kp3D(x=2.0) for _ in range(33)]
b.send_body3d([body_a, body_b], [10, 11], t_now=0.0, force=True)
calls = b._avbody.send_message.call_args_list
assert calls[0].args == ("/pose3d/count", [2])
kp_calls = [c for c in calls if c.args[0] == "/pose3d/kp"]
assert len(kp_calls) == 66
assert kp_calls[0].args[1][0] == 10
assert kp_calls[33].args[1][0] == 11
def test_send_with_body3d_kwargs_dispatches() -> None:
"""Top-level send() routes body3d kwargs to /pose3d."""
b = _make_bridge()
body = [_Kp3D(x=0.5, y=0.5, c=1.0) for _ in range(33)]
body3d = [_Kp3D(x=0.1, y=0.2, z=0.3) for _ in range(33)]
b.send([body], [0], 1.0,
persons_body3d=[body3d], persons_body3d_ids=[0])
av_addrs = {c.args[0] for c in b._avbody.send_message.call_args_list}
assert "/pose3d/count" in av_addrs
assert "/pose3d/kp" in av_addrs
data_only_viz/tests/test_pose_bridge_face_hand.py
+105
View File
@@ -0,0 +1,105 @@
"""Tests for /face/* and /hand/* OSC routes emitted to AVLiveBody."""
from __future__ import annotations
from dataclasses import dataclass
from unittest.mock import MagicMock
@dataclass
class _Kp:
x: float = 0.0
y: float = 0.0
z: float = 0.0
c: float = 1.0
def _make_bridge():
from data_only_viz.pose_bridge import PoseSoundBridge
b = PoseSoundBridge()
b._client = MagicMock()
b._avbody = MagicMock()
return b
def test_send_face_emits_count_and_68_kp() -> None:
from data_only_viz.pose_bridge import FACE_68_FROM_MP
b = _make_bridge()
# One face with 478 landmarks at deterministic coords.
face = [_Kp(x=i / 478.0, y=1.0 - i / 478.0, z=0.01 * i, c=1.0)
for i in range(478)]
b.send_face([face], [3], t_now=0.0, force=True)
calls = b._avbody.send_message.call_args_list
# First call : /face/count <1>
assert calls[0].args[0] == "/face/count"
assert calls[0].args[1] == [1]
# Then 68 /face/kp messages
kp_calls = [c for c in calls if c.args[0] == "/face/kp"]
assert len(kp_calls) == 68
# Format : [pid, slot, x, y, z, c]
first = kp_calls[0].args[1]
assert first[0] == 3
assert first[1] == 0
assert isinstance(first[2], float)
assert isinstance(first[3], float)
assert isinstance(first[4], float)
assert isinstance(first[5], float)
# Slot ordering is strictly increasing 0..67
slots = [c.args[1][1] for c in kp_calls]
assert slots == list(range(68))
# And the x coord of slot 0 matches mp_idx FACE_68_FROM_MP[0]
mp0 = FACE_68_FROM_MP[0]
assert abs(first[2] - mp0 / 478.0) < 1e-6
def test_send_face_empty_emits_count_zero() -> None:
b = _make_bridge()
b.send_face([], [], t_now=0.0, force=True)
calls = b._avbody.send_message.call_args_list
assert len(calls) == 1
assert calls[0].args == ("/face/count", [0])
def test_send_hand_emits_count_and_21_kp() -> None:
b = _make_bridge()
hand_l = [_Kp(x=0.1, y=0.2, z=0.0, c=1.0) for _ in range(21)]
hand_r = [_Kp(x=0.7, y=0.3, z=0.0, c=1.0) for _ in range(21)]
# pid=2 -> left (even), pid=3 -> right (odd)
b.send_hand([hand_l, hand_r], [2, 3], t_now=0.0, force=True)
calls = b._avbody.send_message.call_args_list
assert calls[0].args == ("/hand/count", [1, 1])
kp_calls = [c for c in calls if c.args[0] == "/hand/kp"]
assert len(kp_calls) == 42 # 21 * 2
# First hand should be side=0 (left)
assert kp_calls[0].args[1][1] == 0
# 22nd kp call : start of right hand, side=1
assert kp_calls[21].args[1][1] == 1
def test_send_throttles_below_period() -> None:
"""send_face called twice within < period emits only once."""
b = _make_bridge()
face = [_Kp(x=0.0, y=0.0) for _ in range(478)]
b.send_face([face], [0], t_now=0.0, force=True)
n_first = b._avbody.send_message.call_count
# Second call without force AND inside throttle window : skipped.
b._last_t = 9999.0 # pretend a body send just happened
b.send_face([face], [0], t_now=9999.0 + 0.001, force=False)
assert b._avbody.send_message.call_count == n_first
def test_send_with_face_hand_kwargs_dispatches() -> None:
"""Top-level send() routes face/hand kwargs to /face and /hand."""
b = _make_bridge()
body = [_Kp(x=0.5, y=0.5, c=1.0) for _ in range(33)]
face = [_Kp(x=0.1, y=0.1, c=1.0) for _ in range(478)]
hand = [_Kp(x=0.2, y=0.2, c=1.0) for _ in range(21)]
b.send([body], [0], 1.0,
persons_face=[face], persons_face_ids=[0],
persons_hands=[hand], persons_hands_ids=[1])
av_addrs = {c.args[0] for c in b._avbody.send_message.call_args_list}
assert "/pose/count" in av_addrs
assert "/face/count" in av_addrs
assert "/face/kp" in av_addrs
assert "/hand/count" in av_addrs
assert "/hand/kp" in av_addrs
data_only_viz/training/dataset.py
-83
View File
@@ -15,23 +15,10 @@ class RawFrame:
ts: float
session: str
pid: int
<<<<<<< HEAD
<<<<<<< HEAD
j3d: np.ndarray # (32, 3) float32 (v3: body22 + 10 fingertips)
expression: np.ndarray | None = None # (EXPR_DIM,) or None
mouth_open: float = 0.0
hands_kp: np.ndarray | None = None # (42, 3) or None
=======
j3d: np.ndarray # (32, 3) float32 (v2: body22 + 10 fingertips)
expression: np.ndarray | None = None # (EXPR_DIM,) or None
mouth_open: float = 0.0
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
j3d: np.ndarray # (32, 3) float32 (v3: body22 + 10 fingertips)
expression: np.ndarray | None = None # (EXPR_DIM,) or None
mouth_open: float = 0.0
hands_kp: np.ndarray | None = None # (42, 3) or None
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
@dataclass
@@ -42,14 +29,7 @@ class WindowRow:
first_ts: float
expr_stack: np.ndarray | None = None # (window_len, 10) or None
mouth_open_stack: np.ndarray | None = None # (window_len,) or None
<<<<<<< HEAD
<<<<<<< HEAD
hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
@dataclass
@@ -63,14 +43,7 @@ class DatasetRow:
manually_validated: bool
expr_stack: np.ndarray | None = None # (window_len, 10) or None
mouth_open_stack: np.ndarray | None = None # (window_len,) or None
<<<<<<< HEAD
<<<<<<< HEAD
hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
def load_frames_jsonl(path: Path) -> list[RawFrame]:
@@ -83,16 +56,8 @@ def load_frames_jsonl(path: Path) -> list[RawFrame]:
d = json.loads(line)
expr_raw = d.get("expression")
expr = np.asarray(expr_raw, dtype=np.float32) if expr_raw is not None else None
<<<<<<< HEAD
<<<<<<< HEAD
hands_raw = d.get("hands_kp")
hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
hands_raw = d.get("hands_kp")
hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
rows.append(RawFrame(
ts=float(d["ts"]),
session=str(d["session"]),
@@ -100,14 +65,7 @@ def load_frames_jsonl(path: Path) -> list[RawFrame]:
j3d=np.asarray(d["j3d"], dtype=np.float32),
expression=expr,
mouth_open=float(d.get("mouth_open", 0.0)),
<<<<<<< HEAD
<<<<<<< HEAD
hands_kp=hands_kp,
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
hands_kp=hands_kp,
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
))
return rows
@@ -142,10 +100,6 @@ def sliding_windows(frames: list[RawFrame],
mouth_stack = np.array(
[c.mouth_open for c in chunk], dtype=np.float32
)
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# hands_kp stack: (window_len, 42, 3) if any frame has hands_kp
if any(c.hands_kp is not None for c in chunk):
hands_kp_stack = np.zeros((window_len, 42, 3), dtype=np.float32)
@@ -156,25 +110,11 @@ def sliding_windows(frames: list[RawFrame],
hands_kp_stack[t] = hk
else:
hands_kp_stack = None
<<<<<<< HEAD
yield WindowRow(j3d_stack=stack, session=sess,
pid_local=pid, first_ts=chunk[0].ts,
expr_stack=expr_stack,
mouth_open_stack=mouth_stack,
hands_kp_stack=hands_kp_stack)
=======
yield WindowRow(j3d_stack=stack, session=sess,
pid_local=pid, first_ts=chunk[0].ts,
expr_stack=expr_stack,
mouth_open_stack=mouth_stack)
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
yield WindowRow(j3d_stack=stack, session=sess,
pid_local=pid, first_ts=chunk[0].ts,
expr_stack=expr_stack,
mouth_open_stack=mouth_stack,
hands_kp_stack=hands_kp_stack)
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None:
@@ -193,16 +133,8 @@ def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None:
d["expr_stack"] = r.expr_stack.astype(np.float32).tolist()
if r.mouth_open_stack is not None:
d["mouth_open_stack"] = r.mouth_open_stack.astype(np.float32).tolist()
<<<<<<< HEAD
<<<<<<< HEAD
if r.hands_kp_stack is not None:
d["hands_kp_stack"] = r.hands_kp_stack.astype(np.float32).tolist()
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
if r.hands_kp_stack is not None:
d["hands_kp_stack"] = r.hands_kp_stack.astype(np.float32).tolist()
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
f.write(json.dumps(d) + "\n")
@@ -218,16 +150,8 @@ def load_dataset_jsonl(path: Path) -> list[DatasetRow]:
expr = np.asarray(expr_raw, dtype=np.float32) if expr_raw is not None else None
mouth_raw = d.get("mouth_open_stack")
mouth = np.asarray(mouth_raw, dtype=np.float32) if mouth_raw is not None else None
<<<<<<< HEAD
<<<<<<< HEAD
hands_raw = d.get("hands_kp_stack")
hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
hands_raw = d.get("hands_kp_stack")
hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
out.append(DatasetRow(
window_id=d["window_id"],
label=d["label"],
@@ -238,14 +162,7 @@ def load_dataset_jsonl(path: Path) -> list[DatasetRow]:
manually_validated=bool(d["manually_validated"]),
expr_stack=expr,
mouth_open_stack=mouth,
<<<<<<< HEAD
<<<<<<< HEAD
hands_kp_stack=hands_kp,
=======
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
hands_kp_stack=hands_kp,
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
))
return out
data_only_viz/training/train_action_head.py
-11
View File
@@ -79,10 +79,6 @@ class WindowDataset(Dataset[tuple[torch.Tensor, int]]):
n = min(EXPR_DIM, len(expr_t))
expr_vec[:n] = expr_t[:n]
mouth_t = float(mouth_s[t]) if t < len(mouth_s) else 0.0
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
# hands_kp at frame t (42, 3); zeros if row has none
if row.hands_kp_stack is not None:
hands_t = row.hands_kp_stack[t]
@@ -92,13 +88,6 @@ class WindowDataset(Dataset[tuple[torch.Tensor, int]]):
feat = np.concatenate([
cur.reshape(-1), vel.reshape(-1), accel.reshape(-1),
hands_flat,
<<<<<<< HEAD
=======
feat = np.concatenate([
cur.reshape(-1), vel.reshape(-1), accel.reshape(-1),
>>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face)
=======
>>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips)
expr_vec,
np.array([hip_y, knee_angle, sym, mouth_t], dtype=np.float32),
]).astype(np.float32, copy=False)
data_only_viz/uv.lock
Generated
+1 -136
View File
@@ -21,15 +21,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750, upload-time = "2026-01-28T10:17:04.19Z" },
]
[[package]]
name = "addict"
version = "2.4.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/85/ef/fd7649da8af11d93979831e8f1f8097e85e82d5bfeabc8c68b39175d8e75/addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494", size = 9186, upload-time = "2020-11-21T16:21:31.416Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/6a/00/b08f23b7d7e1e14ce01419a467b583edbb93c6cdb8654e54a9cc579cd61f/addict-2.4.0-py3-none-any.whl", hash = "sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc", size = 3832, upload-time = "2020-11-21T16:21:29.588Z" },
]
[[package]]
name = "annotated-doc"
version = "0.0.4"
@@ -124,21 +115,6 @@ pose = [
{ name = "opencv-python" },
{ name = "ultralytics" },
]
smplerx = [
{ name = "einops" },
{ name = "mmcv-lite" },
{ name = "numpy" },
{ name = "opencv-python" },
{ name = "pillow" },
{ name = "scipy" },
{ name = "smplx" },
{ name = "timm" },
{ name = "torch" },
{ name = "torchvision" },
{ name = "tqdm" },
{ name = "ultralytics" },
{ name = "yacs" },
]
[package.dev-dependencies]
dev = [
@@ -150,24 +126,19 @@ requires-dist = [
{ name = "cloudpickle", marker = "extra == 'detrpose'", specifier = ">=3.0" },
{ name = "coremltools", marker = "extra == 'pose'", specifier = ">=9.0" },
{ name = "einops", marker = "extra == 'multihmr'", specifier = ">=0.8" },
{ name = "einops", marker = "extra == 'smplerx'", specifier = ">=0.8" },
{ name = "huggingface-hub", marker = "extra == 'multihmr'", specifier = ">=0.24" },
{ name = "iopath", marker = "extra == 'detrpose'", specifier = ">=0.1.10" },
{ name = "iopath", marker = "extra == 'multihmr'", specifier = ">=0.1.10" },
{ name = "mediapipe", marker = "extra == 'pose'", specifier = ">=0.10.35" },
{ name = "mmcv-lite", marker = "extra == 'smplerx'", specifier = ">=2.1" },
{ name = "numpy", specifier = ">=1.26,<2" },
{ name = "numpy", marker = "extra == 'multihmr'", specifier = ">=1.26,<2" },
{ name = "numpy", marker = "extra == 'nlf'", specifier = ">=1.26" },
{ name = "numpy", marker = "extra == 'smplerx'", specifier = ">=1.26,<2" },
{ name = "omegaconf", marker = "extra == 'detrpose'", specifier = ">=2.3" },
{ name = "opencv-python", marker = "extra == 'detrpose'", specifier = ">=4.10" },
{ name = "opencv-python", marker = "extra == 'multihmr'", specifier = ">=4.10" },
{ name = "opencv-python", marker = "extra == 'nlf'", specifier = ">=4.10" },
{ name = "opencv-python", marker = "extra == 'pose'", specifier = ">=4.10" },
{ name = "opencv-python", marker = "extra == 'smplerx'", specifier = ">=4.10" },
{ name = "pillow", marker = "extra == 'multihmr'", specifier = ">=10.0" },
{ name = "pillow", marker = "extra == 'smplerx'", specifier = ">=10.0" },
{ name = "pycocotools", marker = "extra == 'detrpose'", specifier = ">=2.0" },
{ name = "pyobjc-core", specifier = ">=10.3" },
{ name = "pyobjc-framework-avfoundation", specifier = ">=10.3" },
@@ -180,29 +151,21 @@ requires-dist = [
{ name = "scipy", specifier = ">=1.13" },
{ name = "scipy", marker = "extra == 'detrpose'", specifier = ">=1.13" },
{ name = "scipy", marker = "extra == 'multihmr'", specifier = ">=1.13" },
{ name = "scipy", marker = "extra == 'smplerx'", specifier = ">=1.13" },
{ name = "smplx", marker = "extra == 'multihmr'", specifier = ">=0.1.28" },
{ name = "smplx", marker = "extra == 'smplerx'", specifier = ">=0.1.28" },
{ name = "timm", marker = "extra == 'smplerx'", specifier = ">=1.0" },
{ name = "torch", marker = "extra == 'detrpose'", specifier = ">=2.4" },
{ name = "torch", marker = "extra == 'multihmr'", specifier = ">=2.4" },
{ name = "torch", marker = "extra == 'nlf'", specifier = ">=2.4" },
{ name = "torch", marker = "extra == 'smplerx'", specifier = ">=2.4" },
{ name = "torchgeometry", marker = "extra == 'multihmr'", specifier = ">=0.1.2" },
{ name = "torchvision", marker = "extra == 'detrpose'", specifier = ">=0.19" },
{ name = "torchvision", marker = "extra == 'multihmr'", specifier = ">=0.19" },
{ name = "torchvision", marker = "extra == 'nlf'", specifier = ">=0.19" },
{ name = "torchvision", marker = "extra == 'smplerx'", specifier = ">=0.19" },
{ name = "tqdm", marker = "extra == 'multihmr'", specifier = ">=4.65" },
{ name = "tqdm", marker = "extra == 'smplerx'", specifier = ">=4.65" },
{ name = "transformers", marker = "extra == 'detrpose'", specifier = ">=4.40" },
{ name = "trimesh", marker = "extra == 'multihmr'", specifier = ">=4.4" },
{ name = "ultralytics", marker = "extra == 'pose'", specifier = ">=8.3" },
{ name = "ultralytics", marker = "extra == 'smplerx'", specifier = ">=8.3" },
{ name = "xtcocotools", marker = "extra == 'detrpose'", specifier = ">=1.14" },
{ name = "yacs", marker = "extra == 'smplerx'", specifier = ">=0.1.8" },
]
provides-extras = ["pose", "detrpose", "nlf", "multihmr", "smplerx"]
provides-extras = ["pose", "detrpose", "nlf", "multihmr"]
[package.metadata.requires-dev]
dev = [{ name = "pytest", specifier = ">=9.0.3" }]
@@ -1140,46 +1103,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/07/b3/5c7fa594c731e8dafab9f1a46ab6cef670fa62dbbfb6248cc70e42ec6fc5/mediapipe-0.10.35-py3-none-win_arm64.whl", hash = "sha256:46255326a6213118aaa518a7aa25e35f93337e82677960cc2a945f117bff8444", size = 9992331, upload-time = "2026-04-27T17:45:36.193Z" },
]
[[package]]
name = "mmcv-lite"
version = "2.2.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "addict" },
{ name = "mmengine" },
{ name = "numpy" },
{ name = "opencv-python" },
{ name = "packaging" },
{ name = "pillow" },
{ name = "pyyaml" },
{ name = "regex", marker = "sys_platform == 'win32'" },
{ name = "yapf" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8c/e7/075329ead4078d77b14e195f04b03162b99c5644d4186505113f62b7ca6c/mmcv-lite-2.2.0.tar.gz", hash = "sha256:62933ea165b2d9ad32e1b72ccd5ccba3cf71b5cd812c4c13c16cb2fcfc46a064", size = 479155, upload-time = "2024-04-24T14:24:38.98Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/59/bd/5d468c171f201d6169bec848d1116e95736854eec72307299bc68939fe45/mmcv_lite-2.2.0-py2.py3-none-any.whl", hash = "sha256:a24ee8dd3df7556dfced282dbfe8c3f87df6de2d4dcaf1207e83e9a2d58455a6", size = 732333, upload-time = "2024-04-24T14:24:28.555Z" },
]
[[package]]
name = "mmengine"
version = "0.10.7"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "addict" },
{ name = "matplotlib" },
{ name = "numpy" },
{ name = "opencv-python" },
{ name = "pyyaml" },
{ name = "regex", marker = "sys_platform == 'win32'" },
{ name = "rich" },
{ name = "termcolor" },
{ name = "yapf" },
]
sdist = { url = "https://files.pythonhosted.org/packages/17/14/959360bbd8374e23fc1b720906999add16a3ac071a501636db12c5861ff5/mmengine-0.10.7.tar.gz", hash = "sha256:d20ffcc31127567e53dceff132612a87f0081de06cbb7ab2bdb7439125a69225", size = 378090, upload-time = "2025-03-04T12:23:09.568Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/98/8e/f98332248aad102511bea4ae19c0ddacd2f0a994f3ca4c82b7a369e0af8b/mmengine-0.10.7-py3-none-any.whl", hash = "sha256:262ac976a925562f78cd5fd14dd1bc9b680ed0aa81f0d85b723ef782f99c54ee", size = 452720, upload-time = "2025-03-04T12:23:06.339Z" },
]
[[package]]
name = "mpmath"
version = "1.3.0"
@@ -1514,15 +1437,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/bc/60/5382c03e1970de634027cee8e1b7d39776b778b81812aaf45b694dfe9e28/pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e", size = 7080946, upload-time = "2026-04-01T14:46:11.734Z" },
]
[[package]]
name = "platformdirs"
version = "4.9.6"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" },
]
[[package]]
name = "pluggy"
version = "1.6.0"
@@ -2249,31 +2163,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" },
]
[[package]]
name = "termcolor"
version = "3.3.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" },
]
[[package]]
name = "timm"
version = "1.0.27"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "huggingface-hub" },
{ name = "pyyaml" },
{ name = "safetensors" },
{ name = "torch" },
{ name = "torchvision" },
]
sdist = { url = "https://files.pythonhosted.org/packages/08/54/ece85b0eef3700c90db8271a43669b05a0ebbe2edb1962329c34374a297e/timm-1.0.27.tar.gz", hash = "sha256:315dfe63186ca9fb7ff941268941231fd5be259f2b4bb4afa28560ae1015cb9a", size = 2439861, upload-time = "2026-05-08T19:38:36.844Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1f/2e/26bab7686ff4aed48f8f5f6c23e2aa37b7a37ddd9effe3aa61e908fd518f/timm-1.0.27-py3-none-any.whl", hash = "sha256:5ff07c9ddf53cbada88eab1c93ff175c64cab683b5a2fddf863bcee985926f89", size = 2589280, upload-time = "2026-05-08T19:38:35.034Z" },
]
[[package]]
name = "tokenizers"
version = "0.22.2"
@@ -2543,27 +2432,3 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/80/e0/01cf7f8b3f4229568b37de680d0eaadc651d5ee36bd483b83658f49dc6c2/xtcocotools-1.14.3-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:126ca596229b2016552bf27cad01f3a2f70a3ff7576a58305a00499cb9e0057d", size = 464351, upload-time = "2023-10-19T07:52:33.199Z" },
{ url = "https://files.pythonhosted.org/packages/a9/10/32bef0fcd29145dcda9bfaa9e11718f40acd444d6804cac870b0437fc7a8/xtcocotools-1.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:47cb5433903f30589343d54530e49abd6b61d0fd119857ba4948b8ce291dbee6", size = 88741, upload-time = "2023-10-19T07:53:51.151Z" },
]
[[package]]
name = "yacs"
version = "0.1.8"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pyyaml" },
]
sdist = { url = "https://files.pythonhosted.org/packages/44/3e/4a45cb0738da6565f134c01d82ba291c746551b5bc82e781ec876eb20909/yacs-0.1.8.tar.gz", hash = "sha256:efc4c732942b3103bea904ee89af98bcd27d01f0ac12d8d4d369f1e7a2914384", size = 11100, upload-time = "2020-08-10T16:37:47.755Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/38/4f/fe9a4d472aa867878ce3bb7efb16654c5d63672b86dc0e6e953a67018433/yacs-0.1.8-py3-none-any.whl", hash = "sha256:99f893e30497a4b66842821bac316386f7bd5c4f47ad35c9073ef089aa33af32", size = 14747, upload-time = "2020-08-10T16:37:46.4Z" },
]
[[package]]
name = "yapf"
version = "0.43.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "platformdirs" },
]
sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907, upload-time = "2024-11-14T00:11:41.584Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/37/81/6acd6601f61e31cfb8729d3da6d5df966f80f374b78eff83760714487338/yapf-0.43.0-py3-none-any.whl", hash = "sha256:224faffbc39c428cb095818cf6ef5511fdab6f7430a10783fdfb292ccf2852ca", size = 256158, upload-time = "2024-11-14T00:11:39.37Z" },
]
launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift
+155
View File
@@ -0,0 +1,155 @@
import SwiftUI
import simd
/// Lightweight SwiftUI overlay that draws the 68-point face skeleton and
/// 21-point hand skeletons sent by data_only_viz/pose_bridge.py over OSC.
/// Sits in the ContentView ZStack above BodyView. Coordinates from the
/// listener are normalised (0..1 in image space) ; here we map them to
/// the overlay's geometry. Rendering is intentionally minimal : small
/// dots + a few polylines for facial features and hand bones.
struct FaceHandOverlay: View {
@ObservedObject var poseListener: PoseOSCListener
var showFace: Bool = true
var showHands: Bool = true
var body: some View {
GeometryReader { geo in
Canvas { ctx, size in
if showFace {
for face in poseListener.faces.values {
drawFace(face, in: &ctx, size: size)
}
}
if showHands {
for hand in poseListener.hands.values {
drawHand(hand, in: &ctx, size: size)
}
}
}
.frame(width: geo.size.width, height: geo.size.height)
.allowsHitTesting(false)
}
}
// MARK: - Face (dlib 68 layout)
/// Index spans in the 68-point dlib convention.
private static let jaw = Array(0..<17)
private static let browR = Array(17..<22)
private static let browL = Array(22..<27)
private static let noseBridge = Array(27..<31)
private static let nostril = Array(31..<36)
private static let eyeR = Array(36..<42)
private static let eyeL = Array(42..<48)
private static let lipOuter = Array(48..<60)
private static let lipInner = Array(60..<68)
private func drawFace(_ face: PoseOSCListener.FaceFrame,
in ctx: inout GraphicsContext,
size: CGSize) {
let stroke = GraphicsContext.Shading.color(.green.opacity(0.85))
let dot = GraphicsContext.Shading.color(.green.opacity(0.95))
drawPolyline(face.points, indices: Self.jaw, closed: false,
in: &ctx, size: size, shading: stroke, width: 1.2)
drawPolyline(face.points, indices: Self.browR, closed: false,
in: &ctx, size: size, shading: stroke, width: 1.2)
drawPolyline(face.points, indices: Self.browL, closed: false,
in: &ctx, size: size, shading: stroke, width: 1.2)
drawPolyline(face.points, indices: Self.noseBridge, closed: false,
in: &ctx, size: size, shading: stroke, width: 1.2)
drawPolyline(face.points, indices: Self.nostril, closed: false,
in: &ctx, size: size, shading: stroke, width: 1.2)
drawPolyline(face.points, indices: Self.eyeR, closed: true,
in: &ctx, size: size, shading: stroke, width: 1.2)
drawPolyline(face.points, indices: Self.eyeL, closed: true,
in: &ctx, size: size, shading: stroke, width: 1.2)
drawPolyline(face.points, indices: Self.lipOuter, closed: true,
in: &ctx, size: size, shading: stroke, width: 1.2)
drawPolyline(face.points, indices: Self.lipInner, closed: true,
in: &ctx, size: size, shading: stroke, width: 1.0)
for i in 0..<68 where face.hasPoint[i] {
let p = mapPoint(face.points[i], size: size)
let r = CGRect(x: p.x - 1.2, y: p.y - 1.2,
width: 2.4, height: 2.4)
ctx.fill(Path(ellipseIn: r), with: dot)
}
}
// MARK: - Hand (MediaPipe 21 landmarks)
/// MediaPipe hand bone connectivity (5 fingers x 4 bones + palm).
private static let handBones: [(Int, Int)] = [
// Thumb
(0, 1), (1, 2), (2, 3), (3, 4),
// Index
(0, 5), (5, 6), (6, 7), (7, 8),
// Middle
(5, 9), (9, 10), (10, 11), (11, 12),
// Ring
(9, 13), (13, 14), (14, 15), (15, 16),
// Pinky
(13, 17), (17, 18), (18, 19), (19, 20),
// Palm closure
(0, 17),
]
private func drawHand(_ hand: PoseOSCListener.HandFrame,
in ctx: inout GraphicsContext,
size: CGSize) {
// Left = cyan, right = magenta.
let color: Color = hand.side == 0 ? .cyan : .pink
let stroke = GraphicsContext.Shading.color(color.opacity(0.85))
let dot = GraphicsContext.Shading.color(color.opacity(0.95))
for (a, b) in Self.handBones {
guard hand.hasPoint[a], hand.hasPoint[b] else { continue }
let pa = mapPoint(hand.points[a], size: size)
let pb = mapPoint(hand.points[b], size: size)
var path = Path()
path.move(to: pa)
path.addLine(to: pb)
ctx.stroke(path, with: stroke, lineWidth: 1.8)
}
for i in 0..<21 where hand.hasPoint[i] {
let p = mapPoint(hand.points[i], size: size)
let r = CGRect(x: p.x - 1.8, y: p.y - 1.8,
width: 3.6, height: 3.6)
ctx.fill(Path(ellipseIn: r), with: dot)
}
}
// MARK: - Helpers
private func drawPolyline(_ pts: [SIMD2<Float>],
indices: [Int],
closed: Bool,
in ctx: inout GraphicsContext,
size: CGSize,
shading: GraphicsContext.Shading,
width: CGFloat) {
guard indices.count >= 2 else { return }
var path = Path()
var started = false
for i in indices {
let p = mapPoint(pts[i], size: size)
if !started {
path.move(to: p)
started = true
} else {
path.addLine(to: p)
}
}
if closed, let first = indices.first {
path.addLine(to: mapPoint(pts[first], size: size))
}
ctx.stroke(path, with: shading, lineWidth: width)
}
private func mapPoint(_ p: SIMD2<Float>, size: CGSize) -> CGPoint {
// Normalised coords come from MediaPipe in image space already.
CGPoint(x: CGFloat(p.x) * size.width,
y: CGFloat(p.y) * size.height)
}
}
launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift
+2 -16
View File
@@ -66,23 +66,9 @@ struct SettingsPanel: View {
layerRow(icon: "circle.dotted",
label: "Fil de fer",
isOn: $settings.showWireframe)
// Squelette toggle : bascule en mode openpos (#9). ON
// synchronise vizMode=9 et showSkeleton=true ; OFF revient
// au mode 0 (storm). Sync bidirectionnel pour que la touche
// "p" et le picker viz updatent le toggle automatiquement.
layerRow(icon: "figure.stand",
label: "Squelette (openpos)",
isOn: Binding(
get: { settings.vizMode == 9 },
set: { on in
if on {
settings.vizMode = 9
settings.showSkeleton = true
} else {
settings.vizMode = 0
settings.showSkeleton = false
}
}))
label: "Squelette (à venir)",
isOn: $settings.showSkeleton)
// Picker viz mode 0..9
HStack(spacing: 4) {
ForEach(0..<10) { i in
launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift
+222
View File
@@ -0,0 +1,222 @@
import Combine
import Foundation
import RealityKit
import SwiftUI
import simd
/// RealityKit renderer for MediaPipe Pose 3D world landmarks (33 joints,
/// metric coords relative to the hip-center). Consumes the `body3d`
/// publisher of `PoseOSCListener` and maintains one entity tree per
/// detected person.
///
/// Coordinate mapping (MediaPipe -> RealityKit):
/// MediaPipe : x = right, y = down, z = forward (away from cam).
/// RealityKit: x = right, y = up, z = backward (toward cam).
/// => convert with (x, -y, -z).
@MainActor
final class Skeleton3DRenderer: ObservableObject {
/// 32 bones connecting MediaPipe Pose 33 landmarks. Indices are
/// the canonical MediaPipe Pose landmark indices. Source: official
/// `mp.solutions.pose.POSE_CONNECTIONS` (Holistic / Pose Landmarker
/// share the same 33-pt schema).
static let POSE_CONNECTIONS: [(Int, Int, BoneChain)] = [
// Face (kept light: nose <-> inner eyes <-> outer eyes <-> ears)
(0, 1, .face), (1, 2, .face), (2, 3, .face), (3, 7, .face),
(0, 4, .face), (4, 5, .face), (5, 6, .face), (6, 8, .face),
(9, 10, .face),
// Torso
(11, 12, .trunk), (11, 23, .trunk), (12, 24, .trunk),
(23, 24, .trunk),
// Left arm
(11, 13, .arm), (13, 15, .arm),
(15, 17, .arm), (15, 19, .arm), (15, 21, .arm), (17, 19, .arm),
// Right arm
(12, 14, .arm), (14, 16, .arm),
(16, 18, .arm), (16, 20, .arm), (16, 22, .arm), (18, 20, .arm),
// Left leg
(23, 25, .leg), (25, 27, .leg),
(27, 29, .leg), (27, 31, .leg), (29, 31, .leg),
// Right leg
(24, 26, .leg), (26, 28, .leg),
(28, 30, .leg), (28, 32, .leg), (30, 32, .leg),
]
enum BoneChain {
case trunk, arm, leg, face
var color: NSColor {
switch self {
case .trunk: return .white
case .arm: return .systemTeal
case .leg: return .systemPink // approx magenta
case .face: return NSColor(white: 0.7, alpha: 1.0)
}
}
}
private static let jointRadius: Float = 0.02 // 2 cm
private static let boneRadius: Float = 0.012 // 1.2 cm
private static let minConfidence: Float = 0.3
private static let retainSec: TimeInterval = 1.0
/// Update throttle : tick at most every `updatePeriod` seconds even
/// if the publisher fires faster (Combine debounce-style on a clock).
private static let updatePeriod: TimeInterval = 1.0 / 30.0
private struct PersonEntities {
var root: Entity
var joints: [ModelEntity] // 33 spheres
var bones: [ModelEntity] // 32 bone entities, same order as POSE_CONNECTIONS
}
private var persons: [Int: PersonEntities] = [:]
private var lastSeenAt: [Int: TimeInterval] = [:]
private weak var rootAnchor: Entity?
private var poseSub: AnyCancellable?
private var lastUpdateAt: TimeInterval = 0
/// Attach to a scene by giving it an AnchorEntity that owns all
/// skeleton entities, and start observing the listener.
func attach(to anchor: Entity, listener: PoseOSCListener) {
rootAnchor = anchor
poseSub = listener.$body3d
.receive(on: DispatchQueue.main)
.sink { [weak self] frames in
Task { @MainActor in self?.update(frames: frames) }
}
}
func detach() {
poseSub?.cancel()
poseSub = nil
for (_, p) in persons { p.root.removeFromParent() }
persons.removeAll()
lastSeenAt.removeAll()
}
// MARK: - Update
private func update(frames: [Int: PoseOSCListener.Pose3DFrame]) {
let now = CACurrentMediaTime()
if now - lastUpdateAt < Self.updatePeriod { return }
lastUpdateAt = now
guard let anchor = rootAnchor else { return }
// Mark fresh pids
for pid in frames.keys { lastSeenAt[pid] = now }
// GC stale persons
let cutoff = now - Self.retainSec
for (pid, p) in persons where (lastSeenAt[pid] ?? 0) < cutoff {
p.root.removeFromParent()
persons.removeValue(forKey: pid)
lastSeenAt.removeValue(forKey: pid)
}
for (pid, frame) in frames {
let entities = persons[pid] ?? makePerson(pid: pid, parent: anchor)
persons[pid] = entities
apply(frame: frame, to: entities)
}
}
private func apply(frame: PoseOSCListener.Pose3DFrame,
to entities: PersonEntities) {
// Convert all 33 keypoints to RealityKit space once.
var rk = [SIMD3<Float>](repeating: .zero, count: 33)
var valid = [Bool](repeating: false, count: 33)
for i in 0..<33 {
let k = frame.kps[i]
let visible = frame.hasPoint[i] && k.w >= Self.minConfidence
valid[i] = visible
// Mediapipe (x right, y down, z forward) -> RK (x right, y up, z back)
rk[i] = SIMD3<Float>(k.x, -k.y, -k.z)
}
// Joints: position spheres and toggle visibility.
for i in 0..<33 {
let joint = entities.joints[i]
if valid[i] {
joint.transform.translation = rk[i]
joint.isEnabled = true
} else {
joint.isEnabled = false
}
}
// Bones: orient + scale length between endpoints.
for (bIdx, (a, b, _)) in Self.POSE_CONNECTIONS.enumerated() {
let bone = entities.bones[bIdx]
if !valid[a] || !valid[b] {
bone.isEnabled = false
continue
}
let pa = rk[a]
let pb = rk[b]
let delta = pb - pa
let len = simd_length(delta)
if len < 1e-5 {
bone.isEnabled = false
continue
}
let mid = (pa + pb) * 0.5
// Bone mesh is a cylinder of height=1 along +Y. Rotate +Y
// onto the (b-a) direction.
let dir = delta / len
let yAxis = SIMD3<Float>(0, 1, 0)
let dot = simd_dot(yAxis, dir)
let rot: simd_quatf
if dot > 0.9999 {
rot = simd_quatf(angle: 0, axis: SIMD3(0, 1, 0))
} else if dot < -0.9999 {
rot = simd_quatf(angle: .pi, axis: SIMD3(1, 0, 0))
} else {
let axis = simd_normalize(simd_cross(yAxis, dir))
let angle = acos(dot)
rot = simd_quatf(angle: angle, axis: axis)
}
bone.transform.translation = mid
bone.transform.rotation = rot
// Scale length only on Y, keep XZ at 1 to preserve radius.
bone.transform.scale = SIMD3<Float>(1, len, 1)
bone.isEnabled = true
}
}
// MARK: - Construction
private func makePerson(pid: Int, parent: Entity) -> PersonEntities {
let root = Entity()
parent.addChild(root)
// Joint sphere mesh shared across joints (cheap to reuse).
let sphereMesh = MeshResource.generateSphere(
radius: Self.jointRadius)
let jointMat = SimpleMaterial(
color: .white, roughness: 0.6, isMetallic: false)
var joints: [ModelEntity] = []
joints.reserveCapacity(33)
for _ in 0..<33 {
let e = ModelEntity(mesh: sphereMesh, materials: [jointMat])
e.isEnabled = false
root.addChild(e)
joints.append(e)
}
// One cylinder per bone (height=1, scaled at runtime).
let cylMesh = MeshResource.generateCylinder(
height: 1.0, radius: Self.boneRadius)
var bones: [ModelEntity] = []
bones.reserveCapacity(Self.POSE_CONNECTIONS.count)
for (_, _, chain) in Self.POSE_CONNECTIONS {
let mat = SimpleMaterial(
color: chain.color, roughness: 0.6, isMetallic: false)
let e = ModelEntity(mesh: cylMesh, materials: [mat])
e.isEnabled = false
root.addChild(e)
bones.append(e)
}
NSLog("Skeleton3DRenderer: spawned pid=%d (33 joints, %d bones)",
pid, bones.count)
return PersonEntities(root: root, joints: joints, bones: bones)
}
}
sound_algo/control/data_feeds.scd
+37
View File
@@ -294,6 +294,43 @@
("[feeds] heartbeat: " ++ if(~feedAlive.value) { "ALIVE" } { "DOWN" }).postln;
};
// =====================================================================
// ACTION-HEAD OSC handlers (/pose/action, /pose/kin, /pose/enter, /pose/leave)
// =====================================================================
~poseState = ~poseState ? Dictionary.new;
~poseKin = ~poseKin ? Dictionary.new;
OSCdef(\poseAction, { |msg|
var pid = msg[1];
~poseState[pid] = (
labelIdx: msg[2],
probs: [msg[3], msg[4], msg[5]],
);
}, '/pose/action');
OSCdef(\poseKin, { |msg|
var pid = msg[1];
~poseKin[pid] = (
speed: msg[2],
accel: msg[3],
symmetry: msg[4],
);
}, '/pose/kin');
OSCdef(\poseEnter, { |msg|
var pid = msg[1];
~poseState[pid] = (labelIdx: 0, probs: [1.0, 0.0, 0.0]);
~poseKin[pid] = (speed: 0, accel: 0, symmetry: 0);
}, '/pose/enter');
OSCdef(\poseLeave, { |msg|
var pid = msg[1];
~poseState.removeAt(pid);
~poseKin.removeAt(pid);
}, '/pose/leave');
"[OK] action-head OSC handlers".postln;
"[data_feeds] OSCdef installes (USGS, SWPC, NETZ, RTE, BLITZ, OPENSKY, BSKY, MEMPOOL, GCN).".postln;
"[data_feeds] usage : ~feeds[\\swpc_kp], ~feedGet.(\\netz_dev, 0), ~feedDump.()".postln;
)