From 531363dea3cb15e0be1cffda138482a77ce375bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Thu, 14 May 2026 00:01:03 +0200 Subject: [PATCH] chore(main): sync working tree to feat/action-head Apres cherry-pick des 14 commits action-head, le working tree main divergeait encore de feat/action-head sur 21 fichiers a cause des resolutions modify/delete qui avaient pris la mauvaise version (c52271e botched merge avait drop ces fichiers, le cherry-pick voyait alors "modify in feat / deleted in main" et conservait la mauvaise version dans certains cas). Cette commit synchronise le working tree main vers feat/action-head en checkout-ant tous les fichiers depuis feat. Equivalent fonctionnel a un fast-forward merge sans toucher l'historique. Files restored : Skeleton3DRenderer.swift, FaceHandOverlay.swift, test_multihmr_coreml.py, test_pose_bridge_*.py (4 test files). Files updated : action_head.py, action_head_pub.py, multi.py, pose_bridge.py, state.py, extract_j3d_offline.py, training/*.py, SettingsPanel.swift, data_feeds.scd, et plus. Result : main == feat/action-head sur le working tree, historique granulaire preserve via les cherry-picks precedents. --- data_only_viz/action_head.py | 84 ------ data_only_viz/action_head_pub.py | 136 ---------- data_only_viz/apple_vision_pose.py | 12 - data_only_viz/multi.py | 27 +- data_only_viz/pose_bridge.py | 239 ++++++++++++++++-- data_only_viz/scripts/convert_dinov2.py | 167 ++++++++++++ data_only_viz/scripts/extract_j3d_offline.py | 69 ----- data_only_viz/state.py | 14 + .../tests/test_action_head_features.py | 18 -- data_only_viz/tests/test_action_head_model.py | 16 -- data_only_viz/tests/test_action_head_pub.py | 8 - data_only_viz/tests/test_multihmr_coreml.py | 96 +++++++ .../tests/test_pose_bridge_action.py | 49 ++++ .../tests/test_pose_bridge_body3d.py | 79 ++++++ .../tests/test_pose_bridge_face_hand.py | 105 ++++++++ data_only_viz/training/dataset.py | 83 ------ data_only_viz/training/train_action_head.py | 11 - data_only_viz/uv.lock | 137 +--------- .../Sources/AVLiveBody/FaceHandOverlay.swift | 155 ++++++++++++ .../Sources/AVLiveBody/SettingsPanel.swift | 18 +- .../AVLiveBody/Skeleton3DRenderer.swift | 222 ++++++++++++++++ sound_algo/control/data_feeds.scd | 37 +++ 22 files changed, 1173 insertions(+), 609 deletions(-) create mode 100644 data_only_viz/scripts/convert_dinov2.py create mode 100644 data_only_viz/tests/test_multihmr_coreml.py create mode 100644 data_only_viz/tests/test_pose_bridge_action.py create mode 100644 data_only_viz/tests/test_pose_bridge_body3d.py create mode 100644 data_only_viz/tests/test_pose_bridge_face_hand.py create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift diff --git a/data_only_viz/action_head.py b/data_only_viz/action_head.py index ae20c5e..c455ede 100644 --- a/data_only_viz/action_head.py +++ b/data_only_viz/action_head.py @@ -32,10 +32,6 @@ LABELS: tuple[str, str, str] = ("debout", "assise", "danse") EXPR_DIM: int = 10 EXTRA_SCALARS: int = 4 # hip_y, knee_angle, sym_score, mouth_open -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # NEW v3 : MediaPipe Hands keypoints block. HANDS_KP_PER_HAND: int = 21 HANDS_KP_TOTAL: int = 2 * HANDS_KP_PER_HAND # 42 @@ -44,30 +40,12 @@ HANDS_KP_FLAT: int = HANDS_KP_TOTAL * HANDS_KP_DIMS # 126 # Layout per step (v3) : # [0 : 96] j3d (32, 3) -<<<<<<< HEAD # [96 : 192] vel (32, 3) # [192 : 288] accel (32, 3) # [288 : 414] hands_kp (42, 3) zero-padded if absent # [414 : 424] expression (10,) # [424 : 428] scalars (hip_y, knee_angle, sym, mouth_open) FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + HANDS_KP_FLAT + EXPR_DIM + EXTRA_SCALARS # 428 -======= -# Layout per step: -# [0 : 96 ] j3d (32, 3) -# [96 : 192] vel (32, 3) -# [192 : 288] accel (32, 3) -# [288 : 298] expression (10,) -# [298 : 302] scalars (hip_y, knee_angle, sym, mouth_open) -FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + EXPR_DIM + EXTRA_SCALARS # 302 ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= -# [96 : 192] vel (32, 3) -# [192 : 288] accel (32, 3) -# [288 : 414] hands_kp (42, 3) zero-padded if absent -# [414 : 424] expression (10,) -# [424 : 428] scalars (hip_y, knee_angle, sym, mouth_open) -FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + HANDS_KP_FLAT + EXPR_DIM + EXTRA_SCALARS # 428 ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # Body joint indices (unchanged from v1, indices 0..21). HIP_LEFT: int = 1 @@ -89,8 +67,6 @@ FINGERTIP_RIGHT_BASE: int = 27 class FeatureExtractor: """Stateless feature builder over a list of recent j3d frames. -<<<<<<< HEAD -<<<<<<< HEAD Vector layout (FEATURE_DIM = 428, v3): [0 : 96] j3d current frame, flattened (32 joints x 3 dims) [96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3) @@ -98,39 +74,13 @@ class FeatureExtractor: [288 : 414] hands_kp (42, 3) MediaPipe Hands, zero-padded if absent [414 : 424] expression PCA coefficients (10,) [424 : 428] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open) -======= - Vector layout (FEATURE_DIM = 302): - [0 : 96 ] j3d current frame, flattened (32 joints x 3 dims) - [96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3) - [192 : 288] acceleration vel[t] - vel[t-1] (32 x 3) - [288 : 298] expression PCA coefficients (10,) - [298 : 302] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open) ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - Vector layout (FEATURE_DIM = 428, v3): - [0 : 96] j3d current frame, flattened (32 joints x 3 dims) - [96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3) - [192 : 288] acceleration vel[t] - vel[t-1] (32 x 3) - [288 : 414] hands_kp (42, 3) MediaPipe Hands, zero-padded if absent - [414 : 424] expression PCA coefficients (10,) - [424 : 428] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open) ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) """ @staticmethod def from_buffer(frames: list[np.ndarray], expr: np.ndarray | None = None, -<<<<<<< HEAD -<<<<<<< HEAD mouth_open: float = 0.0, hands_kp: np.ndarray | None = None) -> np.ndarray: -======= - mouth_open: float = 0.0) -> np.ndarray: ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - mouth_open: float = 0.0, - hands_kp: np.ndarray | None = None) -> np.ndarray: ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) if not frames: return np.zeros(FEATURE_DIM, dtype=np.float32) cur = frames[-1] @@ -142,10 +92,6 @@ class FeatureExtractor: hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5) knee_angle = FeatureExtractor._mean_knee_angle(cur) sym = FeatureExtractor._symmetry_score(vel) -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # hands block (42, 3) -> 126 hands_flat = np.zeros(HANDS_KP_FLAT, dtype=np.float32) if hands_kp is not None: @@ -153,11 +99,6 @@ class FeatureExtractor: if hk.shape == (HANDS_KP_TOTAL, HANDS_KP_DIMS): hands_flat = hk.reshape(-1).astype(np.float32, copy=False) # expression -<<<<<<< HEAD -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) if expr is None: expr_vec = np.zeros(EXPR_DIM, dtype=np.float32) else: @@ -168,14 +109,7 @@ class FeatureExtractor: cur.reshape(-1), vel.reshape(-1), accel.reshape(-1), -<<<<<<< HEAD -<<<<<<< HEAD hands_flat, -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - hands_flat, ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) expr_vec, np.array([hip_y, knee_angle, sym, float(mouth_open)], dtype=np.float32), ]).astype(np.float32, copy=False) @@ -310,17 +244,8 @@ class ActionHead: def step(self, pid: int, j3d: np.ndarray, expr: np.ndarray | None = None, -<<<<<<< HEAD -<<<<<<< HEAD mouth_open: float = 0.0, hands_kp: np.ndarray | None = None) -> tuple[str, np.ndarray, np.ndarray]: -======= - mouth_open: float = 0.0) -> tuple[str, np.ndarray, np.ndarray]: ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - mouth_open: float = 0.0, - hands_kp: np.ndarray | None = None) -> tuple[str, np.ndarray, np.ndarray]: ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) if np.isnan(j3d).any(): streak = self._nan_streak.get(pid, 0) + 1 self._nan_streak[pid] = streak @@ -334,17 +259,8 @@ class ActionHead: if len(frames) < WARMUP_FRAMES: probs = np.array([1.0, 0.0, 0.0], dtype=np.float32) return LABELS[0], probs, np.zeros(3, dtype=np.float32) -<<<<<<< HEAD -<<<<<<< HEAD feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open, hands_kp=hands_kp) -======= - feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open) ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open, - hands_kp=hands_kp) ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) kin = FeatureExtractor.kinetics(frames) h = self._hidden.get(pid) if h is None: diff --git a/data_only_viz/action_head_pub.py b/data_only_viz/action_head_pub.py index 48da68f..51bd033 100644 --- a/data_only_viz/action_head_pub.py +++ b/data_only_viz/action_head_pub.py @@ -17,18 +17,9 @@ import numpy as np from data_only_viz.action_head import ( ActionHead, EXPR_DIM, -<<<<<<< HEAD -<<<<<<< HEAD HANDS_KP_DIMS, HANDS_KP_PER_HAND, HANDS_KP_TOTAL, -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - HANDS_KP_DIMS, - HANDS_KP_PER_HAND, - HANDS_KP_TOTAL, ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) J3D_FINGERS, J3D_FINGERS_PER_HAND, LABELS, @@ -40,30 +31,12 @@ DEFAULT_CKPT = ( Path.home() / ".cache" / "av-live-action" / "checkpoints" / "action_head.pt" ) -<<<<<<< HEAD -<<<<<<< HEAD # Canonical SMPL-X fingertip vertex IDs from smplx.vertex_ids.SMPLX_VERTEX_IDS. # Order : L thumb, L index, L middle, L ring, L pinky, # R thumb, R index, R middle, R ring, R pinky. SMPLX_FINGERTIP_VERTS: tuple[int, ...] = ( 5361, 4933, 5058, 5169, 5286, # L : lthumb, lindex, lmiddle, lring, lpinky 8079, 7669, 7794, 7905, 8022, # R : rthumb, rindex, rmiddle, rring, rpinky -======= -# Approximate fingertip vertex indices on SMPL-X 10475-vert mesh. -# Order: L thumb, L index, L middle, L ring, L pinky, -# R thumb, R index, R middle, R ring, R pinky. -SMPLX_FINGERTIP_VERTS: tuple[int, ...] = ( - 7174, 7397, 7670, 7942, 8214, # L - 4631, 4854, 5127, 5399, 5671, # R ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= -# Canonical SMPL-X fingertip vertex IDs from smplx.vertex_ids.SMPLX_VERTEX_IDS. -# Order : L thumb, L index, L middle, L ring, L pinky, -# R thumb, R index, R middle, R ring, R pinky. -SMPLX_FINGERTIP_VERTS: tuple[int, ...] = ( - 5361, 4933, 5058, 5169, 5286, # L : lthumb, lindex, lmiddle, lring, lpinky - 8079, 7669, 7794, 7905, 8022, # R : rthumb, rindex, rmiddle, rring, rpinky ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) ) # 32 vertex indices on the 10475-vertex SMPL-X mesh: @@ -86,20 +59,11 @@ assert len(SMPLX_JOINT_ANCHOR_VERTS) == 32 SMPLX_UPPER_LIP_VERT: int = 8970 SMPLX_LOWER_LIP_VERT: int = 8855 -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # MediaPipe FaceMesh inner-mouth landmark indices. # 13 = upper inner mid, 14 = lower inner mid. MEDIAPIPE_LIP_UPPER_INNER: int = 13 MEDIAPIPE_LIP_LOWER_INNER: int = 14 -<<<<<<< HEAD -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # MediaPipe HAND fingertip indices (21-kp hand model). MEDIAPIPE_HAND_FINGERTIPS: tuple[int, ...] = (4, 8, 12, 16, 20) @@ -164,27 +128,11 @@ class ActionHeadPublisher(threading.Thread): self._last_body_t = source_t current_pids: set[int] = set() if persons32: -<<<<<<< HEAD -<<<<<<< HEAD for pid, j3d, expr_np, mouth, hands_kp42 in persons32: current_pids.add(pid) label, probs, kin = self.head.step(pid, j3d, expr=expr_np, mouth_open=mouth, hands_kp=hands_kp42) -======= - for pid, j3d, expr, mouth in persons32: - current_pids.add(pid) - label, probs, kin = self.head.step(pid, j3d, - expr=expr, - mouth_open=mouth) ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - for pid, j3d, expr_np, mouth, hands_kp42 in persons32: - current_pids.add(pid) - label, probs, kin = self.head.step(pid, j3d, expr=expr_np, - mouth_open=mouth, - hands_kp=hands_kp42) ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) idx = LABELS.index(label) self.bridge.send_action(pid, idx, probs, t_now, force=True) self.bridge.send_kin(pid, kin, t_now, force=True) @@ -195,32 +143,13 @@ class ActionHeadPublisher(threading.Thread): self.bridge.send_leave(pid=gone) self._last_pids = current_pids -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) def _read_sources(self) -> tuple[ list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] | None, float, str, bool, ]: -<<<<<<< HEAD """Return (persons32, source_t, source_tag, is_new). Each person entry is (pid, j3d32, expr10, mouth_open, hands_kp42x3). -======= - def _read_sources( - self, - ) -> tuple[list[tuple[int, np.ndarray, np.ndarray, float]] | None, - float, str, bool]: - """Return (persons32, source_t, source_tag, is_new). - - Each person entry is (pid, j3d32, expr10, mouth_open). ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - """Return (persons32, source_t, source_tag, is_new). - - Each person entry is (pid, j3d32, expr10, mouth_open, hands_kp42x3). ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) is_new is True when the timestamp advanced (even if person list is empty), so _tick can still run the purge loop. """ @@ -234,10 +163,6 @@ class ActionHeadPublisher(threading.Thread): persons_hands = getattr(self.state, "persons_hands", None) ids_hands = getattr(self.state, "persons_hands_ids", None) t_body = getattr(self.state, "pose_last_t", 0.0) -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # Build pid -> hands_kp(42, 3) map from MediaPipe persons_hands. hands_by_pid: dict[int, np.ndarray] = self._build_hands_map( @@ -249,20 +174,8 @@ class ActionHeadPublisher(threading.Thread): ) # SMPL-X path (preferred) -<<<<<<< HEAD if t_smplx > self._last_smplx_t: out: list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] = [] -======= - hands_ids = list(getattr(self.state, "persons_hands_ids", None) or []) - hands_lists = list(getattr(self.state, "persons_hands", None) or []) - # Prefer smplx when its timestamp advanced. - if t_smplx > self._last_smplx_t: - out: list[tuple[int, np.ndarray, np.ndarray, float]] = [] ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - if t_smplx > self._last_smplx_t: - out: list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] = [] ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) for i, p in enumerate(persons_smplx or []): pid = int(p.get("pid", i)) v3d = p.get("v3d") @@ -284,40 +197,19 @@ class ActionHeadPublisher(threading.Thread): expr_np = np.asarray(expr, dtype=np.float32).flatten() else: expr_np = np.zeros(EXPR_DIM, dtype=np.float32) -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # mouth_open: prefer MediaPipe face lips, fallback SMPL-X v3d. if pid in face_mouth_by_pid: mouth = face_mouth_by_pid[pid] elif v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT): -<<<<<<< HEAD -======= - # mouth_open - if v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT): ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) mouth = float(np.linalg.norm( v3d_np[SMPLX_UPPER_LIP_VERT] - v3d_np[SMPLX_LOWER_LIP_VERT] )) else: mouth = 0.0 -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) hands_kp42 = hands_by_pid.get( pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32) ) out.append((pid, j3d32, expr_np, mouth, hands_kp42)) -<<<<<<< HEAD -======= - out.append((pid, j3d32, expr_np, mouth)) ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) return out or None, t_smplx, "smplx", True # MediaPipe body3d fallback @@ -330,8 +222,6 @@ class ActionHeadPublisher(threading.Thread): if arr is None or arr.shape[0] < 33: continue body22 = arr[list(MEDIAPIPE_TO_22)].astype(np.float32) -<<<<<<< HEAD -<<<<<<< HEAD # fingertips from persons_hands if available tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32) hands_kp42 = hands_by_pid.get( @@ -348,32 +238,6 @@ class ActionHeadPublisher(threading.Thread): mouth = face_mouth_by_pid.get(pid, 0.0) expr_np = np.zeros(EXPR_DIM, dtype=np.float32) out.append((pid, j3d32, expr_np, mouth, hands_kp42)) -======= - # fingertips from hands if available -======= - # fingertips from persons_hands if available ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) - tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32) - hands_kp42 = hands_by_pid.get( - pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32) - ) - # extract fingertips from hands_kp42 (idx 4,8,12,16,20 each side) - for side_idx in (0, 1): - base = side_idx * HANDS_KP_PER_HAND - for k, mp_tip in enumerate(MEDIAPIPE_HAND_FINGERTIPS): - if base + mp_tip < hands_kp42.shape[0]: - tips[side_idx * J3D_FINGERS_PER_HAND + k] = \ - hands_kp42[base + mp_tip] - j3d32 = np.concatenate([body22, tips], axis=0) - mouth = face_mouth_by_pid.get(pid, 0.0) - expr_np = np.zeros(EXPR_DIM, dtype=np.float32) -<<<<<<< HEAD - mouth = 0.0 - out.append((pid, j3d32, expr_np, mouth)) ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - out.append((pid, j3d32, expr_np, mouth, hands_kp42)) ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) return out or None, t_body, "body3d", True return None, 0.0, "", False diff --git a/data_only_viz/apple_vision_pose.py b/data_only_viz/apple_vision_pose.py index 9cb5526..2ccb3f5 100644 --- a/data_only_viz/apple_vision_pose.py +++ b/data_only_viz/apple_vision_pose.py @@ -557,20 +557,8 @@ class AppleVisionPoseWorker: # API d'acces simple. Face parsing depuis Apple Vision est # actuellement bloque ; on garde MediaPipe (CPU XNNPACK) pour # face/hand fin tandis que Vision sert body 2D sur ANE. -<<<<<<< HEAD -<<<<<<< HEAD # Skip pour eviter le spam ObjCPointerWarning a 30 fps. return -======= - n = 0; n_written = 0 - if n_written > 0 and not hasattr(self, "_logged_face_write_" + region_name): - LOG.info("face: %s wrote %d points", region_name, n_written) - setattr(self, "_logged_face_write_" + region_name, True) ->>>>>>> 28d562b (feat(av-live): wire Apple Vision body pose on ANE) -======= - # Skip pour eviter le spam ObjCPointerWarning a 30 fps. - return ->>>>>>> 623c479 (perf(multi-hmr): autocast opt-in) # faceContour fill("faceContour", *FACE_OFFSETS["contour"]) diff --git a/data_only_viz/multi.py b/data_only_viz/multi.py index bd119b5..3f6ed0c 100644 --- a/data_only_viz/multi.py +++ b/data_only_viz/multi.py @@ -19,9 +19,10 @@ import time import urllib.request from pathlib import Path +from .action_head_pub import ActionHeadPublisher from .euro_filter import SkeletonFilter from .pose_bridge import PoseSoundBridge -from .state import PoseKp, State +from .state import Kp3D, PoseKp, State from .tracker import IoUTracker LOG = logging.getLogger("multi") @@ -93,6 +94,8 @@ class MultiWorker: self._smooth_hand = SkeletonFilter(min_cutoff=2.0, beta=0.10) # Pont OSC pose -> sclang self._sound_bridge = PoseSoundBridge(throttle_hz=30.0) + self._action_pub = ActionHeadPublisher(state=self.state, bridge=self._sound_bridge) + self._action_pub.start() def start(self) -> None: self._thread = threading.Thread( @@ -198,6 +201,21 @@ class MultiWorker: x=float(lm.x), y=float(lm.y), z=z, c=float(v))) bodies.append(kp_list) + # pose_world_landmarks : xyz metric, relative to hip-center. + # Aligned 1:1 with pose_landmarks order. Empty fallback if + # the MediaPipe build doesn't populate it. + bodies3d: list[list[Kp3D]] = [] + world_list = getattr(pose_res, "pose_world_landmarks", None) or [] + for landmarks_list in world_list: + kp3_list: list[Kp3D] = [] + for lm in landmarks_list[:33]: + v = lm.visibility if lm.visibility is not None else 1.0 + kp3_list.append(Kp3D( + x=float(lm.x), y=float(lm.y), + z=float(lm.z if lm.z is not None else 0.0), + c=float(v))) + bodies3d.append(kp3_list) + faces = [] for landmarks_list in (face_res.face_landmarks or []): kp_list = [] @@ -230,16 +248,21 @@ class MultiWorker: for i, kps in enumerate(hands)] # Pont sonore : envoi OSC /pose/* a sclang (body + face + hands) + # 3D world landmarks share ids with bodies (same MediaPipe + # detection, just a different coordinate space). + ids_body3d = ids_body[:len(bodies3d)] if bodies3d else [] self._sound_bridge.send( bodies, ids_body, t_now, persons_face=faces, persons_face_ids=ids_face, - persons_hands=hands, persons_hands_ids=ids_hand) + persons_hands=hands, persons_hands_ids=ids_hand, + persons_body3d=bodies3d, persons_body3d_ids=ids_body3d) with self.state.lock(): self.state.persons_body = bodies self.state.persons_face = faces self.state.persons_hands = hands self.state.persons_body_ids = ids_body + self.state.persons_body3d = bodies3d self.state.persons_face_ids = ids_face self.state.persons_hands_ids = ids_hand # Compat single-person (1ere personne) diff --git a/data_only_viz/pose_bridge.py b/data_only_viz/pose_bridge.py index d49525f..4c5a08a 100644 --- a/data_only_viz/pose_bridge.py +++ b/data_only_viz/pose_bridge.py @@ -10,13 +10,21 @@ Routes emises : /pose/head position du nez (visage) /pose/sho_span ecart epaules (estime distance camera) /pose/limb_span envergure brassse (poignet a poignet) + /face/count nombre de visages detectes + /face/kp 68-pt subset (dlib mapping) + /hand/count nombre de mains gauche / droite + /hand/kp 21 landmarks + /pose3d/count nombre de squelettes 3D + /pose3d/kp 33 MediaPipe world landmarks (metres) Mapping pose -> son est defini cote SC dans sound_algo/data_only/scenes.scd -(scene `live_pose`). +(scene `live_pose`). Face / hand keypoints are consumed by the Swift +launcher (AVLiveBody) on 127.0.0.1:57126 for skeleton overlay rendering. """ from __future__ import annotations import logging +from typing import Any, Iterable, Sequence from pythonosc.udp_client import SimpleUDPClient @@ -33,6 +41,42 @@ LEFT_HIP = 23 RIGHT_HIP = 24 +# MediaPipe FaceMesh (468 landmarks) -> 68-point dlib-style subset. +# Mapping inspired by community references (Google MediaPipe -> +# iBUG 68 facial landmarks). Order matches dlib 68-point convention : +# [0..16] jaw contour (left to right) +# [17..21] right brow +# [22..26] left brow +# [27..30] nose bridge (top to tip) +# [31..35] nostril base (right to left) +# [36..41] right eye (CCW from outer corner) +# [42..47] left eye (CCW from inner corner) +# [48..59] outer lip (CCW from right corner) +# [60..67] inner lip (CCW from right corner) +FACE_68_FROM_MP: tuple[int, ...] = ( + # Jaw (17) + 127, 234, 132, 172, 150, 176, 148, 152, + 377, 400, 365, 397, 361, 401, 366, 447, 356, + # Right brow (5) — mediapipe perspective is mirrored vs subject + 70, 63, 105, 66, 107, + # Left brow (5) + 336, 296, 334, 293, 300, + # Nose bridge (4) + 168, 6, 197, 195, + # Nostril base (5) + 98, 97, 2, 326, 327, + # Right eye (6) + 33, 160, 158, 133, 153, 144, + # Left eye (6) + 362, 385, 387, 263, 373, 380, + # Outer lip (12) + 61, 39, 37, 0, 267, 269, 291, 405, 314, 17, 84, 181, + # Inner lip (8) + 78, 81, 13, 311, 308, 402, 14, 178, +) +assert len(FACE_68_FROM_MP) == 68 + + class PoseSoundBridge: """Envoie les keypoints en OSC vers sclang. Throttle a 30 Hz max.""" @@ -45,9 +89,17 @@ class PoseSoundBridge: self._period = 1.0 / max(1.0, throttle_hz) self._last_t = 0.0 - def send(self, persons_body: list, persons_body_ids: list, t_now: float) -> None: + def send(self, persons_body: list, persons_body_ids: list, t_now: float, + *, + persons_face: Sequence[Sequence[Any]] | None = None, + persons_face_ids: Sequence[int] | None = None, + persons_hands: Sequence[Sequence[Any]] | None = None, + persons_hands_ids: Sequence[int] | None = None, + persons_body3d: Sequence[Sequence[Any]] | None = None, + persons_body3d_ids: Sequence[int] | None = None) -> None: """Envoie les keypoints de toutes les personnes detectees. - Throttle automatiquement.""" + Throttle automatiquement. Face / hand sont optionnels et envoyes + sur le meme socket :57126 vers AVLiveBody.""" if t_now - self._last_t < self._period: return self._last_t = t_now @@ -59,22 +111,20 @@ class PoseSoundBridge: except OSError: pass except OSError: return # SC pas la, on continue silencieusement - if n == 0: - return - for i, body in enumerate(persons_body): - pid = persons_body_ids[i] if i < len(persons_body_ids) else i - self._emit_person(int(pid), body) - # Skeleton complet pour AV-Live-Body overlay : 33 kp body. - # On envoie en flat list pour minimiser le surcout OSC. - flat = [] - for kp in body: - flat.extend([float(kp.x), float(kp.y), float(kp.c)]) - try: - self._avbody.send_message( - "/pose/skel", [int(pid)] + flat) - except OSError: - pass + if n > 0: + for i, body in enumerate(persons_body): + pid = persons_body_ids[i] if i < len(persons_body_ids) else i + self._emit_person(int(pid), body) + + # Face / hand : independant de la presence de body kp (utile en + # mode face-only ou hand-only). + if persons_face is not None: + self._send_face(persons_face, persons_face_ids or []) + if persons_hands is not None: + self._send_hand(persons_hands, persons_hands_ids or []) + if persons_body3d is not None: + self._send_body3d(persons_body3d, persons_body3d_ids or []) # ------------------------------------------------------------------ def _emit_person(self, pid: int, body: list) -> None: @@ -123,3 +173,156 @@ class PoseSoundBridge: cli.send_message("/pose/limb_span", [pid, float(span)]) try: self._avbody.send_message("/pose/limb_span", [pid, float(span)]) except OSError: pass + + # ------------------------------------------------------------------ + def send_face(self, persons_face: Sequence[Sequence[Any]], + persons_face_ids: Sequence[int], t_now: float, + force: bool = False) -> None: + """Public throttled entry point for face keypoints. + + Emits a 68-point dlib-style subset of the 468 MediaPipe FaceMesh + landmarks per person on /face/count + /face/kp routes. + """ + if not force and (t_now - self._last_t) < self._period: + return + self._send_face(persons_face, persons_face_ids) + + def send_hand(self, persons_hands: Sequence[Sequence[Any]], + persons_hands_ids: Sequence[int], t_now: float, + force: bool = False) -> None: + """Public throttled entry point for hand keypoints. + + Emits the full 21-landmark hand skeleton per detected hand on + /hand/count + /hand/kp routes. Side is inferred from id parity + (MediaPipe Hand task does not flag left/right reliably) : we + treat odd ids as right, even as left, which matches the + convention used by the smoother / tracker upstream. + """ + if not force and (t_now - self._last_t) < self._period: + return + self._send_hand(persons_hands, persons_hands_ids) + + def _send_face(self, persons_face: Sequence[Sequence[Any]], + persons_face_ids: Sequence[int]) -> None: + n = len(persons_face) + try: + self._avbody.send_message("/face/count", [int(n)]) + except OSError: + return + for i, face in enumerate(persons_face): + if not face: + continue + pid = persons_face_ids[i] if i < len(persons_face_ids) else i + n_lm = len(face) + for slot, mp_idx in enumerate(FACE_68_FROM_MP): + if mp_idx >= n_lm: + continue + kp = face[mp_idx] + try: + self._avbody.send_message("/face/kp", [ + int(pid), int(slot), + float(kp.x), float(kp.y), + float(getattr(kp, "z", 0.0)), + float(getattr(kp, "c", 1.0)), + ]) + except OSError: + return + + def _send_hand(self, persons_hands: Sequence[Sequence[Any]], + persons_hands_ids: Sequence[int]) -> None: + n_left = 0 + n_right = 0 + for i in range(len(persons_hands)): + pid = persons_hands_ids[i] if i < len(persons_hands_ids) else i + if int(pid) % 2 == 0: + n_left += 1 + else: + n_right += 1 + try: + self._avbody.send_message("/hand/count", [int(n_left), int(n_right)]) + except OSError: + return + for i, hand in enumerate(persons_hands): + if not hand: + continue + pid = persons_hands_ids[i] if i < len(persons_hands_ids) else i + side = 1 if int(pid) % 2 else 0 + for idx, kp in enumerate(hand[:21]): + try: + self._avbody.send_message("/hand/kp", [ + int(pid), int(side), int(idx), + float(kp.x), float(kp.y), + float(getattr(kp, "z", 0.0)), + float(getattr(kp, "c", 1.0)), + ]) + except OSError: + return + + def send_body3d(self, persons_body3d: Sequence[Sequence[Any]], + persons_body3d_ids: Sequence[int], t_now: float, + force: bool = False) -> None: + """Public throttled entry point for 3D body world landmarks. + + Emits 33 MediaPipe pose_world_landmarks per person on + /pose3d/count + /pose3d/kp routes. Coordinates are in meters, + relative to the hip-center (MediaPipe convention: x=right, + y=down, z=forward from the camera). + """ + if not force and (t_now - self._last_t) < self._period: + return + self._send_body3d(persons_body3d, persons_body3d_ids) + + def _send_body3d(self, persons_body3d: Sequence[Sequence[Any]], + persons_body3d_ids: Sequence[int]) -> None: + n = len(persons_body3d) + try: + self._avbody.send_message("/pose3d/count", [int(n)]) + except OSError: + return + for i, body in enumerate(persons_body3d): + if not body: + continue + pid = persons_body3d_ids[i] if i < len(persons_body3d_ids) else i + for idx, kp in enumerate(body[:33]): + try: + self._avbody.send_message("/pose3d/kp", [ + int(pid), int(idx), + float(kp.x), float(kp.y), + float(getattr(kp, "z", 0.0)), + float(getattr(kp, "c", 1.0)), + ]) + except OSError: + return + + # ------------------------------------------------------------------ + def send_action(self, pid: int, label_idx: int, + probs, t_now: float, force: bool = False) -> None: + """Send action classification result via /pose/action OSC route. + + Sends: [pid (int), label_idx (int), prob_0 (float), prob_1 (float), prob_2 (float)] + """ + if not force and (t_now - self._last_t) < self._period: + return + p = [float(probs[0]), float(probs[1]), float(probs[2])] + self._client.send_message("/pose/action", [int(pid), int(label_idx), *p]) + + def send_kin(self, pid: int, kin, + t_now: float, force: bool = False) -> None: + """Send kinematic angles via /pose/kin OSC route. + + Sends: [pid (int), kin_0 (float), kin_1 (float), kin_2 (float)] + """ + if not force and (t_now - self._last_t) < self._period: + return + self._client.send_message( + "/pose/kin", + [int(pid), float(kin[0]), float(kin[1]), float(kin[2])], + ) + + def send_enter(self, pid: int) -> None: + """Send lifecycle event when person enters frame.""" + self._client.send_message("/pose/enter", [int(pid)]) + + def send_leave(self, pid: int) -> None: + """Send lifecycle event when person leaves frame.""" + self._client.send_message("/pose/leave", [int(pid)]) diff --git a/data_only_viz/scripts/convert_dinov2.py b/data_only_viz/scripts/convert_dinov2.py new file mode 100644 index 0000000..16ea5ae --- /dev/null +++ b/data_only_viz/scripts/convert_dinov2.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Convert DINOv2 ViT-S/14 to a CoreML .mlpackage for ANE-friendly inference. + +The wrapped module takes (1, 3, 224, 224) RGB float32 in [0, 1], applies +ImageNet normalization internally, runs the ViT, and returns the CLS +embedding (1, 384) L2-normalised. We trace + convert with +``coremltools.convert(... compute_units=ComputeUnit.ALL, compute_precision=FP16)``. + +Run with the Python 3.12 venv that has coremltools and torch:: + + /tmp/coreml312/bin/python -m data_only_viz.scripts.convert_dinov2 [--force] + +Output: + ~/.cache/av-live-multihmr/dinov2_vits14.mlpackage +""" +from __future__ import annotations + +import argparse +import logging +import sys +import time +from pathlib import Path + +import numpy as np + +LOG = logging.getLogger("convert_dinov2") + +OUT_DIR = Path.home() / ".cache" / "av-live-multihmr" +OUT_PATH = OUT_DIR / "dinov2_vits14.mlpackage" + +# ImageNet stats (DINOv2 expects these). +_IMAGENET_MEAN = (0.485, 0.456, 0.406) +_IMAGENET_STD = (0.229, 0.224, 0.225) + + +def _build_wrapper(): + import torch + import torch.nn as nn + + # Load DINOv2 ViT-S/14 from local torch.hub cache (offline ok). + backbone = torch.hub.load( + "facebookresearch/dinov2", + "dinov2_vits14", + source="github", + trust_repo=True, + ) + backbone.eval() + + # Monkey-patch interpolate_pos_encoding so we don't hit the bicubic + # upsample op (unsupported by coremltools). At fixed 224x224 input + # with patch=14 the layout is exactly 16x16 = 256 patches, which + # matches the pretrained pos_embed grid -> no interpolation needed. + def _no_interp_pos_encoding(self, x, w, h): + # Just return the stored pos_embed (cast to current dtype). + return self.pos_embed.to(x.dtype) + + import types + backbone.interpolate_pos_encoding = types.MethodType( + _no_interp_pos_encoding, backbone) + + mean = torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1) + std = torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1) + + class DinoV2Wrapper(nn.Module): + def __init__(self): + super().__init__() + self.backbone = backbone + self.register_buffer("mean", mean) + self.register_buffer("std", std) + + def forward(self, x: "torch.Tensor") -> "torch.Tensor": + # x: (1, 3, 224, 224) in [0, 1]. Normalise then forward. + x = (x - self.mean) / self.std + cls = self.backbone(x) # (1, 384) - CLS token by default + # L2 normalise so cosine sim = dot product. + cls = cls / (cls.norm(dim=-1, keepdim=True) + 1e-8) + return cls + + wrap = DinoV2Wrapper().eval() + return wrap + + +def convert(force: bool = False) -> Path: + import torch + import coremltools as ct + + OUT_DIR.mkdir(parents=True, exist_ok=True) + if OUT_PATH.exists() and not force: + LOG.info("already converted: %s", OUT_PATH) + return OUT_PATH + + LOG.info("loading DINOv2 ViT-S/14 from torch.hub ...") + wrap = _build_wrapper() + example = torch.rand(1, 3, 224, 224, dtype=torch.float32) + with torch.no_grad(): + ref_out = wrap(example) + LOG.info("torch out shape=%s norm=%.4f", tuple(ref_out.shape), + float(ref_out.norm(dim=-1).mean())) + + LOG.info("tracing ...") + with torch.no_grad(): + traced = torch.jit.trace(wrap, example, strict=False) + + LOG.info("ct.convert (mlprogram FP16, computeUnits=ALL) ...") + mlmodel = ct.convert( + traced, + source="pytorch", + convert_to="mlprogram", + inputs=[ct.TensorType(name="image", shape=example.shape, + dtype=np.float32)], + outputs=[ct.TensorType(name="embedding", dtype=np.float32)], + compute_precision=ct.precision.FLOAT16, + compute_units=ct.ComputeUnit.ALL, + minimum_deployment_target=ct.target.macOS14, + ) + mlmodel.short_description = "DINOv2 ViT-S/14 person re-id (384-D, L2)" + mlmodel.save(str(OUT_PATH)) + LOG.info("saved %s", OUT_PATH) + + # Sanity numerical check + pred = mlmodel.predict({"image": example.numpy().astype(np.float32)}) + coreml_out = list(pred.values())[0].reshape(-1) + ref_np = ref_out.numpy().reshape(-1) + cos = float(np.dot(coreml_out, ref_np) / + (np.linalg.norm(coreml_out) * np.linalg.norm(ref_np) + 1e-8)) + LOG.info("CoreML vs Torch cosine on random input: %.4f", cos) + return OUT_PATH + + +def bench(n_iter: int = 30) -> None: + import coremltools as ct + LOG.info("bench: load mlpackage ...") + m = ct.models.MLModel(str(OUT_PATH), + compute_units=ct.ComputeUnit.ALL) + crop = np.random.rand(1, 3, 224, 224).astype(np.float32) + # warmup + for _ in range(3): + m.predict({"image": crop}) + times = [] + for _ in range(n_iter): + t0 = time.perf_counter() + m.predict({"image": crop}) + times.append((time.perf_counter() - t0) * 1e3) + times.sort() + p50 = times[len(times) // 2] + p95 = times[int(len(times) * 0.95)] + LOG.info("bench %d iter: p50=%.2f ms p95=%.2f ms mean=%.2f ms (~%.1f fps)", + n_iter, p50, p95, sum(times) / len(times), 1000.0 / p50) + + +def main() -> int: + logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(name)s %(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--force", action="store_true") + ap.add_argument("--bench-only", action="store_true") + ap.add_argument("--n-iter", type=int, default=30) + args = ap.parse_args() + + if not args.bench_only: + convert(force=args.force) + bench(n_iter=args.n_iter) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/data_only_viz/scripts/extract_j3d_offline.py b/data_only_viz/scripts/extract_j3d_offline.py index 5482fdf..6fb5b09 100644 --- a/data_only_viz/scripts/extract_j3d_offline.py +++ b/data_only_viz/scripts/extract_j3d_offline.py @@ -1,12 +1,4 @@ -<<<<<<< HEAD -<<<<<<< HEAD """Extract j3d (32 SMPL-X joint anchors) from a recorded MP4 using the -======= -"""Extract j3d (22 SMPL-X joint anchors) from a recorded MP4 using the ->>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes) -======= -"""Extract j3d (32 SMPL-X joint anchors) from a recorded MP4 using the ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) Multi-HMR CoreML backend, write per-frame per-person jsonl rows. Usage: @@ -25,27 +17,12 @@ from pathlib import Path import cv2 import numpy as np -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD from data_only_viz.action_head import EXPR_DIM, HANDS_KP_DIMS, HANDS_KP_TOTAL -======= -from data_only_viz.action_head import EXPR_DIM ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= -from data_only_viz.action_head import EXPR_DIM, HANDS_KP_DIMS, HANDS_KP_TOTAL ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) from data_only_viz.action_head_pub import ( SMPLX_JOINT_ANCHOR_VERTS, SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT, ) -<<<<<<< HEAD -======= -from data_only_viz.action_head_pub import SMPLX_JOINT_ANCHOR_VERTS ->>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes) -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend LOG = logging.getLogger("extract_j3d_offline") @@ -75,21 +52,11 @@ def _frame_to_chw(frame_bgr: np.ndarray, size: int = IMG_SIZE) -> np.ndarray: return rgb.transpose(2, 0, 1) # CHW -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) def _person_to_j3d32( person: dict, anchors: tuple[int, ...], ) -> tuple[np.ndarray, np.ndarray, float] | None: """Return (j3d32, expression, mouth_open) or None if v3d absent/too small.""" -<<<<<<< HEAD -======= -def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | None: ->>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes) -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) v3d = person.get("v3d") if v3d is None: return None @@ -99,10 +66,6 @@ def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | Non v3d_np = np.asarray(v3d, dtype=np.float32) if v3d_np.shape[0] < max(anchors) + 1: return None -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) j3d32 = v3d_np[list(anchors)].astype(np.float32) # expression expr = person.get("expression") @@ -120,12 +83,6 @@ def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | Non else: mouth = 0.0 return j3d32, expr_np, mouth -<<<<<<< HEAD -======= - return v3d_np[list(anchors)].astype(np.float32) ->>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes) -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) def extract(session: str, video: Path, out: Path, @@ -157,46 +114,20 @@ def extract(session: str, video: Path, out: Path, continue ts = n_frames / fps for i, person in enumerate(persons): -<<<<<<< HEAD -<<<<<<< HEAD result = _person_to_j3d32(person, anchors) if result is None: continue j3d32, expr_np, mouth = result -======= - j3d = _person_to_j3d22(person, anchors) - if j3d is None: - continue ->>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes) -======= - result = _person_to_j3d32(person, anchors) - if result is None: - continue - j3d32, expr_np, mouth = result ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) f.write(json.dumps({ "ts": ts, "session": session, "pid": int(person.get("pid", i)), -<<<<<<< HEAD -<<<<<<< HEAD "j3d": j3d32.tolist(), "expression": expr_np.tolist(), "mouth_open": mouth, "hands_kp": np.zeros( (HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32 ).tolist(), -<<<<<<< HEAD -======= - "j3d": j3d.tolist(), ->>>>>>> 2a732fa (fix(data-only-viz): action-head review fixes) -======= - "j3d": j3d32.tolist(), - "expression": expr_np.tolist(), - "mouth_open": mouth, ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) }) + "\n") n_rows += 1 n_frames += 1 diff --git a/data_only_viz/state.py b/data_only_viz/state.py index e921f78..54523ac 100644 --- a/data_only_viz/state.py +++ b/data_only_viz/state.py @@ -21,6 +21,16 @@ class PoseKp: c: float = 0.0 +@dataclass +class Kp3D: + """3D keypoint in metric coordinates relative to hip-center. + Used for MediaPipe pose_world_landmarks (xyz in meters).""" + x: float = 0.0 + y: float = 0.0 + z: float = 0.0 + c: float = 0.0 + + @dataclass class SMPLXPerson: """Resultats Multi-HMR pour une personne : params SMPL-X + vertices @@ -92,6 +102,10 @@ class State: persons_body: list[list[PoseKp]] = field(default_factory=list) persons_face: list[list[PoseKp]] = field(default_factory=list) persons_hands: list[list[PoseKp]] = field(default_factory=list) + # MediaPipe pose_world_landmarks per person : 33 keypoints in meters, + # relative to the hip-center. Optional companion of persons_body + # (image-space xy). Empty if no detection or backend doesn't emit it. + persons_body3d: list[list[Kp3D]] = field(default_factory=list) # IDs persistants entre frames (ByteTrack-like via Hungarian IoU). # Couleur du skeleton dans le shader Metal = ID % palette_size. persons_body_ids: list[int] = field(default_factory=list) diff --git a/data_only_viz/tests/test_action_head_features.py b/data_only_viz/tests/test_action_head_features.py index 08d333d..b6df78d 100644 --- a/data_only_viz/tests/test_action_head_features.py +++ b/data_only_viz/tests/test_action_head_features.py @@ -12,19 +12,9 @@ def test_module_imports() -> None: assert hasattr(action_head, "ActionHead") assert action_head.WINDOW_LEN == 16 assert action_head.J3D_JOINTS == 32 -<<<<<<< HEAD -<<<<<<< HEAD assert action_head.FEATURE_DIM == 428 assert action_head.HANDS_KP_TOTAL == 42 assert action_head.HANDS_KP_FLAT == 126 -======= - assert action_head.FEATURE_DIM == 302 ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - assert action_head.FEATURE_DIM == 428 - assert action_head.HANDS_KP_TOTAL == 42 - assert action_head.HANDS_KP_FLAT == 126 ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) assert action_head.NUM_CLASSES == 3 assert action_head.LABELS == ("debout", "assise", "danse") @@ -79,15 +69,7 @@ def test_feature_extractor_shape_full_buffer() -> None: from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, FEATURE_DIM frames = [_rand_j3d(i) for i in range(WINDOW_LEN)] feat = FeatureExtractor.from_buffer(frames) -<<<<<<< HEAD -<<<<<<< HEAD assert feat.shape == (428,) -======= - assert feat.shape == (302,) ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - assert feat.shape == (428,) ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) assert feat.dtype == np.float32 assert not np.isnan(feat).any() diff --git a/data_only_viz/tests/test_action_head_model.py b/data_only_viz/tests/test_action_head_model.py index ab0e558..939d8c4 100644 --- a/data_only_viz/tests/test_action_head_model.py +++ b/data_only_viz/tests/test_action_head_model.py @@ -24,27 +24,11 @@ def test_model_forward_shape() -> None: assert h_new.shape == h.shape -<<<<<<< HEAD -<<<<<<< HEAD def test_model_param_count_under_100k() -> None: from data_only_viz.action_head import ActionHeadModel model = ActionHeadModel() n = sum(p.numel() for p in model.parameters()) assert n < 100_000, f"too many params: {n}" -======= -def test_model_param_count_under_80k() -> None: - from data_only_viz.action_head import ActionHeadModel - model = ActionHeadModel() - n = sum(p.numel() for p in model.parameters()) - assert n < 80_000, f"too many params: {n}" ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= -def test_model_param_count_under_100k() -> None: - from data_only_viz.action_head import ActionHeadModel - model = ActionHeadModel() - n = sum(p.numel() for p in model.parameters()) - assert n < 100_000, f"too many params: {n}" ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) def test_action_head_step_warmup_returns_debout() -> None: diff --git a/data_only_viz/tests/test_action_head_pub.py b/data_only_viz/tests/test_action_head_pub.py index 05d6444..540e5ad 100644 --- a/data_only_viz/tests/test_action_head_pub.py +++ b/data_only_viz/tests/test_action_head_pub.py @@ -19,16 +19,8 @@ class _FakeState: self.pose_last_t = 0.0 self.persons_hands = [] self.persons_hands_ids = [] -<<<<<<< HEAD -<<<<<<< HEAD self.persons_face = [] self.persons_face_ids = [] -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - self.persons_face = [] - self.persons_face_ids = [] ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) self._lock = threading.RLock() def lock(self): diff --git a/data_only_viz/tests/test_multihmr_coreml.py b/data_only_viz/tests/test_multihmr_coreml.py new file mode 100644 index 0000000..2eb7338 --- /dev/null +++ b/data_only_viz/tests/test_multihmr_coreml.py @@ -0,0 +1,96 @@ +"""Tests for the Multi-HMR CoreML backend. + +Skipped unless the .mlpackage exists at the standard cache path. +""" +from __future__ import annotations + +import time +from pathlib import Path + +import numpy as np +import pytest + +MLPACKAGE = ( + Path.home() / ".cache" / "av-live-multihmr" + / "multihmr_full_672_s.mlpackage" +) + +pytestmark = pytest.mark.skipif( + not MLPACKAGE.exists(), + reason=f"mlpackage missing at {MLPACKAGE}", +) + + +def _make_K() -> np.ndarray: + f = 672.0 + return np.array([[f, 0.0, 336.0], + [0.0, f, 336.0], + [0.0, 0.0, 1.0]], dtype=np.float32) + + +def test_is_available_true(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + assert MultiHMRCoreMLBackend.is_available(MLPACKAGE) is True + + +def test_load_model(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(MLPACKAGE) + assert backend._model is not None + + +def test_infer_random_image_shapes(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(MLPACKAGE) + rng = np.random.default_rng(0) + img = rng.random((3, 672, 672), dtype=np.float32) + K = _make_K() + # threshold = -inf so we get all K=4 humans back + humans = backend.infer(img, K, det_thresh=-1.0) + assert len(humans) == 4 + for h in humans: + v = h["v3d"].detach().cpu().numpy() + assert v.shape == (10475, 3) + assert v.dtype == np.float32 + t = h["transl_pelvis"].detach().cpu().numpy() + assert t.shape == (1, 3) + s = float(h["scores"].item()) + assert isinstance(s, float) + beta = h["shape"].detach().cpu().numpy() + assert beta.shape == (10,) + expr = h["expression"].detach().cpu().numpy() + assert expr.shape == (10,) + + +def test_infer_latency_under_target(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(MLPACKAGE) + K = _make_K() + rng = np.random.default_rng(42) + img = rng.random((3, 672, 672), dtype=np.float32) + # warmup + backend.infer(img, K, det_thresh=-1.0) + # measure + n = 5 + times = [] + for _ in range(n): + t0 = time.monotonic() + backend.infer(img, K, det_thresh=-1.0) + times.append((time.monotonic() - t0) * 1e3) + times.sort() + median_ms = times[n // 2] + print(f"median latency: {median_ms:.1f} ms (n={n})") + # Target 50ms = 20fps. M5 bench shows ~29ms. Generous margin. + assert median_ms < 80.0, f"median {median_ms:.1f}ms > 80ms target" + + +def test_filter_threshold(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(MLPACKAGE) + rng = np.random.default_rng(0) + img = rng.random((3, 672, 672), dtype=np.float32) + K = _make_K() + high = backend.infer(img, K, det_thresh=999.0) + assert high == [] # nothing passes + low = backend.infer(img, K, det_thresh=-1.0) + assert len(low) == 4 diff --git a/data_only_viz/tests/test_pose_bridge_action.py b/data_only_viz/tests/test_pose_bridge_action.py new file mode 100644 index 0000000..c8cd185 --- /dev/null +++ b/data_only_viz/tests/test_pose_bridge_action.py @@ -0,0 +1,49 @@ +"""Tests for /pose/action and /pose/kin OSC routes.""" +from __future__ import annotations + +from unittest.mock import MagicMock + +import numpy as np + + +def test_send_action_formats_5_args() -> None: + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b.send_action(pid=7, label_idx=2, + probs=np.array([0.1, 0.2, 0.7], dtype=np.float32), + t_now=0.0, force=True) + b._client.send_message.assert_called_once() + address, args = b._client.send_message.call_args.args + assert address == "/pose/action" + assert args[0] == 7 + assert args[1] == 2 + assert all(isinstance(v, float) for v in args[2:5]) + assert abs(sum(args[2:5]) - 1.0) < 1e-5 + + +def test_send_kin_formats_4_args() -> None: + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b.send_kin(pid=3, kin=np.array([0.5, 1.2, -0.3], dtype=np.float32), + t_now=0.0, force=True) + b._client.send_message.assert_called_once() + address, args = b._client.send_message.call_args.args + assert address == "/pose/kin" + assert args[0] == 3 + assert len(args) == 4 + assert abs(args[1] - 0.5) < 1e-6 + assert abs(args[2] - 1.2) < 1e-6 + assert abs(args[3] - (-0.3)) < 1e-6 + + +def test_send_lifecycle_enter_leave() -> None: + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b.send_enter(pid=4) + b.send_leave(pid=4) + calls = [c.args[0] for c in b._client.send_message.call_args_list] + assert "/pose/enter" in calls + assert "/pose/leave" in calls diff --git a/data_only_viz/tests/test_pose_bridge_body3d.py b/data_only_viz/tests/test_pose_bridge_body3d.py new file mode 100644 index 0000000..9a09e18 --- /dev/null +++ b/data_only_viz/tests/test_pose_bridge_body3d.py @@ -0,0 +1,79 @@ +"""Tests for /pose3d/* OSC routes emitted to AVLiveBody.""" +from __future__ import annotations + +from dataclasses import dataclass +from unittest.mock import MagicMock + + +@dataclass +class _Kp3D: + x: float = 0.0 + y: float = 0.0 + z: float = 0.0 + c: float = 1.0 + + +def _make_bridge(): + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b._avbody = MagicMock() + return b + + +def test_send_body3d_emits_count_and_33_kp() -> None: + b = _make_bridge() + # One person, 33 keypoints with deterministic xyz. + body = [_Kp3D(x=0.01 * i, y=-0.02 * i, z=0.03 * i, c=1.0) + for i in range(33)] + b.send_body3d([body], [7], t_now=0.0, force=True) + + calls = b._avbody.send_message.call_args_list + assert calls[0].args == ("/pose3d/count", [1]) + kp_calls = [c for c in calls if c.args[0] == "/pose3d/kp"] + assert len(kp_calls) == 33 + # Format : [pid, idx, x, y, z, c] + first = kp_calls[0].args[1] + assert first[0] == 7 + assert first[1] == 0 + assert abs(first[2] - 0.0) < 1e-6 + assert abs(first[4] - 0.0) < 1e-6 + # idx ordering strictly 0..32 + idxs = [c.args[1][1] for c in kp_calls] + assert idxs == list(range(33)) + # Last kp z should be 0.03 * 32 + last = kp_calls[-1].args[1] + assert abs(last[4] - 0.03 * 32) < 1e-6 + + +def test_send_body3d_empty_emits_count_zero() -> None: + b = _make_bridge() + b.send_body3d([], [], t_now=0.0, force=True) + calls = b._avbody.send_message.call_args_list + assert len(calls) == 1 + assert calls[0].args == ("/pose3d/count", [0]) + + +def test_send_body3d_multi_person() -> None: + b = _make_bridge() + body_a = [_Kp3D(x=1.0) for _ in range(33)] + body_b = [_Kp3D(x=2.0) for _ in range(33)] + b.send_body3d([body_a, body_b], [10, 11], t_now=0.0, force=True) + calls = b._avbody.send_message.call_args_list + assert calls[0].args == ("/pose3d/count", [2]) + kp_calls = [c for c in calls if c.args[0] == "/pose3d/kp"] + assert len(kp_calls) == 66 + assert kp_calls[0].args[1][0] == 10 + assert kp_calls[33].args[1][0] == 11 + + +def test_send_with_body3d_kwargs_dispatches() -> None: + """Top-level send() routes body3d kwargs to /pose3d.""" + b = _make_bridge() + body = [_Kp3D(x=0.5, y=0.5, c=1.0) for _ in range(33)] + body3d = [_Kp3D(x=0.1, y=0.2, z=0.3) for _ in range(33)] + b.send([body], [0], 1.0, + persons_body3d=[body3d], persons_body3d_ids=[0]) + av_addrs = {c.args[0] for c in b._avbody.send_message.call_args_list} + assert "/pose3d/count" in av_addrs + assert "/pose3d/kp" in av_addrs diff --git a/data_only_viz/tests/test_pose_bridge_face_hand.py b/data_only_viz/tests/test_pose_bridge_face_hand.py new file mode 100644 index 0000000..3add1e5 --- /dev/null +++ b/data_only_viz/tests/test_pose_bridge_face_hand.py @@ -0,0 +1,105 @@ +"""Tests for /face/* and /hand/* OSC routes emitted to AVLiveBody.""" +from __future__ import annotations + +from dataclasses import dataclass +from unittest.mock import MagicMock + + +@dataclass +class _Kp: + x: float = 0.0 + y: float = 0.0 + z: float = 0.0 + c: float = 1.0 + + +def _make_bridge(): + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b._avbody = MagicMock() + return b + + +def test_send_face_emits_count_and_68_kp() -> None: + from data_only_viz.pose_bridge import FACE_68_FROM_MP + b = _make_bridge() + # One face with 478 landmarks at deterministic coords. + face = [_Kp(x=i / 478.0, y=1.0 - i / 478.0, z=0.01 * i, c=1.0) + for i in range(478)] + b.send_face([face], [3], t_now=0.0, force=True) + + calls = b._avbody.send_message.call_args_list + # First call : /face/count <1> + assert calls[0].args[0] == "/face/count" + assert calls[0].args[1] == [1] + # Then 68 /face/kp messages + kp_calls = [c for c in calls if c.args[0] == "/face/kp"] + assert len(kp_calls) == 68 + # Format : [pid, slot, x, y, z, c] + first = kp_calls[0].args[1] + assert first[0] == 3 + assert first[1] == 0 + assert isinstance(first[2], float) + assert isinstance(first[3], float) + assert isinstance(first[4], float) + assert isinstance(first[5], float) + # Slot ordering is strictly increasing 0..67 + slots = [c.args[1][1] for c in kp_calls] + assert slots == list(range(68)) + # And the x coord of slot 0 matches mp_idx FACE_68_FROM_MP[0] + mp0 = FACE_68_FROM_MP[0] + assert abs(first[2] - mp0 / 478.0) < 1e-6 + + +def test_send_face_empty_emits_count_zero() -> None: + b = _make_bridge() + b.send_face([], [], t_now=0.0, force=True) + calls = b._avbody.send_message.call_args_list + assert len(calls) == 1 + assert calls[0].args == ("/face/count", [0]) + + +def test_send_hand_emits_count_and_21_kp() -> None: + b = _make_bridge() + hand_l = [_Kp(x=0.1, y=0.2, z=0.0, c=1.0) for _ in range(21)] + hand_r = [_Kp(x=0.7, y=0.3, z=0.0, c=1.0) for _ in range(21)] + # pid=2 -> left (even), pid=3 -> right (odd) + b.send_hand([hand_l, hand_r], [2, 3], t_now=0.0, force=True) + calls = b._avbody.send_message.call_args_list + assert calls[0].args == ("/hand/count", [1, 1]) + kp_calls = [c for c in calls if c.args[0] == "/hand/kp"] + assert len(kp_calls) == 42 # 21 * 2 + # First hand should be side=0 (left) + assert kp_calls[0].args[1][1] == 0 + # 22nd kp call : start of right hand, side=1 + assert kp_calls[21].args[1][1] == 1 + + +def test_send_throttles_below_period() -> None: + """send_face called twice within < period emits only once.""" + b = _make_bridge() + face = [_Kp(x=0.0, y=0.0) for _ in range(478)] + b.send_face([face], [0], t_now=0.0, force=True) + n_first = b._avbody.send_message.call_count + # Second call without force AND inside throttle window : skipped. + b._last_t = 9999.0 # pretend a body send just happened + b.send_face([face], [0], t_now=9999.0 + 0.001, force=False) + assert b._avbody.send_message.call_count == n_first + + +def test_send_with_face_hand_kwargs_dispatches() -> None: + """Top-level send() routes face/hand kwargs to /face and /hand.""" + b = _make_bridge() + body = [_Kp(x=0.5, y=0.5, c=1.0) for _ in range(33)] + face = [_Kp(x=0.1, y=0.1, c=1.0) for _ in range(478)] + hand = [_Kp(x=0.2, y=0.2, c=1.0) for _ in range(21)] + b.send([body], [0], 1.0, + persons_face=[face], persons_face_ids=[0], + persons_hands=[hand], persons_hands_ids=[1]) + av_addrs = {c.args[0] for c in b._avbody.send_message.call_args_list} + assert "/pose/count" in av_addrs + assert "/face/count" in av_addrs + assert "/face/kp" in av_addrs + assert "/hand/count" in av_addrs + assert "/hand/kp" in av_addrs diff --git a/data_only_viz/training/dataset.py b/data_only_viz/training/dataset.py index 6e4c9af..27f2ecd 100644 --- a/data_only_viz/training/dataset.py +++ b/data_only_viz/training/dataset.py @@ -15,23 +15,10 @@ class RawFrame: ts: float session: str pid: int -<<<<<<< HEAD -<<<<<<< HEAD j3d: np.ndarray # (32, 3) float32 (v3: body22 + 10 fingertips) expression: np.ndarray | None = None # (EXPR_DIM,) or None mouth_open: float = 0.0 hands_kp: np.ndarray | None = None # (42, 3) or None -======= - j3d: np.ndarray # (32, 3) float32 (v2: body22 + 10 fingertips) - expression: np.ndarray | None = None # (EXPR_DIM,) or None - mouth_open: float = 0.0 ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - j3d: np.ndarray # (32, 3) float32 (v3: body22 + 10 fingertips) - expression: np.ndarray | None = None # (EXPR_DIM,) or None - mouth_open: float = 0.0 - hands_kp: np.ndarray | None = None # (42, 3) or None ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) @dataclass @@ -42,14 +29,7 @@ class WindowRow: first_ts: float expr_stack: np.ndarray | None = None # (window_len, 10) or None mouth_open_stack: np.ndarray | None = None # (window_len,) or None -<<<<<<< HEAD -<<<<<<< HEAD hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) @dataclass @@ -63,14 +43,7 @@ class DatasetRow: manually_validated: bool expr_stack: np.ndarray | None = None # (window_len, 10) or None mouth_open_stack: np.ndarray | None = None # (window_len,) or None -<<<<<<< HEAD -<<<<<<< HEAD hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) def load_frames_jsonl(path: Path) -> list[RawFrame]: @@ -83,16 +56,8 @@ def load_frames_jsonl(path: Path) -> list[RawFrame]: d = json.loads(line) expr_raw = d.get("expression") expr = np.asarray(expr_raw, dtype=np.float32) if expr_raw is not None else None -<<<<<<< HEAD -<<<<<<< HEAD hands_raw = d.get("hands_kp") hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - hands_raw = d.get("hands_kp") - hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) rows.append(RawFrame( ts=float(d["ts"]), session=str(d["session"]), @@ -100,14 +65,7 @@ def load_frames_jsonl(path: Path) -> list[RawFrame]: j3d=np.asarray(d["j3d"], dtype=np.float32), expression=expr, mouth_open=float(d.get("mouth_open", 0.0)), -<<<<<<< HEAD -<<<<<<< HEAD hands_kp=hands_kp, -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - hands_kp=hands_kp, ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) )) return rows @@ -142,10 +100,6 @@ def sliding_windows(frames: list[RawFrame], mouth_stack = np.array( [c.mouth_open for c in chunk], dtype=np.float32 ) -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # hands_kp stack: (window_len, 42, 3) if any frame has hands_kp if any(c.hands_kp is not None for c in chunk): hands_kp_stack = np.zeros((window_len, 42, 3), dtype=np.float32) @@ -156,25 +110,11 @@ def sliding_windows(frames: list[RawFrame], hands_kp_stack[t] = hk else: hands_kp_stack = None -<<<<<<< HEAD yield WindowRow(j3d_stack=stack, session=sess, pid_local=pid, first_ts=chunk[0].ts, expr_stack=expr_stack, mouth_open_stack=mouth_stack, hands_kp_stack=hands_kp_stack) -======= - yield WindowRow(j3d_stack=stack, session=sess, - pid_local=pid, first_ts=chunk[0].ts, - expr_stack=expr_stack, - mouth_open_stack=mouth_stack) ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - yield WindowRow(j3d_stack=stack, session=sess, - pid_local=pid, first_ts=chunk[0].ts, - expr_stack=expr_stack, - mouth_open_stack=mouth_stack, - hands_kp_stack=hands_kp_stack) ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None: @@ -193,16 +133,8 @@ def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None: d["expr_stack"] = r.expr_stack.astype(np.float32).tolist() if r.mouth_open_stack is not None: d["mouth_open_stack"] = r.mouth_open_stack.astype(np.float32).tolist() -<<<<<<< HEAD -<<<<<<< HEAD if r.hands_kp_stack is not None: d["hands_kp_stack"] = r.hands_kp_stack.astype(np.float32).tolist() -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - if r.hands_kp_stack is not None: - d["hands_kp_stack"] = r.hands_kp_stack.astype(np.float32).tolist() ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) f.write(json.dumps(d) + "\n") @@ -218,16 +150,8 @@ def load_dataset_jsonl(path: Path) -> list[DatasetRow]: expr = np.asarray(expr_raw, dtype=np.float32) if expr_raw is not None else None mouth_raw = d.get("mouth_open_stack") mouth = np.asarray(mouth_raw, dtype=np.float32) if mouth_raw is not None else None -<<<<<<< HEAD -<<<<<<< HEAD hands_raw = d.get("hands_kp_stack") hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - hands_raw = d.get("hands_kp_stack") - hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) out.append(DatasetRow( window_id=d["window_id"], label=d["label"], @@ -238,14 +162,7 @@ def load_dataset_jsonl(path: Path) -> list[DatasetRow]: manually_validated=bool(d["manually_validated"]), expr_stack=expr, mouth_open_stack=mouth, -<<<<<<< HEAD -<<<<<<< HEAD hands_kp_stack=hands_kp, -======= ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= - hands_kp_stack=hands_kp, ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) )) return out diff --git a/data_only_viz/training/train_action_head.py b/data_only_viz/training/train_action_head.py index e8a9685..fd4ea53 100644 --- a/data_only_viz/training/train_action_head.py +++ b/data_only_viz/training/train_action_head.py @@ -79,10 +79,6 @@ class WindowDataset(Dataset[tuple[torch.Tensor, int]]): n = min(EXPR_DIM, len(expr_t)) expr_vec[:n] = expr_t[:n] mouth_t = float(mouth_s[t]) if t < len(mouth_s) else 0.0 -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) # hands_kp at frame t (42, 3); zeros if row has none if row.hands_kp_stack is not None: hands_t = row.hands_kp_stack[t] @@ -92,13 +88,6 @@ class WindowDataset(Dataset[tuple[torch.Tensor, int]]): feat = np.concatenate([ cur.reshape(-1), vel.reshape(-1), accel.reshape(-1), hands_flat, -<<<<<<< HEAD -======= - feat = np.concatenate([ - cur.reshape(-1), vel.reshape(-1), accel.reshape(-1), ->>>>>>> aedcb0f (feat(data-only-viz): action-head v2 fingers+face) -======= ->>>>>>> beb94d2 (feat(data-only-viz): action-head v3 hands+lips) expr_vec, np.array([hip_y, knee_angle, sym, mouth_t], dtype=np.float32), ]).astype(np.float32, copy=False) diff --git a/data_only_viz/uv.lock b/data_only_viz/uv.lock index b5b3d0f..fa66763 100644 --- a/data_only_viz/uv.lock +++ b/data_only_viz/uv.lock @@ -21,15 +21,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750, upload-time = "2026-01-28T10:17:04.19Z" }, ] -[[package]] -name = "addict" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/85/ef/fd7649da8af11d93979831e8f1f8097e85e82d5bfeabc8c68b39175d8e75/addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494", size = 9186, upload-time = "2020-11-21T16:21:31.416Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/00/b08f23b7d7e1e14ce01419a467b583edbb93c6cdb8654e54a9cc579cd61f/addict-2.4.0-py3-none-any.whl", hash = "sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc", size = 3832, upload-time = "2020-11-21T16:21:29.588Z" }, -] - [[package]] name = "annotated-doc" version = "0.0.4" @@ -124,21 +115,6 @@ pose = [ { name = "opencv-python" }, { name = "ultralytics" }, ] -smplerx = [ - { name = "einops" }, - { name = "mmcv-lite" }, - { name = "numpy" }, - { name = "opencv-python" }, - { name = "pillow" }, - { name = "scipy" }, - { name = "smplx" }, - { name = "timm" }, - { name = "torch" }, - { name = "torchvision" }, - { name = "tqdm" }, - { name = "ultralytics" }, - { name = "yacs" }, -] [package.dev-dependencies] dev = [ @@ -150,24 +126,19 @@ requires-dist = [ { name = "cloudpickle", marker = "extra == 'detrpose'", specifier = ">=3.0" }, { name = "coremltools", marker = "extra == 'pose'", specifier = ">=9.0" }, { name = "einops", marker = "extra == 'multihmr'", specifier = ">=0.8" }, - { name = "einops", marker = "extra == 'smplerx'", specifier = ">=0.8" }, { name = "huggingface-hub", marker = "extra == 'multihmr'", specifier = ">=0.24" }, { name = "iopath", marker = "extra == 'detrpose'", specifier = ">=0.1.10" }, { name = "iopath", marker = "extra == 'multihmr'", specifier = ">=0.1.10" }, { name = "mediapipe", marker = "extra == 'pose'", specifier = ">=0.10.35" }, - { name = "mmcv-lite", marker = "extra == 'smplerx'", specifier = ">=2.1" }, { name = "numpy", specifier = ">=1.26,<2" }, { name = "numpy", marker = "extra == 'multihmr'", specifier = ">=1.26,<2" }, { name = "numpy", marker = "extra == 'nlf'", specifier = ">=1.26" }, - { name = "numpy", marker = "extra == 'smplerx'", specifier = ">=1.26,<2" }, { name = "omegaconf", marker = "extra == 'detrpose'", specifier = ">=2.3" }, { name = "opencv-python", marker = "extra == 'detrpose'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'multihmr'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'nlf'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'pose'", specifier = ">=4.10" }, - { name = "opencv-python", marker = "extra == 'smplerx'", specifier = ">=4.10" }, { name = "pillow", marker = "extra == 'multihmr'", specifier = ">=10.0" }, - { name = "pillow", marker = "extra == 'smplerx'", specifier = ">=10.0" }, { name = "pycocotools", marker = "extra == 'detrpose'", specifier = ">=2.0" }, { name = "pyobjc-core", specifier = ">=10.3" }, { name = "pyobjc-framework-avfoundation", specifier = ">=10.3" }, @@ -180,29 +151,21 @@ requires-dist = [ { name = "scipy", specifier = ">=1.13" }, { name = "scipy", marker = "extra == 'detrpose'", specifier = ">=1.13" }, { name = "scipy", marker = "extra == 'multihmr'", specifier = ">=1.13" }, - { name = "scipy", marker = "extra == 'smplerx'", specifier = ">=1.13" }, { name = "smplx", marker = "extra == 'multihmr'", specifier = ">=0.1.28" }, - { name = "smplx", marker = "extra == 'smplerx'", specifier = ">=0.1.28" }, - { name = "timm", marker = "extra == 'smplerx'", specifier = ">=1.0" }, { name = "torch", marker = "extra == 'detrpose'", specifier = ">=2.4" }, { name = "torch", marker = "extra == 'multihmr'", specifier = ">=2.4" }, { name = "torch", marker = "extra == 'nlf'", specifier = ">=2.4" }, - { name = "torch", marker = "extra == 'smplerx'", specifier = ">=2.4" }, { name = "torchgeometry", marker = "extra == 'multihmr'", specifier = ">=0.1.2" }, { name = "torchvision", marker = "extra == 'detrpose'", specifier = ">=0.19" }, { name = "torchvision", marker = "extra == 'multihmr'", specifier = ">=0.19" }, { name = "torchvision", marker = "extra == 'nlf'", specifier = ">=0.19" }, - { name = "torchvision", marker = "extra == 'smplerx'", specifier = ">=0.19" }, { name = "tqdm", marker = "extra == 'multihmr'", specifier = ">=4.65" }, - { name = "tqdm", marker = "extra == 'smplerx'", specifier = ">=4.65" }, { name = "transformers", marker = "extra == 'detrpose'", specifier = ">=4.40" }, { name = "trimesh", marker = "extra == 'multihmr'", specifier = ">=4.4" }, { name = "ultralytics", marker = "extra == 'pose'", specifier = ">=8.3" }, - { name = "ultralytics", marker = "extra == 'smplerx'", specifier = ">=8.3" }, { name = "xtcocotools", marker = "extra == 'detrpose'", specifier = ">=1.14" }, - { name = "yacs", marker = "extra == 'smplerx'", specifier = ">=0.1.8" }, ] -provides-extras = ["pose", "detrpose", "nlf", "multihmr", "smplerx"] +provides-extras = ["pose", "detrpose", "nlf", "multihmr"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.3" }] @@ -1140,46 +1103,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/b3/5c7fa594c731e8dafab9f1a46ab6cef670fa62dbbfb6248cc70e42ec6fc5/mediapipe-0.10.35-py3-none-win_arm64.whl", hash = "sha256:46255326a6213118aaa518a7aa25e35f93337e82677960cc2a945f117bff8444", size = 9992331, upload-time = "2026-04-27T17:45:36.193Z" }, ] -[[package]] -name = "mmcv-lite" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "addict" }, - { name = "mmengine" }, - { name = "numpy" }, - { name = "opencv-python" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyyaml" }, - { name = "regex", marker = "sys_platform == 'win32'" }, - { name = "yapf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8c/e7/075329ead4078d77b14e195f04b03162b99c5644d4186505113f62b7ca6c/mmcv-lite-2.2.0.tar.gz", hash = "sha256:62933ea165b2d9ad32e1b72ccd5ccba3cf71b5cd812c4c13c16cb2fcfc46a064", size = 479155, upload-time = "2024-04-24T14:24:38.98Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/59/bd/5d468c171f201d6169bec848d1116e95736854eec72307299bc68939fe45/mmcv_lite-2.2.0-py2.py3-none-any.whl", hash = "sha256:a24ee8dd3df7556dfced282dbfe8c3f87df6de2d4dcaf1207e83e9a2d58455a6", size = 732333, upload-time = "2024-04-24T14:24:28.555Z" }, -] - -[[package]] -name = "mmengine" -version = "0.10.7" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "addict" }, - { name = "matplotlib" }, - { name = "numpy" }, - { name = "opencv-python" }, - { name = "pyyaml" }, - { name = "regex", marker = "sys_platform == 'win32'" }, - { name = "rich" }, - { name = "termcolor" }, - { name = "yapf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/17/14/959360bbd8374e23fc1b720906999add16a3ac071a501636db12c5861ff5/mmengine-0.10.7.tar.gz", hash = "sha256:d20ffcc31127567e53dceff132612a87f0081de06cbb7ab2bdb7439125a69225", size = 378090, upload-time = "2025-03-04T12:23:09.568Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/98/8e/f98332248aad102511bea4ae19c0ddacd2f0a994f3ca4c82b7a369e0af8b/mmengine-0.10.7-py3-none-any.whl", hash = "sha256:262ac976a925562f78cd5fd14dd1bc9b680ed0aa81f0d85b723ef782f99c54ee", size = 452720, upload-time = "2025-03-04T12:23:06.339Z" }, -] - [[package]] name = "mpmath" version = "1.3.0" @@ -1514,15 +1437,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/60/5382c03e1970de634027cee8e1b7d39776b778b81812aaf45b694dfe9e28/pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e", size = 7080946, upload-time = "2026-04-01T14:46:11.734Z" }, ] -[[package]] -name = "platformdirs" -version = "4.9.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, -] - [[package]] name = "pluggy" version = "1.6.0" @@ -2249,31 +2163,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] -[[package]] -name = "termcolor" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, -] - -[[package]] -name = "timm" -version = "1.0.27" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "huggingface-hub" }, - { name = "pyyaml" }, - { name = "safetensors" }, - { name = "torch" }, - { name = "torchvision" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/08/54/ece85b0eef3700c90db8271a43669b05a0ebbe2edb1962329c34374a297e/timm-1.0.27.tar.gz", hash = "sha256:315dfe63186ca9fb7ff941268941231fd5be259f2b4bb4afa28560ae1015cb9a", size = 2439861, upload-time = "2026-05-08T19:38:36.844Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/2e/26bab7686ff4aed48f8f5f6c23e2aa37b7a37ddd9effe3aa61e908fd518f/timm-1.0.27-py3-none-any.whl", hash = "sha256:5ff07c9ddf53cbada88eab1c93ff175c64cab683b5a2fddf863bcee985926f89", size = 2589280, upload-time = "2026-05-08T19:38:35.034Z" }, -] - [[package]] name = "tokenizers" version = "0.22.2" @@ -2543,27 +2432,3 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/e0/01cf7f8b3f4229568b37de680d0eaadc651d5ee36bd483b83658f49dc6c2/xtcocotools-1.14.3-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:126ca596229b2016552bf27cad01f3a2f70a3ff7576a58305a00499cb9e0057d", size = 464351, upload-time = "2023-10-19T07:52:33.199Z" }, { url = "https://files.pythonhosted.org/packages/a9/10/32bef0fcd29145dcda9bfaa9e11718f40acd444d6804cac870b0437fc7a8/xtcocotools-1.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:47cb5433903f30589343d54530e49abd6b61d0fd119857ba4948b8ce291dbee6", size = 88741, upload-time = "2023-10-19T07:53:51.151Z" }, ] - -[[package]] -name = "yacs" -version = "0.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/44/3e/4a45cb0738da6565f134c01d82ba291c746551b5bc82e781ec876eb20909/yacs-0.1.8.tar.gz", hash = "sha256:efc4c732942b3103bea904ee89af98bcd27d01f0ac12d8d4d369f1e7a2914384", size = 11100, upload-time = "2020-08-10T16:37:47.755Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/4f/fe9a4d472aa867878ce3bb7efb16654c5d63672b86dc0e6e953a67018433/yacs-0.1.8-py3-none-any.whl", hash = "sha256:99f893e30497a4b66842821bac316386f7bd5c4f47ad35c9073ef089aa33af32", size = 14747, upload-time = "2020-08-10T16:37:46.4Z" }, -] - -[[package]] -name = "yapf" -version = "0.43.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "platformdirs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907, upload-time = "2024-11-14T00:11:41.584Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/81/6acd6601f61e31cfb8729d3da6d5df966f80f374b78eff83760714487338/yapf-0.43.0-py3-none-any.whl", hash = "sha256:224faffbc39c428cb095818cf6ef5511fdab6f7430a10783fdfb292ccf2852ca", size = 256158, upload-time = "2024-11-14T00:11:39.37Z" }, -] diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift new file mode 100644 index 0000000..8bdeb98 --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift @@ -0,0 +1,155 @@ +import SwiftUI +import simd + +/// Lightweight SwiftUI overlay that draws the 68-point face skeleton and +/// 21-point hand skeletons sent by data_only_viz/pose_bridge.py over OSC. +/// Sits in the ContentView ZStack above BodyView. Coordinates from the +/// listener are normalised (0..1 in image space) ; here we map them to +/// the overlay's geometry. Rendering is intentionally minimal : small +/// dots + a few polylines for facial features and hand bones. +struct FaceHandOverlay: View { + @ObservedObject var poseListener: PoseOSCListener + var showFace: Bool = true + var showHands: Bool = true + + var body: some View { + GeometryReader { geo in + Canvas { ctx, size in + if showFace { + for face in poseListener.faces.values { + drawFace(face, in: &ctx, size: size) + } + } + if showHands { + for hand in poseListener.hands.values { + drawHand(hand, in: &ctx, size: size) + } + } + } + .frame(width: geo.size.width, height: geo.size.height) + .allowsHitTesting(false) + } + } + + // MARK: - Face (dlib 68 layout) + + /// Index spans in the 68-point dlib convention. + private static let jaw = Array(0..<17) + private static let browR = Array(17..<22) + private static let browL = Array(22..<27) + private static let noseBridge = Array(27..<31) + private static let nostril = Array(31..<36) + private static let eyeR = Array(36..<42) + private static let eyeL = Array(42..<48) + private static let lipOuter = Array(48..<60) + private static let lipInner = Array(60..<68) + + private func drawFace(_ face: PoseOSCListener.FaceFrame, + in ctx: inout GraphicsContext, + size: CGSize) { + let stroke = GraphicsContext.Shading.color(.green.opacity(0.85)) + let dot = GraphicsContext.Shading.color(.green.opacity(0.95)) + + drawPolyline(face.points, indices: Self.jaw, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.browR, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.browL, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.noseBridge, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.nostril, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.eyeR, closed: true, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.eyeL, closed: true, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.lipOuter, closed: true, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.lipInner, closed: true, + in: &ctx, size: size, shading: stroke, width: 1.0) + + for i in 0..<68 where face.hasPoint[i] { + let p = mapPoint(face.points[i], size: size) + let r = CGRect(x: p.x - 1.2, y: p.y - 1.2, + width: 2.4, height: 2.4) + ctx.fill(Path(ellipseIn: r), with: dot) + } + } + + // MARK: - Hand (MediaPipe 21 landmarks) + + /// MediaPipe hand bone connectivity (5 fingers x 4 bones + palm). + private static let handBones: [(Int, Int)] = [ + // Thumb + (0, 1), (1, 2), (2, 3), (3, 4), + // Index + (0, 5), (5, 6), (6, 7), (7, 8), + // Middle + (5, 9), (9, 10), (10, 11), (11, 12), + // Ring + (9, 13), (13, 14), (14, 15), (15, 16), + // Pinky + (13, 17), (17, 18), (18, 19), (19, 20), + // Palm closure + (0, 17), + ] + + private func drawHand(_ hand: PoseOSCListener.HandFrame, + in ctx: inout GraphicsContext, + size: CGSize) { + // Left = cyan, right = magenta. + let color: Color = hand.side == 0 ? .cyan : .pink + let stroke = GraphicsContext.Shading.color(color.opacity(0.85)) + let dot = GraphicsContext.Shading.color(color.opacity(0.95)) + + for (a, b) in Self.handBones { + guard hand.hasPoint[a], hand.hasPoint[b] else { continue } + let pa = mapPoint(hand.points[a], size: size) + let pb = mapPoint(hand.points[b], size: size) + var path = Path() + path.move(to: pa) + path.addLine(to: pb) + ctx.stroke(path, with: stroke, lineWidth: 1.8) + } + for i in 0..<21 where hand.hasPoint[i] { + let p = mapPoint(hand.points[i], size: size) + let r = CGRect(x: p.x - 1.8, y: p.y - 1.8, + width: 3.6, height: 3.6) + ctx.fill(Path(ellipseIn: r), with: dot) + } + } + + // MARK: - Helpers + + private func drawPolyline(_ pts: [SIMD2], + indices: [Int], + closed: Bool, + in ctx: inout GraphicsContext, + size: CGSize, + shading: GraphicsContext.Shading, + width: CGFloat) { + guard indices.count >= 2 else { return } + var path = Path() + var started = false + for i in indices { + let p = mapPoint(pts[i], size: size) + if !started { + path.move(to: p) + started = true + } else { + path.addLine(to: p) + } + } + if closed, let first = indices.first { + path.addLine(to: mapPoint(pts[first], size: size)) + } + ctx.stroke(path, with: shading, lineWidth: width) + } + + private func mapPoint(_ p: SIMD2, size: CGSize) -> CGPoint { + // Normalised coords come from MediaPipe in image space already. + CGPoint(x: CGFloat(p.x) * size.width, + y: CGFloat(p.y) * size.height) + } +} diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift index cc9fb4e..ffdbd41 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift @@ -66,23 +66,9 @@ struct SettingsPanel: View { layerRow(icon: "circle.dotted", label: "Fil de fer", isOn: $settings.showWireframe) - // Squelette toggle : bascule en mode openpos (#9). ON - // synchronise vizMode=9 et showSkeleton=true ; OFF revient - // au mode 0 (storm). Sync bidirectionnel pour que la touche - // "p" et le picker viz updatent le toggle automatiquement. layerRow(icon: "figure.stand", - label: "Squelette (openpos)", - isOn: Binding( - get: { settings.vizMode == 9 }, - set: { on in - if on { - settings.vizMode = 9 - settings.showSkeleton = true - } else { - settings.vizMode = 0 - settings.showSkeleton = false - } - })) + label: "Squelette (à venir)", + isOn: $settings.showSkeleton) // Picker viz mode 0..9 HStack(spacing: 4) { ForEach(0..<10) { i in diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift new file mode 100644 index 0000000..0a6c2f3 --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift @@ -0,0 +1,222 @@ +import Combine +import Foundation +import RealityKit +import SwiftUI +import simd + +/// RealityKit renderer for MediaPipe Pose 3D world landmarks (33 joints, +/// metric coords relative to the hip-center). Consumes the `body3d` +/// publisher of `PoseOSCListener` and maintains one entity tree per +/// detected person. +/// +/// Coordinate mapping (MediaPipe -> RealityKit): +/// MediaPipe : x = right, y = down, z = forward (away from cam). +/// RealityKit: x = right, y = up, z = backward (toward cam). +/// => convert with (x, -y, -z). +@MainActor +final class Skeleton3DRenderer: ObservableObject { + /// 32 bones connecting MediaPipe Pose 33 landmarks. Indices are + /// the canonical MediaPipe Pose landmark indices. Source: official + /// `mp.solutions.pose.POSE_CONNECTIONS` (Holistic / Pose Landmarker + /// share the same 33-pt schema). + static let POSE_CONNECTIONS: [(Int, Int, BoneChain)] = [ + // Face (kept light: nose <-> inner eyes <-> outer eyes <-> ears) + (0, 1, .face), (1, 2, .face), (2, 3, .face), (3, 7, .face), + (0, 4, .face), (4, 5, .face), (5, 6, .face), (6, 8, .face), + (9, 10, .face), + // Torso + (11, 12, .trunk), (11, 23, .trunk), (12, 24, .trunk), + (23, 24, .trunk), + // Left arm + (11, 13, .arm), (13, 15, .arm), + (15, 17, .arm), (15, 19, .arm), (15, 21, .arm), (17, 19, .arm), + // Right arm + (12, 14, .arm), (14, 16, .arm), + (16, 18, .arm), (16, 20, .arm), (16, 22, .arm), (18, 20, .arm), + // Left leg + (23, 25, .leg), (25, 27, .leg), + (27, 29, .leg), (27, 31, .leg), (29, 31, .leg), + // Right leg + (24, 26, .leg), (26, 28, .leg), + (28, 30, .leg), (28, 32, .leg), (30, 32, .leg), + ] + + enum BoneChain { + case trunk, arm, leg, face + var color: NSColor { + switch self { + case .trunk: return .white + case .arm: return .systemTeal + case .leg: return .systemPink // approx magenta + case .face: return NSColor(white: 0.7, alpha: 1.0) + } + } + } + + private static let jointRadius: Float = 0.02 // 2 cm + private static let boneRadius: Float = 0.012 // 1.2 cm + private static let minConfidence: Float = 0.3 + private static let retainSec: TimeInterval = 1.0 + + /// Update throttle : tick at most every `updatePeriod` seconds even + /// if the publisher fires faster (Combine debounce-style on a clock). + private static let updatePeriod: TimeInterval = 1.0 / 30.0 + + private struct PersonEntities { + var root: Entity + var joints: [ModelEntity] // 33 spheres + var bones: [ModelEntity] // 32 bone entities, same order as POSE_CONNECTIONS + } + + private var persons: [Int: PersonEntities] = [:] + private var lastSeenAt: [Int: TimeInterval] = [:] + private weak var rootAnchor: Entity? + private var poseSub: AnyCancellable? + private var lastUpdateAt: TimeInterval = 0 + + /// Attach to a scene by giving it an AnchorEntity that owns all + /// skeleton entities, and start observing the listener. + func attach(to anchor: Entity, listener: PoseOSCListener) { + rootAnchor = anchor + poseSub = listener.$body3d + .receive(on: DispatchQueue.main) + .sink { [weak self] frames in + Task { @MainActor in self?.update(frames: frames) } + } + } + + func detach() { + poseSub?.cancel() + poseSub = nil + for (_, p) in persons { p.root.removeFromParent() } + persons.removeAll() + lastSeenAt.removeAll() + } + + // MARK: - Update + + private func update(frames: [Int: PoseOSCListener.Pose3DFrame]) { + let now = CACurrentMediaTime() + if now - lastUpdateAt < Self.updatePeriod { return } + lastUpdateAt = now + + guard let anchor = rootAnchor else { return } + + // Mark fresh pids + for pid in frames.keys { lastSeenAt[pid] = now } + // GC stale persons + let cutoff = now - Self.retainSec + for (pid, p) in persons where (lastSeenAt[pid] ?? 0) < cutoff { + p.root.removeFromParent() + persons.removeValue(forKey: pid) + lastSeenAt.removeValue(forKey: pid) + } + + for (pid, frame) in frames { + let entities = persons[pid] ?? makePerson(pid: pid, parent: anchor) + persons[pid] = entities + apply(frame: frame, to: entities) + } + } + + private func apply(frame: PoseOSCListener.Pose3DFrame, + to entities: PersonEntities) { + // Convert all 33 keypoints to RealityKit space once. + var rk = [SIMD3](repeating: .zero, count: 33) + var valid = [Bool](repeating: false, count: 33) + for i in 0..<33 { + let k = frame.kps[i] + let visible = frame.hasPoint[i] && k.w >= Self.minConfidence + valid[i] = visible + // Mediapipe (x right, y down, z forward) -> RK (x right, y up, z back) + rk[i] = SIMD3(k.x, -k.y, -k.z) + } + + // Joints: position spheres and toggle visibility. + for i in 0..<33 { + let joint = entities.joints[i] + if valid[i] { + joint.transform.translation = rk[i] + joint.isEnabled = true + } else { + joint.isEnabled = false + } + } + + // Bones: orient + scale length between endpoints. + for (bIdx, (a, b, _)) in Self.POSE_CONNECTIONS.enumerated() { + let bone = entities.bones[bIdx] + if !valid[a] || !valid[b] { + bone.isEnabled = false + continue + } + let pa = rk[a] + let pb = rk[b] + let delta = pb - pa + let len = simd_length(delta) + if len < 1e-5 { + bone.isEnabled = false + continue + } + let mid = (pa + pb) * 0.5 + // Bone mesh is a cylinder of height=1 along +Y. Rotate +Y + // onto the (b-a) direction. + let dir = delta / len + let yAxis = SIMD3(0, 1, 0) + let dot = simd_dot(yAxis, dir) + let rot: simd_quatf + if dot > 0.9999 { + rot = simd_quatf(angle: 0, axis: SIMD3(0, 1, 0)) + } else if dot < -0.9999 { + rot = simd_quatf(angle: .pi, axis: SIMD3(1, 0, 0)) + } else { + let axis = simd_normalize(simd_cross(yAxis, dir)) + let angle = acos(dot) + rot = simd_quatf(angle: angle, axis: axis) + } + bone.transform.translation = mid + bone.transform.rotation = rot + // Scale length only on Y, keep XZ at 1 to preserve radius. + bone.transform.scale = SIMD3(1, len, 1) + bone.isEnabled = true + } + } + + // MARK: - Construction + + private func makePerson(pid: Int, parent: Entity) -> PersonEntities { + let root = Entity() + parent.addChild(root) + + // Joint sphere mesh shared across joints (cheap to reuse). + let sphereMesh = MeshResource.generateSphere( + radius: Self.jointRadius) + let jointMat = SimpleMaterial( + color: .white, roughness: 0.6, isMetallic: false) + var joints: [ModelEntity] = [] + joints.reserveCapacity(33) + for _ in 0..<33 { + let e = ModelEntity(mesh: sphereMesh, materials: [jointMat]) + e.isEnabled = false + root.addChild(e) + joints.append(e) + } + + // One cylinder per bone (height=1, scaled at runtime). + let cylMesh = MeshResource.generateCylinder( + height: 1.0, radius: Self.boneRadius) + var bones: [ModelEntity] = [] + bones.reserveCapacity(Self.POSE_CONNECTIONS.count) + for (_, _, chain) in Self.POSE_CONNECTIONS { + let mat = SimpleMaterial( + color: chain.color, roughness: 0.6, isMetallic: false) + let e = ModelEntity(mesh: cylMesh, materials: [mat]) + e.isEnabled = false + root.addChild(e) + bones.append(e) + } + NSLog("Skeleton3DRenderer: spawned pid=%d (33 joints, %d bones)", + pid, bones.count) + return PersonEntities(root: root, joints: joints, bones: bones) + } +} diff --git a/sound_algo/control/data_feeds.scd b/sound_algo/control/data_feeds.scd index 586d715..c4a52f6 100644 --- a/sound_algo/control/data_feeds.scd +++ b/sound_algo/control/data_feeds.scd @@ -294,6 +294,43 @@ ("[feeds] heartbeat: " ++ if(~feedAlive.value) { "ALIVE" } { "DOWN" }).postln; }; +// ===================================================================== +// ACTION-HEAD OSC handlers (/pose/action, /pose/kin, /pose/enter, /pose/leave) +// ===================================================================== +~poseState = ~poseState ? Dictionary.new; +~poseKin = ~poseKin ? Dictionary.new; + +OSCdef(\poseAction, { |msg| + var pid = msg[1]; + ~poseState[pid] = ( + labelIdx: msg[2], + probs: [msg[3], msg[4], msg[5]], + ); +}, '/pose/action'); + +OSCdef(\poseKin, { |msg| + var pid = msg[1]; + ~poseKin[pid] = ( + speed: msg[2], + accel: msg[3], + symmetry: msg[4], + ); +}, '/pose/kin'); + +OSCdef(\poseEnter, { |msg| + var pid = msg[1]; + ~poseState[pid] = (labelIdx: 0, probs: [1.0, 0.0, 0.0]); + ~poseKin[pid] = (speed: 0, accel: 0, symmetry: 0); +}, '/pose/enter'); + +OSCdef(\poseLeave, { |msg| + var pid = msg[1]; + ~poseState.removeAt(pid); + ~poseKin.removeAt(pid); +}, '/pose/leave'); + +"[OK] action-head OSC handlers".postln; + "[data_feeds] OSCdef installes (USGS, SWPC, NETZ, RTE, BLITZ, OPENSKY, BSKY, MEMPOOL, GCN).".postln; "[data_feeds] usage : ~feeds[\\swpc_kp], ~feedGet.(\\netz_dev, 0), ~feedDump.()".postln; )