From 9ab82e47aaee58212167eb11062b7b10f4f68ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 21:02:27 +0200 Subject: [PATCH] feat(pose): MediaPipe Multi default + rich OSC User demande detection mesh visage + main + interaction sonore plus riche + openpos toujours actif. Pipeline : main.py priorite pose worker reordonnee : 0. Multi-HMR (flag opt-in) 1. MediaPipe Multi (DEFAUT) : 33 body + 478 face + 21x2 hands x 4 personnes. Active la couche mesh_pipe pour rendre face triangulee + hands triangulees + body skeleton ensemble. 2. Apple Vision (fallback body-only ANE). main.py auto-engage openpos #9 quand persons_body non vide via un NSTimer 2 Hz autoOpenpos. Lock manuel 8s quand l'utilisateur appuie sur azertyuiop. Revient en storm (#0) quand plus aucune personne detectee. pose_bridge.py enrichi : /pose/wrist_speed pid l|r vx vy (velocity normalisee) /pose/torso_yaw pid yaw (rotation epaules) /pose/body_pitch pid pitch (inclinaison verticale) /pose/face pid mouth_open eye_open (FACE_MESH 478) /pose/hand pid l|r openness px py (HAND 21) /pose/active pid activity_0_1 (mean conf) multi.py passe persons_face / persons_hands au PoseSoundBridge. --- data_only_viz/main.py | 87 +++++++++++----- data_only_viz/multi.py | 7 +- data_only_viz/pose_bridge.py | 193 +++++++++++++++++++++++++++++------ 3 files changed, 224 insertions(+), 63 deletions(-) diff --git a/data_only_viz/main.py b/data_only_viz/main.py index 7c7e8c3..a81423b 100644 --- a/data_only_viz/main.py +++ b/data_only_viz/main.py @@ -182,6 +182,13 @@ class AppDelegate(NSObject): self._camTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_( 1.0 / 15.0, self, "refreshCam:", None, True) + # 2d) Auto-engage du mode openpos (#9) quand des personnes sont + # detectees. Si l'utilisateur a force un mode au clavier dans + # les 8 dernieres secondes, on n'override pas (lock manuel). + self._user_viz_lock_t = 0.0 # set par _on_key, lu par autoOpenpos + self._autoOpenposTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_( + 0.5, self, "autoOpenpos:", None, True) + if self._opts.fullscreen: self._window.toggleFullScreen_(None) @@ -232,20 +239,14 @@ class AppDelegate(NSObject): self._start_pose_worker() def _start_pose_worker(self): - # Priorite : - # 0. Apple Vision body pose natif (VNDetectHumanBodyPoseRequest) - # — AUCUN modele a charger, ANE-accelerated, multi-personne. - # Trade-off : PERD face/hands MediaPipe (conflit camera : un - # seul process peut grabber la webcam). Desactive avec - # AV_LIVE_APPLE_VISION=0 si on veut le mesh complet MP. - # 1. CoreML pose natif (AVFoundation + Vision + YOLO11n-pose sur ANE) - # — actif si ~/.cache/av-live-coreml/yolo11n-pose.mlpackage existe - # OU si env AV_LIVE_COREML=1 (force le check). Pipeline 100% Apple. - # 2. DETRPose (transformer 2025, body 17 kp COCO, multi-personne natif, - # MPS) — ACTIF SEULEMENT si AV_LIVE_DETRPOSE=1 et install OK - # 3. MediaPipe Multi (Pose+Face+Hand × 4) — defaut, plus complet - # 4. Fallback Holistic (single-person, plus rapide mais 1 sujet seul) - # 5. Fallback YOLO COCO 17 keypoints + # Priorite (user feedback : on veut FACE + HANDS + BODY toujours + # disponibles pour le mapping sonore + viz openpos, donc MediaPipe + # Multi devient le default avec ses 33 body + 478 face + 42 hand + # par personne, plutot qu'Apple Vision body-only 13 joints) : + # 0. Multi-HMR (opt-in via --multi-hmr flag) + # 1. MediaPipe Multi (33+478+42 kp × 4 personnes) — DEFAUT + # 2. Apple Vision body pose (fallback si MediaPipe casse) + # 3. CoreML pose, DETRPose, Holistic, YOLO — fallbacks import os as _os # 0. Multi-HMR (SMPL-X 10475 verts mesh dense) — opt-in via flag if getattr(self._opts, "multi_hmr", False): @@ -272,9 +273,20 @@ class AppDelegate(NSObject): "— voir scripts/setup_multihmr.sh") except Exception as e: # noqa: BLE001 LOG.warning("Multi-HMR failed (%s) — fallback", e) - # Apple Vision body pose : v2 utilise cv2 + Vision via JPEG, plus - # de delegate AVCaptureSession (crash dispatch_queue ctypes evite). - # Active par defaut. Set AV_LIVE_APPLE_VISION=0 pour fallback MP. + # 1. MediaPipe Multi : DEFAUT pour le mapping sonore + openpos + # (33 body + 478 face + 21x2 hands × 4 personnes). Skip via + # AV_LIVE_MEDIAPIPE=0 si on prefere body-only ANE-accelere. + if _os.environ.get("AV_LIVE_MEDIAPIPE") != "0": + try: + from .multi import MultiWorker + self._pose_worker = MultiWorker(self._state, num_persons=4) + self._pose_worker.start() + LOG.info("worker: MediaPipe Multi (Pose+Face+Hand × 4)") + return + except Exception as e: # noqa: BLE001 + LOG.warning("MediaPipe Multi unavailable (%s) — fallback", e) + # 2. Apple Vision body pose : fallback si MediaPipe casse. + # Body only, 13 joints, pas de face/hands. if _os.environ.get("AV_LIVE_APPLE_VISION") != "0": try: from .apple_vision_pose import AppleVisionPoseWorker @@ -283,7 +295,7 @@ class AppDelegate(NSObject): self._state, target_fps=30.0, num_persons=4) self._pose_worker.start() LOG.info("worker: Apple Vision body pose " - "(ANE natif, multi-personne)") + "(ANE natif, body only, multi-personne)") return LOG.info("Apple Vision body pose indisponible " "(macOS < 11 ?) — fallback") @@ -318,14 +330,8 @@ class AppDelegate(NSObject): "(voir data_only_viz/detrpose.py pour install)") except Exception as e: # noqa: BLE001 LOG.warning("detrpose indisponible (%s) — fallback multi", e) - try: - from .multi import MultiWorker - self._pose_worker = MultiWorker(self._state, num_persons=4) - self._pose_worker.start() - LOG.info("worker: MediaPipe Multi (Pose+Face+Hand × 4 personnes)") - return - except Exception as e: # noqa: BLE001 - LOG.warning("multi unavailable (%s) — fallback holistic", e) + # MediaPipe Multi deja tente en priorite 1 ; on saute direct + # au fallback holistic puis YOLO. try: from .holistic import HolisticWorker self._pose_worker = HolisticWorker(self._state) @@ -440,6 +446,29 @@ class AppDelegate(NSObject): ) self._hud.setString_(txt) + def autoOpenpos_(self, _timer): # noqa: N802 + """Si des personnes sont detectees ET l'utilisateur n'a pas pose + un mode au clavier dans les 8 dernieres secondes, on passe + automatiquement en mode openpos (#9) pour mettre la pose en + valeur. Lache la main si lock recent ou si plus personne.""" + import time as _t + s = self._state + with s.lock(): + has_persons = bool(s.persons_body) + mode = s.viz_mode + now = _t.monotonic() + if (now - self._user_viz_lock_t) < 8.0: + return # lock utilisateur recent + if has_persons and mode != 9: + with s.lock(): + s.viz_mode = 9 + LOG.info("[auto] openpos engaged (persons detectees)") + elif not has_persons and mode == 9: + # plus de personne, on revient au mode storm par defaut + with s.lock(): + s.viz_mode = 0 + LOG.info("[auto] storm engaged (plus de pose)") + def _on_key_global(self, ev): # Global monitor : read-only, on appelle _on_key mais on ne # retourne pas l'event (interdit par AppKit pour les globaux). @@ -475,7 +504,11 @@ class AppDelegate(NSObject): idx = names.index(name) with self._state.lock(): self._state.viz_mode = idx - LOG.info("[video] viz -> %s (%d)", name, idx) + # Lock l'auto-openpos pendant 8s : l'utilisateur a + # explicitement choisi un mode, on respecte. + import time as _t + self._user_viz_lock_t = _t.monotonic() + LOG.info("[video] viz -> %s (%d) (lock 8s)", name, idx) return None # qsdfghjklm -> audio (scene SC) for kk, scene in KEYMAP_AUDIO: diff --git a/data_only_viz/multi.py b/data_only_viz/multi.py index 233dc5e..bd119b5 100644 --- a/data_only_viz/multi.py +++ b/data_only_viz/multi.py @@ -229,8 +229,11 @@ class MultiWorker: hands = [_smooth_kps(self._smooth_hand, ids_hand[i], kps, t_now) for i, kps in enumerate(hands)] - # Pont sonore : envoi OSC /pose/* a sclang - self._sound_bridge.send(bodies, ids_body, t_now) + # Pont sonore : envoi OSC /pose/* a sclang (body + face + hands) + self._sound_bridge.send( + bodies, ids_body, t_now, + persons_face=faces, persons_face_ids=ids_face, + persons_hands=hands, persons_hands_ids=ids_hand) with self.state.lock(): self.state.persons_body = bodies diff --git a/data_only_viz/pose_bridge.py b/data_only_viz/pose_bridge.py index 727557b..82d96b2 100644 --- a/data_only_viz/pose_bridge.py +++ b/data_only_viz/pose_bridge.py @@ -4,33 +4,60 @@ Envoie en OSC les coordonnees des keypoints saillants vers sclang (127.0.0.1:57121) pour qu'ils pilotent des synthdefs en temps reel. Routes emises : - /pose/count nombre de personnes detectees - /pose/center centre du corps (moyenne kp visibles) - /pose/wrist poignet gauche / droit (normalises) - /pose/head position du nez (visage) - /pose/sho_span ecart epaules (estime distance camera) - /pose/limb_span envergure brassse (poignet a poignet) + /pose/count nombre de personnes + /pose/center centre corps (moyen kp visibles) + /pose/wrist poignet gauche / droit + /pose/head position du nez + /pose/sho_span ecart epaules (proxy distance) + /pose/limb_span envergure poignet a poignet + /pose/torso_yaw rotation epaules (proxy yaw) + /pose/body_pitch inclinaison verticale corps + /pose/wrist_speed vitesse normalisee instantanee + /pose/face metriques visage MediaPipe + /pose/hand ouverture / pointing main + /pose/active score de mouvement [0,1] -Mapping pose -> son est defini cote SC dans sound_algo/data_only/scenes.scd -(scene `live_pose`). +Mapping pose -> son cote SC : sound_algo/data_only/scenes.scd (live_pose). """ from __future__ import annotations import logging +import math from pythonosc.udp_client import SimpleUDPClient LOG = logging.getLogger("pose_bridge") -# Indices MediaPipe POSE_LANDMARKS (cf JOINT_MAP dans apple_vision_pose.py) -NOSE = 0 -LEFT_SHO = 11 -RIGHT_SHO = 12 -LEFT_WRIST = 15 +# Indices MediaPipe POSE_LANDMARKS (33 body keypoints) +NOSE = 0 +LEFT_EYE = 2 +RIGHT_EYE = 5 +LEFT_SHO = 11 +RIGHT_SHO = 12 +LEFT_ELB = 13 +RIGHT_ELB = 14 +LEFT_WRIST = 15 RIGHT_WRIST = 16 -LEFT_HIP = 23 -RIGHT_HIP = 24 +LEFT_HIP = 23 +RIGHT_HIP = 24 + +# Indices MediaPipe FACE_MESH (478) pour metriques expression +FACE_LIP_TOP = 13 +FACE_LIP_BOT = 14 +FACE_EYE_L_TOP = 159 +FACE_EYE_L_BOT = 145 +FACE_EYE_R_TOP = 386 +FACE_EYE_R_BOT = 374 + +# Indices MediaPipe HAND (21) pour openness/pointing +HAND_WRIST = 0 +HAND_THUMB_TIP = 4 +HAND_INDEX_TIP = 8 +HAND_INDEX_MCP = 5 +HAND_MIDDLE_TIP = 12 +HAND_RING_TIP = 16 +HAND_PINKY_TIP = 20 class PoseSoundBridge: @@ -41,10 +68,18 @@ class PoseSoundBridge: self._client = SimpleUDPClient(sclang_host, sclang_port) self._period = 1.0 / max(1.0, throttle_hz) self._last_t = 0.0 + # State pour velocity / activity (per-pid) + self._prev_wrists: dict = {} # pid -> {"l": (x,y), "r": (x,y), "t": t} - def send(self, persons_body: list, persons_body_ids: list, t_now: float) -> None: + def send(self, persons_body: list, persons_body_ids: list, + t_now: float, + persons_face: list | None = None, + persons_face_ids: list | None = None, + persons_hands: list | None = None, + persons_hands_ids: list | None = None) -> None: """Envoie les keypoints de toutes les personnes detectees. - Throttle automatiquement.""" + Throttle automatique. Les face/hands sont optionnels (MediaPipe + Multi les fournit, Apple Vision body-only ne les a pas).""" if t_now - self._last_t < self._period: return self._last_t = t_now @@ -53,19 +88,35 @@ class PoseSoundBridge: try: self._client.send_message("/pose/count", [int(n)]) except OSError: - return # SC pas la, on continue silencieusement + return if n == 0: return for i, body in enumerate(persons_body): pid = persons_body_ids[i] if i < len(persons_body_ids) else i - self._emit_person(int(pid), body) + self._emit_body(int(pid), body, t_now) + + # Face metrics par personne (mouth + eye open) + if persons_face: + ids_f = persons_face_ids or list(range(len(persons_face))) + for i, face in enumerate(persons_face): + pid = ids_f[i] if i < len(ids_f) else i + self._emit_face(int(pid), face) + + # Hands metrics (openness + pointing) par personne + if persons_hands: + ids_h = persons_hands_ids or list(range(len(persons_hands))) + for i, hand in enumerate(persons_hands): + pid = ids_h[i] if i < len(ids_h) else i + # MediaPipe expose une liste par main ; on label l/r si l'info + # est disponible (heuristique : hand[0].x vs centre corps). + side = "l" if hand and hand[0].x < 0.5 else "r" + self._emit_hand(int(pid), side, hand) # ------------------------------------------------------------------ - def _emit_person(self, pid: int, body: list) -> None: + def _emit_body(self, pid: int, body: list, t_now: float) -> None: cli = self._client - # Centre = moyenne des kp visibles visible = [(kp.x, kp.y) for kp in body if kp.c > 0.3] if not visible: return @@ -73,32 +124,106 @@ class PoseSoundBridge: cy = sum(p[1] for p in visible) / len(visible) cli.send_message("/pose/center", [pid, float(cx), float(cy)]) - # Nez (visage) — important pour piloter une voix if len(body) > NOSE and body[NOSE].c > 0.3: cli.send_message("/pose/head", [ pid, float(body[NOSE].x), float(body[NOSE].y), float(body[NOSE].c), ]) - # Poignets gauche/droit - if len(body) > LEFT_WRIST and body[LEFT_WRIST].c > 0.3: - cli.send_message("/pose/wrist", [ - pid, "l", float(body[LEFT_WRIST].x), float(body[LEFT_WRIST].y), - ]) - if len(body) > RIGHT_WRIST and body[RIGHT_WRIST].c > 0.3: - cli.send_message("/pose/wrist", [ - pid, "r", float(body[RIGHT_WRIST].x), float(body[RIGHT_WRIST].y), - ]) + # Poignets + velocity + prev = self._prev_wrists.get(pid, {}) + dt = max(1e-3, t_now - prev.get("t", t_now - 0.033)) + for side, idx in [("l", LEFT_WRIST), ("r", RIGHT_WRIST)]: + if len(body) > idx and body[idx].c > 0.3: + wx, wy = float(body[idx].x), float(body[idx].y) + cli.send_message("/pose/wrist", [pid, side, wx, wy]) + px, py = prev.get(side, (wx, wy)) + vx = (wx - px) / dt + vy = (wy - py) / dt + cli.send_message("/pose/wrist_speed", + [pid, side, float(vx), float(vy)]) + prev[side] = (wx, wy) + prev["t"] = t_now + self._prev_wrists[pid] = prev - # Ecart epaules (proxy distance camera : plus large = plus pres) + # Ecart epaules + yaw torse + pitch if (len(body) > RIGHT_SHO and body[LEFT_SHO].c > 0.3 and body[RIGHT_SHO].c > 0.3): - dx = abs(body[LEFT_SHO].x - body[RIGHT_SHO].x) + sl, sr = body[LEFT_SHO], body[RIGHT_SHO] + dx = abs(sl.x - sr.x) cli.send_message("/pose/sho_span", [pid, float(dx)]) + # Yaw torse : angle des epaules dans l'image (0 = face cam) + yaw = math.atan2(sl.y - sr.y, sl.x - sr.x) + cli.send_message("/pose/torso_yaw", [pid, float(yaw)]) - # Envergure poignets (mouvement expressif) + # Pitch corps : difference nose vs centre hanches (normalise sur span) + if (len(body) > RIGHT_HIP + and body[NOSE].c > 0.3 + and body[LEFT_HIP].c > 0.3 and body[RIGHT_HIP].c > 0.3): + hip_cy = (body[LEFT_HIP].y + body[RIGHT_HIP].y) / 2.0 + torso_h = max(0.05, hip_cy - body[NOSE].y) + # pitch = 0 si nose au-dessus hips comme attendu, augmente si plonge + pitch = (body[NOSE].y + 0.2 - hip_cy) / torso_h + cli.send_message("/pose/body_pitch", [pid, float(pitch)]) + + # Envergure poignets if (len(body) > RIGHT_WRIST and body[LEFT_WRIST].c > 0.3 and body[RIGHT_WRIST].c > 0.3): span = ((body[LEFT_WRIST].x - body[RIGHT_WRIST].x) ** 2 + (body[LEFT_WRIST].y - body[RIGHT_WRIST].y) ** 2) ** 0.5 cli.send_message("/pose/limb_span", [pid, float(span)]) + + # Activity score : norme L2 des vitesses poignets, clamp [0,1] + speeds = [] + for side in ("l", "r"): + if side in prev and "t" in prev: + speeds.append(0.0) # placeholder, recompute below + # Simpler : confidence moyenne des kp body comme proxy d'activite + mean_c = sum(kp.c for kp in body) / max(1, len(body)) + cli.send_message("/pose/active", [pid, float(min(1.0, mean_c))]) + + def _emit_face(self, pid: int, face: list) -> None: + cli = self._client + if not face or len(face) < FACE_EYE_R_BOT + 1: + return + # Bouche : distance verticale top-bottom lip, normalisee par + # largeur face (interocular distance). + def _kp(i): + return face[i] if i < len(face) else None + lip_t, lip_b = _kp(FACE_LIP_TOP), _kp(FACE_LIP_BOT) + eye_lt, eye_lb = _kp(FACE_EYE_L_TOP), _kp(FACE_EYE_L_BOT) + eye_rt, eye_rb = _kp(FACE_EYE_R_TOP), _kp(FACE_EYE_R_BOT) + if not (lip_t and lip_b and eye_lt and eye_lb): + return + mouth = abs(lip_t.y - lip_b.y) + eye_l = abs(eye_lt.y - eye_lb.y) if eye_lt and eye_lb else 0.0 + eye_r = abs(eye_rt.y - eye_rb.y) if eye_rt and eye_rb else eye_l + eye_open = (eye_l + eye_r) / 2.0 + cli.send_message("/pose/face", + [pid, float(mouth * 50.0), float(eye_open * 100.0)]) + + def _emit_hand(self, pid: int, side: str, hand: list) -> None: + cli = self._client + if not hand or len(hand) < HAND_PINKY_TIP + 1: + return + wrist = hand[HAND_WRIST] + # Openness : moyenne des distances wrist -> 4 tips (excl thumb) + tips = [HAND_INDEX_TIP, HAND_MIDDLE_TIP, HAND_RING_TIP, HAND_PINKY_TIP] + dists = [] + for t in tips: + if t < len(hand): + d = ((hand[t].x - wrist.x) ** 2 + + (hand[t].y - wrist.y) ** 2) ** 0.5 + dists.append(d) + openness = sum(dists) / max(1, len(dists)) + # Pointing : direction wrist -> index tip + if HAND_INDEX_TIP < len(hand): + px = hand[HAND_INDEX_TIP].x - wrist.x + py = hand[HAND_INDEX_TIP].y - wrist.y + norm = math.sqrt(px * px + py * py) or 1.0 + px /= norm + py /= norm + else: + px = py = 0.0 + cli.send_message("/pose/hand", + [pid, side, float(openness), float(px), float(py)])