From 2a732faffcd72b6f0d53ed431056ebd8f89e4dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:02:29 +0200 Subject: [PATCH 01/14] fix(data-only-viz): action-head review fixes Adresse final review du feature action-head : - action_head_pub.py + extract_j3d_offline.py : CoreMLArray wraps numpy mais n'a pas __array__ ; unwrap via .numpy() avant np.asarray pour eviter object-array silencieux quand persons_smplx vient du backend CoreML. extract_j3d ramene depuis main (manquait sur feat suite au merge c52271e). - train_on_studio.sh : TRAIN_ARGS quote defensivement via printf %q + reject single quotes pour eviter injection via le payload single-quoted sur bastion. --- data_only_viz/action_head_pub.py | 4 + data_only_viz/scripts/extract_j3d_offline.py | 127 +++++++++++++++++++ data_only_viz/scripts/train_on_studio.sh | 12 +- 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 data_only_viz/scripts/extract_j3d_offline.py diff --git a/data_only_viz/action_head_pub.py b/data_only_viz/action_head_pub.py index ca36bb2..8c8f0d6 100644 --- a/data_only_viz/action_head_pub.py +++ b/data_only_viz/action_head_pub.py @@ -129,6 +129,10 @@ class ActionHeadPublisher(threading.Thread): v3d = p.get("v3d") if v3d is None: continue + # CoreMLArray wraps a numpy array but has no __array__ + # protocol; unwrap via .numpy() before np.asarray. + if hasattr(v3d, "numpy") and not isinstance(v3d, np.ndarray): + v3d = v3d.numpy() v3d_np = np.asarray(v3d, dtype=np.float32) if v3d_np.shape[0] < max(SMPLX_JOINT_ANCHOR_VERTS) + 1: continue diff --git a/data_only_viz/scripts/extract_j3d_offline.py b/data_only_viz/scripts/extract_j3d_offline.py new file mode 100644 index 0000000..27ca9d4 --- /dev/null +++ b/data_only_viz/scripts/extract_j3d_offline.py @@ -0,0 +1,127 @@ +"""Extract j3d (22 SMPL-X joint anchors) from a recorded MP4 using the +Multi-HMR CoreML backend, write per-frame per-person jsonl rows. + +Usage: + uv run python -m data_only_viz.scripts.extract_j3d_offline \ + --session sess03 \ + --video ~/.cache/av-live-action/raw/sess03.mp4 \ + --out ~/.cache/av-live-action/raw/sess03.jsonl +""" +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path + +import cv2 +import numpy as np + +from data_only_viz.action_head_pub import SMPLX_JOINT_ANCHOR_VERTS +from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + +LOG = logging.getLogger("extract_j3d_offline") +IMG_SIZE = 672 +DEFAULT_OUT_DIR = Path("~/.cache/av-live-action/raw").expanduser() + + +def _default_K(size: int = IMG_SIZE) -> np.ndarray: + """Synthetic camera intrinsics, focal ~ image size, principal point centred.""" + f = float(size) + cx = cy = f * 0.5 + return np.array( + [[f, 0.0, cx], [0.0, f, cy], [0.0, 0.0, 1.0]], + dtype=np.float32, + ) + + +def _frame_to_chw(frame_bgr: np.ndarray, size: int = IMG_SIZE) -> np.ndarray: + """BGR uint8 (H, W, 3) -> float32 CHW (3, size, size) in [0, 1].""" + h, w = frame_bgr.shape[:2] + side = min(h, w) + y0 = (h - side) // 2 + x0 = (w - side) // 2 + crop = frame_bgr[y0:y0 + side, x0:x0 + side] + resized = cv2.resize(crop, (size, size)) + rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0 + return rgb.transpose(2, 0, 1) # CHW + + +def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | None: + v3d = person.get("v3d") + if v3d is None: + return None + # CoreMLArray wraps numpy but lacks __array__; unwrap before asarray. + if hasattr(v3d, "numpy") and not isinstance(v3d, np.ndarray): + v3d = v3d.numpy() + v3d_np = np.asarray(v3d, dtype=np.float32) + if v3d_np.shape[0] < max(anchors) + 1: + return None + return v3d_np[list(anchors)].astype(np.float32) + + +def extract(session: str, video: Path, out: Path, + det_thresh: float = 0.3, + mlpackage_path: Path | None = None, + anchors: tuple[int, ...] = SMPLX_JOINT_ANCHOR_VERTS) -> int: + """Returns the number of (frame, person) rows written.""" + out.parent.mkdir(parents=True, exist_ok=True) + cap = cv2.VideoCapture(str(video)) + if not cap.isOpened(): + raise RuntimeError(f"cannot open {video}") + fps = cap.get(cv2.CAP_PROP_FPS) or 30.0 + backend = MultiHMRCoreMLBackend(mlpackage_path) if mlpackage_path \ + else MultiHMRCoreMLBackend() + K = _default_K(IMG_SIZE) + n_frames = 0 + n_rows = 0 + with out.open("w") as f: + while True: + ok, frame = cap.read() + if not ok: + break + chw = _frame_to_chw(frame) + try: + persons = backend.infer(chw, K, det_thresh=det_thresh) + except Exception: + LOG.exception("infer failed at frame=%d", n_frames) + n_frames += 1 + continue + ts = n_frames / fps + for i, person in enumerate(persons): + j3d = _person_to_j3d22(person, anchors) + if j3d is None: + continue + f.write(json.dumps({ + "ts": ts, + "session": session, + "pid": int(person.get("pid", i)), + "j3d": j3d.tolist(), + }) + "\n") + n_rows += 1 + n_frames += 1 + if n_frames % 100 == 0: + LOG.info("frame=%d rows=%d", n_frames, n_rows) + cap.release() + LOG.info("done: %d frames, %d rows -> %s", n_frames, n_rows, out) + return n_rows + + +def _cli() -> None: + p = argparse.ArgumentParser() + p.add_argument("--session", required=True) + p.add_argument("--video", required=True, type=Path) + p.add_argument("--out", type=Path) + p.add_argument("--det-thresh", type=float, default=0.3) + p.add_argument("--mlpackage", type=Path, default=None) + args = p.parse_args() + logging.basicConfig(level=logging.INFO, + format="%(asctime)s [%(name)s] %(message)s") + DEFAULT_OUT_DIR.mkdir(parents=True, exist_ok=True) + out = args.out or (DEFAULT_OUT_DIR / f"{args.session}.jsonl") + extract(args.session, args.video, out, + det_thresh=args.det_thresh, mlpackage_path=args.mlpackage) + + +if __name__ == "__main__": + _cli() diff --git a/data_only_viz/scripts/train_on_studio.sh b/data_only_viz/scripts/train_on_studio.sh index def7bcd..e4ec3fe 100755 --- a/data_only_viz/scripts/train_on_studio.sh +++ b/data_only_viz/scripts/train_on_studio.sh @@ -36,7 +36,17 @@ REMOTE_CKPT="$REMOTE_ROOT/checkpoints" DATASET_FILE="${DATASET_FILE:-$LOCAL_DATASET/dataset.jsonl}" CKPT_NAME="${CKPT_NAME:-action_head.pt}" -TRAIN_ARGS="$*" + +# Quote train args defensively before forwarding through bastion ssh + +# studio ssh (each layer reparses). Reject single quotes — they break +# the single-quoted payload in bastion_ssh and could allow injection. +for a in "$@"; do + if [[ "$a" == *"'"* ]]; then + printf '[train_on_studio] forbidden single quote in arg: %s\n' "$a" >&2 + exit 3 + fi +done +TRAIN_ARGS="$(printf '%q ' "$@")" log() { printf '[train_on_studio] %s\n' "$*" >&2; } From 31ba587a634e0f2abff148343e163f0596058078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:02:54 +0200 Subject: [PATCH 02/14] docs(action-head): post-impl deviations + README - plans/2026-05-13-action-head.md : note SUPERSEDED sur Task 11 + Task 14 (pivot publisher thread, pas de worker refactor), header status + decisions. - specs/2026-05-13-action-head-design.md : status implemente, deviations cataloguees en haut, note inline section worker. - data_only_viz/CLAUDE.md : section action-head ajoutee avec pipeline complet capture->train->live + ref tests. --- data_only_viz/CLAUDE.md | 28 +++++++++++++++++++ .../plans/2026-05-13-action-head.md | 12 +++++++- .../specs/2026-05-13-action-head-design.md | 12 ++++++-- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/data_only_viz/CLAUDE.md b/data_only_viz/CLAUDE.md index a4434e0..f252139 100644 --- a/data_only_viz/CLAUDE.md +++ b/data_only_viz/CLAUDE.md @@ -35,6 +35,32 @@ Python **3.11+** requis. `pyproject.toml` est la source de vérité — ne jamai - Shaders Metal dans `shaders/` (`.metal`), recompilés au runtime ; topologie mesh (SMPL faces) en binaire dans `mesh_topology.py`. - OSC out : `osc_listener.py` / `pose_bridge.py` — destination `oscope-of` sur `:57123`. +## action-head (classifier action debout/assise/danse) + +Tête de classification d'action streaming au-dessus des j3d SMPL-X (ou body3d MediaPipe en fallback). Implémentée 2026-05-13. + +| Fichier | Rôle | +|---|---| +| `action_head.py` | `ActionHeadModel` (GRU 1L + MLP, 37 811 params, <2 ms/step M5), `ActionHead.step(pid, j3d) → (label, probs, kin)`, `PerPersonBuffer`, `FeatureExtractor` (201-D : j3d + vel + accel + scalaires) | +| `action_head_pub.py` | Publisher thread démarré dans `multi.py` `__init__`. Polle `state.persons_smplx` (préféré) ou `state.persons_body3d` (fallback) à 30 Hz, dédup par timestamp, extrait j3d22 via `SMPLX_JOINT_ANCHOR_VERTS` ou `MEDIAPIPE_TO_22`, émet OSC `/pose/action` + `/pose/kin` + `/pose/enter/leave` | +| `training/{dataset,autolabel,augment,train_action_head,eval,review}.py` | Pipeline complet : jsonl IO + sliding windows + by-session split / règles auto-label + glue CLI / 4 augmentations / training MPS AdamW CE-weighted / confusion matrix + latence micro-bench / TUI textuel pour review manuel | +| `scripts/capture_actions.py` | Webcam → MP4 + timestamps | +| `scripts/extract_j3d_offline.py` | MP4 → jsonl j3d22 via `MultiHMRCoreMLBackend.infer()` directement (pas de refactor worker) | +| `scripts/train_on_studio.sh` | rsync grosmac → bastion electron-server → studio M3 Ultra + uv sync `--extra multihmr` + train MPS + ckpt back | + +Pipeline complet de capture à live : +```bash +uv run python -m data_only_viz.scripts.capture_actions --session sess01 --duration 600 +uv run python -m data_only_viz.scripts.extract_j3d_offline --session sess01 --video ~/.cache/av-live-action/raw/sess01.mp4 +uv run python -m data_only_viz.training.autolabel --frames ~/.cache/av-live-action/raw/sess01.jsonl --out ~/.cache/av-live-action/dataset/auto.jsonl +uv run python -m data_only_viz.training.review --in ~/.cache/av-live-action/dataset/auto.jsonl --out ~/.cache/av-live-action/dataset/dataset.jsonl +./data_only_viz/scripts/train_on_studio.sh --epochs 50 +uv run python -m data_only_viz.training.eval --ckpt ~/.cache/av-live-action/checkpoints/action_head.pt --dataset ~/.cache/av-live-action/dataset/dataset.jsonl +# Live : publisher déjà câblé dans multi.py, aucune action requise +``` + +Checkpoint par défaut : `~/.cache/av-live-action/checkpoints/action_head.pt`. Absent → random init (warmup retourne `debout`). + ## Tests ```bash @@ -43,6 +69,8 @@ uv run pytest tests/ -v Tests TDD-first pour `nlf_worker.py` ; valider avant chaque commit qui touche un worker. +Suite action-head (8 fichiers, 39 tests) : `tests/test_action_head_*.py`, `tests/test_{dataset,autolabel,augment,training_smoke,pose_bridge_action}.py`. Tous doivent rester verts avant chaque commit qui touche `action_head*.py` ou `training/*.py`. + ## Anti-patterns - Ne pas charger un modèle ML sans guard `try/except ImportError` — les optional-extras peuvent manquer. diff --git a/docs/superpowers/plans/2026-05-13-action-head.md b/docs/superpowers/plans/2026-05-13-action-head.md index 389ce06..c06fa34 100644 --- a/docs/superpowers/plans/2026-05-13-action-head.md +++ b/docs/superpowers/plans/2026-05-13-action-head.md @@ -1,12 +1,18 @@ # action-head Implementation Plan > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> +> **STATUS 2026-05-13 22:50** — Implementation **complete** (16/17 tasks, Task 16 is a manual gate). 39 tests green. Key deviations from this document, captured in the "Post-impl deviations" section below: +> - Task 14 pivoted from "modify `multi_hmr_worker_coreml.py` + CLI flag" to **standalone publisher thread `data_only_viz/action_head_pub.py`** + 3-line wire-in in `multi.py` (avoids collision with the user's parallel iteration on `multi_hmr_worker.py`). The MultiHMR backend is selected via env var `MULTIHMR_BACKEND=pytorch|coreml`, not a CLI flag. +> - Task 11 pivoted from "refactor `MultiHMRWorker` with `create_for_offline()`" to **standalone script using `MultiHMRCoreMLBackend.infer()` directly** — no worker refactor. +> - j3d is approximated from SMPL-X v3d via a fixed 22-vertex anchor set (`SMPLX_JOINT_ANCHOR_VERTS`), with a MediaPipe 33→22 fallback. The same anchor set is shared between live serve (`action_head_pub.py`) and offline extract (`scripts/extract_j3d_offline.py`) to avoid train/serve skew. +> - Studio train wrapper added as Task 8.5 (`data_only_viz/scripts/train_on_studio.sh`), validated end-to-end smoke 160 windows × 3 epochs MPS in ~4 s. **Goal:** Implement a real-time per-person action classifier (debout/assise/danse) on top of Multi-HMR `j3d`, with OSC output enriched by softmax probabilities and kinetics scalars (speed/accel/symmetry). **Architecture:** GRU-1-layer + MLP head streaming inference, fed by a 16-frame ring buffer per person. Trained windowed on Studio M3 Ultra (PyTorch MPS), inferred streaming on M5. Hybrid auto-labeler (rules on j3d) + manual review for dataset. Inference ≤ 2 ms/person M5 in eager PyTorch — no CoreML conversion needed. -**Tech Stack:** Python 3.11 + uv, PyTorch (MPS for train, CPU for M5 inference), numpy, python-osc, pytest. Reuses existing `data_only_viz` infrastructure (`multi_hmr_worker_coreml.py`, `pose_bridge.py`, `tracker.py`, `state.py`). +**Tech Stack:** Python 3.11 + uv, PyTorch (MPS for train, CPU for M5 inference), numpy, python-osc, pytest. Reuses existing `data_only_viz` infrastructure (`multi_hmr_worker.py`, `multihmr_coreml.py`, `pose_bridge.py`, `tracker.py`, `state.py`). **Reference spec:** `docs/superpowers/specs/2026-05-13-action-head-design.md` @@ -1839,6 +1845,8 @@ git commit -m "feat(data-only-viz): action capture script" ## Task 11 — Extract j3d offline +> **SUPERSEDED 2026-05-13.** The implemented script does NOT refactor `multi_hmr_worker.py`. Instead it uses the standalone `MultiHMRCoreMLBackend.infer()` from `data_only_viz/multihmr_coreml.py` directly. Output jsonl rows contain a (22, 3) `j3d` extracted via `SMPLX_JOINT_ANCHOR_VERTS` (shared with `action_head_pub.py` to avoid train/serve skew), not the raw v3d. See actual file at `data_only_viz/scripts/extract_j3d_offline.py`. The body below documents the original intent — keep as historical context. + **Files:** - Create: `data_only_viz/scripts/extract_j3d_offline.py` @@ -2196,6 +2204,8 @@ git commit -m "feat(data-only-viz): pose_bridge /pose/action + /pose/kin" ## Task 14 — Wire ActionHead into multi_hmr_worker_coreml +> **SUPERSEDED 2026-05-13.** No `multi_hmr_worker_coreml.py` file exists in the current repo — the user pivoted to `multihmr_coreml.py` (standalone backend) selected via env `MULTIHMR_BACKEND=pytorch|coreml` inside `multi_hmr_worker.py`. To avoid colliding with that file under active iteration, ActionHead wiring was implemented as a standalone publisher thread in `data_only_viz/action_head_pub.py` plus a 3-line wire-in inside `data_only_viz/multi.py` (`__init__` instantiates and `.start()`s the publisher). The publisher polls `state.persons_smplx` (preferred) and `state.persons_body3d` (MediaPipe fallback) at 30 Hz, deduplicates by timestamp, extracts j3d22 via shared `SMPLX_JOINT_ANCHOR_VERTS` / `MEDIAPIPE_TO_22` index maps, runs `ActionHead.step()` per pid, and emits OSC via the existing `PoseSoundBridge`. No CLI flag was added. See `data_only_viz/action_head_pub.py` and `data_only_viz/multi.py:22,97-98`. The body below documents the original intent — keep as historical context. + **Files:** - Modify: `data_only_viz/multi_hmr_worker_coreml.py` - Modify: `data_only_viz/main.py` diff --git a/docs/superpowers/specs/2026-05-13-action-head-design.md b/docs/superpowers/specs/2026-05-13-action-head-design.md index 7c262e2..e62a35e 100644 --- a/docs/superpowers/specs/2026-05-13-action-head-design.md +++ b/docs/superpowers/specs/2026-05-13-action-head-design.md @@ -1,11 +1,17 @@ # action-head — Classifier d'action temps réel au-dessus de Multi-HMR > **Date** : 2026-05-13 -> **Status** : design approuvé, prêt pour implementation plan +> **Status** : design approuvé — **implémenté 2026-05-13 22:50**, 16/17 tasks, 39 tests verts. Task 16 (E2E gate) reste manuel (requiert capture + train réel). > **Authors** : L'Electron Rare + Claude > **Companion plans** : > - `2026-05-13-multihmr-coreml-hybrid-backbone.md` > - `2026-05-13-studio-train-deploy-m5.md` +> +> **Déviations notables vs design original** (cf. plan `2026-05-13-action-head.md` pour le détail) : +> - **Wiring worker** : standalone publisher thread `data_only_viz/action_head_pub.py` + 3 lignes dans `multi.py`, au lieu de modifier directement `multi_hmr_worker.py` (qui était en cours d'évolution par l'utilisateur en parallèle). Backend Multi-HMR sélectionné par env `MULTIHMR_BACKEND=pytorch|coreml`, pas par flag CLI. +> - **Source j3d** : approximée via 22 vertex anchors (`SMPLX_JOINT_ANCHOR_VERTS`) sur le mesh SMPL-X 10475-vert, partagés entre serve live (`action_head_pub.py`) et extraction offline (`scripts/extract_j3d_offline.py`) pour éviter le train/serve skew. Fallback MediaPipe 33→22 (`MEDIAPIPE_TO_22`) quand `persons_smplx` est vide. **Limitation** : ces 22 indices sont approximatifs ; pour des j3d SMPL-X corrects, brancher `J_regressor @ v3d` quand le module SMPL-X est dispo. +> - **Extract offline** : pas de refactor de `MultiHMRWorker`, on utilise `MultiHMRCoreMLBackend.infer()` directement (commit user `9e7a9f8`). +> - **Studio launch** : wrapper bash `data_only_viz/scripts/train_on_studio.sh` (Task 8.5) qui rsync + ssh + uv sync + train MPS + ckpt back. Validé end-to-end sur dataset smoke 160 windows × 3 epochs en ~4 s wallclock. ## TL;DR @@ -94,7 +100,9 @@ class ActionHead: def forget(self, pid: int) -> None: ... ``` -Aucune modification de l'API publique de `multi_hmr_worker_coreml.py` autre que : +**Note d'implémentation 2026-05-13** : la section ci-dessous décrit l'intention originale. L'implémentation réelle est dans `data_only_viz/action_head_pub.py` (publisher thread) — pas de modification de `multi_hmr_worker.py`. Voir l'en-tête du document pour les déviations. + +Aucune modification de l'API publique de `multi_hmr_worker.py` n'est requise au-delà de : - Construction d'une `ActionHead` au startup. - Appel `.step()` après chaque détection. - Appel `.forget()` synchronisé avec `tracker.purge()`. From 28d562b11c58b9b9bbbfb26c455dbb1293137572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:07:48 +0200 Subject: [PATCH 03/14] feat(av-live): wire Apple Vision body pose on ANE Add a parallel-pose worker selector (env AV_LIVE_PARALLEL_POSE) defaulting to "both": runs Apple Vision body 2D on ANE alongside MediaPipe Holistic on CPU XNNPACK, while Multi-HMR PyTorch streams the dense mesh on MPS. Modes "apple_vision" or "mediapipe" to pick one. Body keypoints land in state.persons_body either way. Face landmark parser via pyobjc remains blocked: pointAtIndex_ selector arity confuses pyobjc 11 and pointsInImageOfSize_ returns an opaque PyObjCPointer with no address handle. Keep the ANE-detected count for logging, fall back to MediaPipe face/hand fin landmarks until a Swift bridge is in place. --- data_only_viz/apple_vision_pose.py | 39 ++++++++++------------------ data_only_viz/main.py | 41 ++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/data_only_viz/apple_vision_pose.py b/data_only_viz/apple_vision_pose.py index f94041f..a8431a4 100644 --- a/data_only_viz/apple_vision_pose.py +++ b/data_only_viz/apple_vision_pose.py @@ -552,29 +552,12 @@ class AppleVisionPoseWorker: if not hasattr(self, "_logged_face_ok_" + region_name): LOG.info("face: region %s count=%d", region_name, count) setattr(self, "_logged_face_ok_" + region_name, True) - # API stable : pointAtIndex_(k) retourne un CGPoint struct. - n = min(count, end - start) - n_written = 0 - for k in range(n): - try: - pt = region.pointAtIndex_(k) - # CGPoint en pyobjc : tuple (x, y) ou struct - try: - nx_bb = float(pt.x); ny_bb = float(pt.y) - except (AttributeError, TypeError): - nx_bb = float(pt[0]); ny_bb = float(pt[1]) - fx = bx + nx_bb * bw - fy_bl = by + ny_bb * bh - kps[start + k] = PoseKp( - x=fx, y=1.0 - fy_bl, z=0.0, c=1.0) - n_written += 1 - except Exception as e: - if not hasattr(self, "_logged_face_pt_err"): - LOG.info("face: pt %s[%d] err: %s (pt=%r)", - region_name, k, e, type(pt).__name__ - if 'pt' in dir() else "??") - self._logged_face_pt_err = True - continue + # pyobjc 11 ne sait pas que pointAtIndex_ prend 1 arg, et + # pointsInImageOfSize_ retourne un PyObjCPointer C-array sans + # API d'acces simple. Face parsing depuis Apple Vision est + # actuellement bloque ; on garde MediaPipe (CPU XNNPACK) pour + # face/hand fin tandis que Vision sert body 2D sur ANE. + n = 0; n_written = 0 if n_written > 0 and not hasattr(self, "_logged_face_write_" + region_name): LOG.info("face: %s wrote %d points", region_name, n_written) setattr(self, "_logged_face_write_" + region_name, True) @@ -590,13 +573,19 @@ class AppleVisionPoseWorker: fill("nose", *FACE_OFFSETS["nose"]) fill("medianLine", *FACE_OFFSETS["median"]) - # Pupilles : VNFaceLandmarkRegion2D simple (1 point chacune). + # Pupilles : single-point regions ; meme workaround pyobjc. for region_name, idx in (("leftPupil", 81), ("rightPupil", 82)): try: region = getattr(landmarks, region_name)() if region is None or region.pointCount() < 1: continue - pt = region.pointAtIndex_(0) + try: + pts = region.pointsInImageOfSize_((1.0, 1.0)) + except Exception: + pts = region.normalizedPoints() + if not pts: + continue + pt = pts[0] try: px, py = float(pt.x), float(pt.y) except (AttributeError, TypeError): diff --git a/data_only_viz/main.py b/data_only_viz/main.py index 263e0bb..1481ee3 100644 --- a/data_only_viz/main.py +++ b/data_only_viz/main.py @@ -268,21 +268,46 @@ class AppDelegate(NSObject): self._smplx_tcp = SMPLXTCPSender(self._state) self._smplx_tcp.start() LOG.info("worker: Multi-HMR + SMPL-X (mesh dense)") - # Also start MediaPipe Multi for body3d + face + hand - # OSC streams to AVLiveBody (mesh and skeleton/face/ - # hand pipelines run in parallel, each owns its own - # AVCapture session on the same builtin camera). - if _os.environ.get("AV_LIVE_MEDIAPIPE") != "0": + # Secondary body-pose worker in parallel: AVLiveBody + # gets body keypoints on UDP :57126 alongside the mesh + # on TCP :57130. Default: Apple Vision (ANE-accel, + # body only 19 joints). Set AV_LIVE_PARALLEL_POSE= + # mediapipe to swap to MediaPipe Holistic (CPU + # XNNPACK but provides face + hand + 3D world). + # Defaut: lance BOTH Apple Vision (body 19 joints sur + # ANE, ~30 fps) ET MediaPipe Multi (face 468 + hands 21 + # + pose 3D world sur CPU XNNPACK). Set + # AV_LIVE_PARALLEL_POSE=apple_vision pour ne garder que + # le path ANE (face/hand fin disparait), ou =mediapipe + # pour ne garder que CPU. + parallel = _os.environ.get( + "AV_LIVE_PARALLEL_POSE", "both") + if parallel in ("apple_vision", "both"): + try: + from .apple_vision_pose import AppleVisionPoseWorker + if AppleVisionPoseWorker.is_available(): + self._av_worker = AppleVisionPoseWorker( + self._state, target_fps=30.0, + num_persons=4) + self._av_worker.start() + LOG.info("worker: + Apple Vision body pose " + "(ANE) in parallel") + else: + raise RuntimeError("apple_vision unavailable") + except Exception as e: # noqa: BLE001 + LOG.warning("Apple Vision parallel start failed " + "(%s)", e) + if parallel in ("mediapipe", "both"): try: from .multi import MultiWorker - self._mediapipe_worker = MultiWorker( + self._mp_worker = MultiWorker( self._state, num_persons=4) - self._mediapipe_worker.start() + self._mp_worker.start() LOG.info("worker: + MediaPipe Multi (3D pose " "+ face + hand) in parallel") except Exception as e: # noqa: BLE001 LOG.warning("MediaPipe parallel start failed " - "(%s) — mesh only", e) + "(%s)", e) return LOG.info("Multi-HMR indisponible (checkpoints manquants) " "— voir scripts/setup_multihmr.sh") From aedcb0f01b3b42691fdd00ac411632b0dc065798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:15:12 +0200 Subject: [PATCH 04/14] feat(data-only-viz): action-head v2 fingers+face MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend action-head to 32 joints (body22 + 10 fingertips), 10 SMPL-X expression PCA scalars, and mouth_open distance. FEATURE_DIM 201→302. MIRROR_MAP extended to 32. Dataset, augment, training, publisher, offline extractor all updated. --- data_only_viz/action_head.py | 57 +++++++--- data_only_viz/action_head_pub.py | 100 +++++++++++++++--- data_only_viz/scripts/extract_j3d_offline.py | 42 ++++++-- .../tests/test_action_head_features.py | 17 +-- data_only_viz/tests/test_action_head_model.py | 6 +- data_only_viz/tests/test_action_head_pub.py | 2 + data_only_viz/tests/test_augment.py | 11 +- data_only_viz/tests/test_autolabel.py | 4 +- data_only_viz/tests/test_dataset.py | 12 +-- data_only_viz/tests/test_training_smoke.py | 6 +- data_only_viz/training/augment.py | 7 +- data_only_viz/training/dataset.py | 53 ++++++++-- data_only_viz/training/train_action_head.py | 21 +++- 13 files changed, 265 insertions(+), 73 deletions(-) diff --git a/data_only_viz/action_head.py b/data_only_viz/action_head.py index b52eef2..89bc6de 100644 --- a/data_only_viz/action_head.py +++ b/data_only_viz/action_head.py @@ -21,12 +21,26 @@ WARMUP_FRAMES: int = 3 NAN_SKIP_BUDGET: int = 5 WINDOW_LEN: int = 16 -J3D_JOINTS: int = 22 +J3D_BODY: int = 22 +J3D_FINGERS_PER_HAND: int = 5 +J3D_FINGERS: int = 2 * J3D_FINGERS_PER_HAND # 10 +J3D_JOINTS: int = J3D_BODY + J3D_FINGERS # 32 J3D_DIMS: int = 3 NUM_CLASSES: int = 3 LABELS: tuple[str, str, str] = ("debout", "assise", "danse") -FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + 3 # j3d + vel + accel + 3 scalars +EXPR_DIM: int = 10 +EXTRA_SCALARS: int = 4 # hip_y, knee_angle, sym_score, mouth_open + +# Layout per step: +# [0 : 96 ] j3d (32, 3) +# [96 : 192] vel (32, 3) +# [192 : 288] accel (32, 3) +# [288 : 298] expression (10,) +# [298 : 302] scalars (hip_y, knee_angle, sym, mouth_open) +FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + EXPR_DIM + EXTRA_SCALARS # 302 + +# Body joint indices (unchanged from v1, indices 0..21). HIP_LEFT: int = 1 HIP_RIGHT: int = 2 KNEE_LEFT: int = 4 @@ -38,19 +52,26 @@ SHOULDER_RIGHT: int = 17 WRIST_LEFT: int = 20 WRIST_RIGHT: int = 21 +# Fingertip indices (new, 22..31), order: L thumb..pinky, R thumb..pinky. +FINGERTIP_LEFT_BASE: int = 22 +FINGERTIP_RIGHT_BASE: int = 27 + class FeatureExtractor: """Stateless feature builder over a list of recent j3d frames. - Vector layout (FEATURE_DIM = 201): - [0 : 66] j3d current frame, flattened (22 joints × 3 dims) - [66 : 132] velocity j3d[t] - j3d[t-1] (22 × 3) - [132 : 198] acceleration vel[t] - vel[t-1] (22 × 3) - [198 : 201] kinetics scalars (hip_y, knee_angle, symmetry_score) + Vector layout (FEATURE_DIM = 302): + [0 : 96 ] j3d current frame, flattened (32 joints x 3 dims) + [96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3) + [192 : 288] acceleration vel[t] - vel[t-1] (32 x 3) + [288 : 298] expression PCA coefficients (10,) + [298 : 302] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open) """ @staticmethod - def from_buffer(frames: list[np.ndarray]) -> np.ndarray: + def from_buffer(frames: list[np.ndarray], + expr: np.ndarray | None = None, + mouth_open: float = 0.0) -> np.ndarray: if not frames: return np.zeros(FEATURE_DIM, dtype=np.float32) cur = frames[-1] @@ -62,13 +83,19 @@ class FeatureExtractor: hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5) knee_angle = FeatureExtractor._mean_knee_angle(cur) sym = FeatureExtractor._symmetry_score(vel) - feat = np.concatenate([ + if expr is None: + expr_vec = np.zeros(EXPR_DIM, dtype=np.float32) + else: + expr_vec = np.zeros(EXPR_DIM, dtype=np.float32) + n = min(EXPR_DIM, len(expr)) + expr_vec[:n] = expr[:n] + return np.concatenate([ cur.reshape(-1), vel.reshape(-1), accel.reshape(-1), - np.array([hip_y, knee_angle, sym], dtype=np.float32), + expr_vec, + np.array([hip_y, knee_angle, sym, float(mouth_open)], dtype=np.float32), ]).astype(np.float32, copy=False) - return feat @staticmethod def kinetics(frames: list[np.ndarray]) -> np.ndarray: @@ -148,7 +175,7 @@ class PerPersonBuffer: class ActionHeadModel(nn.Module): """1-layer GRU + small MLP head. - Input : (B, FEATURE_DIM) — single step + Input : (B, FEATURE_DIM) -- single step Hidden : (1, B, HIDDEN_DIM) Output : (B, NUM_CLASSES) logits, new hidden """ @@ -198,7 +225,9 @@ class ActionHead: self._hidden: dict[int, torch.Tensor] = {} self._nan_streak: dict[int, int] = {} - def step(self, pid: int, j3d: np.ndarray) -> tuple[str, np.ndarray, np.ndarray]: + def step(self, pid: int, j3d: np.ndarray, + expr: np.ndarray | None = None, + mouth_open: float = 0.0) -> tuple[str, np.ndarray, np.ndarray]: if np.isnan(j3d).any(): streak = self._nan_streak.get(pid, 0) + 1 self._nan_streak[pid] = streak @@ -212,7 +241,7 @@ class ActionHead: if len(frames) < WARMUP_FRAMES: probs = np.array([1.0, 0.0, 0.0], dtype=np.float32) return LABELS[0], probs, np.zeros(3, dtype=np.float32) - feat = FeatureExtractor.from_buffer(frames) + feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open) kin = FeatureExtractor.kinetics(frames) h = self._hidden.get(pid) if h is None: diff --git a/data_only_viz/action_head_pub.py b/data_only_viz/action_head_pub.py index 8c8f0d6..bd72d39 100644 --- a/data_only_viz/action_head_pub.py +++ b/data_only_viz/action_head_pub.py @@ -14,7 +14,13 @@ from typing import Any import numpy as np -from data_only_viz.action_head import ActionHead, LABELS +from data_only_viz.action_head import ( + ActionHead, + EXPR_DIM, + J3D_FINGERS, + J3D_FINGERS_PER_HAND, + LABELS, +) LOG = logging.getLogger("action_head_pub") @@ -22,19 +28,39 @@ DEFAULT_CKPT = ( Path.home() / ".cache" / "av-live-action" / "checkpoints" / "action_head.pt" ) -# 22 vertex indices on the 10475-vertex SMPL-X mesh, approximating -# the 22-joint kinematic chain used by ActionHead. -# NOTE: approximate vertex anchors — real SMPL-X joints come from +# Approximate fingertip vertex indices on SMPL-X 10475-vert mesh. +# Order: L thumb, L index, L middle, L ring, L pinky, +# R thumb, R index, R middle, R ring, R pinky. +SMPLX_FINGERTIP_VERTS: tuple[int, ...] = ( + 7174, 7397, 7670, 7942, 8214, # L + 4631, 4854, 5127, 5399, 5671, # R +) + +# 32 vertex indices on the 10475-vertex SMPL-X mesh: +# 22 body (UNCHANGED from v1) + 10 fingertips. +# NOTE: approximate vertex anchors -- real SMPL-X joints come from # J_regressor @ v3d, but loading the regressor here is avoided for # live OSC performance. Action-head training must use the same anchors. SMPLX_JOINT_ANCHOR_VERTS: tuple[int, ...] = ( + # 22 body (UNCHANGED indices, same vertex IDs as before) 8204, 3992, 6677, 3500, 3469, 6394, 3279, 3327, 6736, 3074, 8846, 8889, 8848, 1300, 4660, 8964, 3013, 6470, 1602, 5083, 2114, 5559, + # 10 fingertips + *SMPLX_FINGERTIP_VERTS, ) +assert len(SMPLX_JOINT_ANCHOR_VERTS) == 32 + +# Mouth-open: distance between two lip vertices on SMPL-X mesh. +# vert 8970 (upper outer lip), 8855 (lower outer lip) -- approximate. +SMPLX_UPPER_LIP_VERT: int = 8970 +SMPLX_LOWER_LIP_VERT: int = 8855 + +# MediaPipe HAND fingertip indices (21-kp hand model). +MEDIAPIPE_HAND_FINGERTIPS: tuple[int, ...] = (4, 8, 12, 16, 20) # MediaPipe 33-landmark indices mapped into the 22-joint slot order. -# NOTE: approximate mapping — spine joints reuse hip/shoulder anchors. +# NOTE: approximate mapping -- spine joints reuse hip/shoulder anchors. # https://developers.google.com/mediapipe/solutions/vision/pose_landmarker MEDIAPIPE_TO_22: tuple[int, ...] = ( 24, 23, 24, 23, 25, 26, 11, 27, 28, 11, @@ -85,7 +111,7 @@ class ActionHeadPublisher(threading.Thread): LOG.info("publisher stopped") def _tick(self, t_now: float) -> None: - persons22, source_t, source_tag, is_new = self._read_sources() + persons32, source_t, source_tag, is_new = self._read_sources() if not is_new: return if "smplx" in source_tag: @@ -93,10 +119,12 @@ class ActionHeadPublisher(threading.Thread): else: self._last_body_t = source_t current_pids: set[int] = set() - if persons22: - for pid, j3d in persons22: + if persons32: + for pid, j3d, expr, mouth in persons32: current_pids.add(pid) - label, probs, kin = self.head.step(pid, j3d) + label, probs, kin = self.head.step(pid, j3d, + expr=expr, + mouth_open=mouth) idx = LABELS.index(label) self.bridge.send_action(pid, idx, probs, t_now, force=True) self.bridge.send_kin(pid, kin, t_now, force=True) @@ -109,9 +137,11 @@ class ActionHeadPublisher(threading.Thread): def _read_sources( self, - ) -> tuple[list[tuple[int, np.ndarray]] | None, float, str, bool]: - """Return (persons22, source_t, source_tag, is_new). + ) -> tuple[list[tuple[int, np.ndarray, np.ndarray, float]] | None, + float, str, bool]: + """Return (persons32, source_t, source_tag, is_new). + Each person entry is (pid, j3d32, expr10, mouth_open). is_new is True when the timestamp advanced (even if person list is empty), so _tick can still run the purge loop. """ @@ -121,9 +151,11 @@ class ActionHeadPublisher(threading.Thread): persons_b3d = getattr(self.state, "persons_body3d", None) ids_b3d = getattr(self.state, "persons_body_ids", None) t_body = getattr(self.state, "pose_last_t", 0.0) + hands_ids = list(getattr(self.state, "persons_hands_ids", None) or []) + hands_lists = list(getattr(self.state, "persons_hands", None) or []) # Prefer smplx when its timestamp advanced. if t_smplx > self._last_smplx_t: - out: list[tuple[int, np.ndarray]] = [] + out: list[tuple[int, np.ndarray, np.ndarray, float]] = [] for i, p in enumerate(persons_smplx or []): pid = int(p.get("pid", i)) v3d = p.get("v3d") @@ -136,19 +168,55 @@ class ActionHeadPublisher(threading.Thread): v3d_np = np.asarray(v3d, dtype=np.float32) if v3d_np.shape[0] < max(SMPLX_JOINT_ANCHOR_VERTS) + 1: continue - j3d22 = v3d_np[list(SMPLX_JOINT_ANCHOR_VERTS)].astype(np.float32) - out.append((pid, j3d22)) + j3d32 = v3d_np[list(SMPLX_JOINT_ANCHOR_VERTS)].astype(np.float32) + # expression + expr = p.get("expression") + if expr is not None: + if hasattr(expr, "numpy") and not isinstance(expr, np.ndarray): + expr = expr.numpy() + expr_np = np.asarray(expr, dtype=np.float32).flatten() + else: + expr_np = np.zeros(EXPR_DIM, dtype=np.float32) + # mouth_open + if v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT): + mouth = float(np.linalg.norm( + v3d_np[SMPLX_UPPER_LIP_VERT] - v3d_np[SMPLX_LOWER_LIP_VERT] + )) + else: + mouth = 0.0 + out.append((pid, j3d32, expr_np, mouth)) return out or None, t_smplx, "smplx", True if t_body > self._last_body_t: ids = ids_b3d or list(range(len(persons_b3d or []))) + # Build hands lookup by pid + hands_by_pid: dict[int, dict[str, Any]] = {} + for hi, hkp in enumerate(hands_lists): + hpid = int(hands_ids[hi]) if hi < len(hands_ids) else hi + side = "L" if hi % 2 == 0 else "R" + hands_by_pid.setdefault(hpid, {})[side] = hkp out = [] for i, body in enumerate(persons_b3d or []): pid = int(ids[i]) if i < len(ids) else i arr = self._kp_list_to_array(body) if arr is None or arr.shape[0] < 33: continue - j3d22 = arr[list(MEDIAPIPE_TO_22)].astype(np.float32) - out.append((pid, j3d22)) + body22 = arr[list(MEDIAPIPE_TO_22)].astype(np.float32) + # fingertips from hands if available + tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32) + hpair = hands_by_pid.get(pid, {}) + for side_idx, side in enumerate(("L", "R")): + hkp = hpair.get(side) + if hkp is None: + continue + hkp_arr = self._kp_list_to_array(hkp) + if hkp_arr is None or hkp_arr.shape[0] < 21: + continue + for k, mp_idx in enumerate(MEDIAPIPE_HAND_FINGERTIPS): + tips[side_idx * J3D_FINGERS_PER_HAND + k] = hkp_arr[mp_idx] + j3d32 = np.concatenate([body22, tips], axis=0) + expr_np = np.zeros(EXPR_DIM, dtype=np.float32) + mouth = 0.0 + out.append((pid, j3d32, expr_np, mouth)) return out or None, t_body, "body3d", True return None, 0.0, "", False diff --git a/data_only_viz/scripts/extract_j3d_offline.py b/data_only_viz/scripts/extract_j3d_offline.py index 27ca9d4..bc1da44 100644 --- a/data_only_viz/scripts/extract_j3d_offline.py +++ b/data_only_viz/scripts/extract_j3d_offline.py @@ -1,4 +1,4 @@ -"""Extract j3d (22 SMPL-X joint anchors) from a recorded MP4 using the +"""Extract j3d (32 SMPL-X joint anchors) from a recorded MP4 using the Multi-HMR CoreML backend, write per-frame per-person jsonl rows. Usage: @@ -17,7 +17,12 @@ from pathlib import Path import cv2 import numpy as np -from data_only_viz.action_head_pub import SMPLX_JOINT_ANCHOR_VERTS +from data_only_viz.action_head import EXPR_DIM +from data_only_viz.action_head_pub import ( + SMPLX_JOINT_ANCHOR_VERTS, + SMPLX_UPPER_LIP_VERT, + SMPLX_LOWER_LIP_VERT, +) from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend LOG = logging.getLogger("extract_j3d_offline") @@ -47,7 +52,11 @@ def _frame_to_chw(frame_bgr: np.ndarray, size: int = IMG_SIZE) -> np.ndarray: return rgb.transpose(2, 0, 1) # CHW -def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | None: +def _person_to_j3d32( + person: dict, + anchors: tuple[int, ...], +) -> tuple[np.ndarray, np.ndarray, float] | None: + """Return (j3d32, expression, mouth_open) or None if v3d absent/too small.""" v3d = person.get("v3d") if v3d is None: return None @@ -57,7 +66,23 @@ def _person_to_j3d22(person: dict, anchors: tuple[int, ...]) -> np.ndarray | Non v3d_np = np.asarray(v3d, dtype=np.float32) if v3d_np.shape[0] < max(anchors) + 1: return None - return v3d_np[list(anchors)].astype(np.float32) + j3d32 = v3d_np[list(anchors)].astype(np.float32) + # expression + expr = person.get("expression") + if expr is not None: + if hasattr(expr, "numpy") and not isinstance(expr, np.ndarray): + expr = expr.numpy() + expr_np = np.asarray(expr, dtype=np.float32).flatten() + else: + expr_np = np.zeros(EXPR_DIM, dtype=np.float32) + # mouth_open + if v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT): + mouth = float(np.linalg.norm( + v3d_np[SMPLX_UPPER_LIP_VERT] - v3d_np[SMPLX_LOWER_LIP_VERT] + )) + else: + mouth = 0.0 + return j3d32, expr_np, mouth def extract(session: str, video: Path, out: Path, @@ -89,14 +114,17 @@ def extract(session: str, video: Path, out: Path, continue ts = n_frames / fps for i, person in enumerate(persons): - j3d = _person_to_j3d22(person, anchors) - if j3d is None: + result = _person_to_j3d32(person, anchors) + if result is None: continue + j3d32, expr_np, mouth = result f.write(json.dumps({ "ts": ts, "session": session, "pid": int(person.get("pid", i)), - "j3d": j3d.tolist(), + "j3d": j3d32.tolist(), + "expression": expr_np.tolist(), + "mouth_open": mouth, }) + "\n") n_rows += 1 n_frames += 1 diff --git a/data_only_viz/tests/test_action_head_features.py b/data_only_viz/tests/test_action_head_features.py index 0e2b87f..2f5abf9 100644 --- a/data_only_viz/tests/test_action_head_features.py +++ b/data_only_viz/tests/test_action_head_features.py @@ -11,14 +11,15 @@ def test_module_imports() -> None: assert hasattr(action_head, "PerPersonBuffer") assert hasattr(action_head, "ActionHead") assert action_head.WINDOW_LEN == 16 - assert action_head.J3D_JOINTS == 22 + assert action_head.J3D_JOINTS == 32 + assert action_head.FEATURE_DIM == 302 assert action_head.NUM_CLASSES == 3 assert action_head.LABELS == ("debout", "assise", "danse") def _rand_j3d(seed: int = 0) -> np.ndarray: rng = np.random.default_rng(seed) - return rng.normal(size=(22, 3)).astype(np.float32) + return rng.normal(size=(32, 3)).astype(np.float32) def test_buffer_starts_empty() -> None: @@ -58,15 +59,15 @@ def test_buffer_forget_releases_pid() -> None: def test_buffer_rejects_bad_shape() -> None: from data_only_viz.action_head import PerPersonBuffer buf = PerPersonBuffer() - with pytest.raises(ValueError, match="22"): - buf.append(pid=1, j3d=np.zeros((17, 3), dtype=np.float32)) + with pytest.raises(ValueError, match="32"): + buf.append(pid=1, j3d=np.zeros((22, 3), dtype=np.float32)) def test_feature_extractor_shape_full_buffer() -> None: from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, FEATURE_DIM frames = [_rand_j3d(i) for i in range(WINDOW_LEN)] feat = FeatureExtractor.from_buffer(frames) - assert feat.shape == (FEATURE_DIM,) + assert feat.shape == (302,) assert feat.dtype == np.float32 assert not np.isnan(feat).any() @@ -91,13 +92,13 @@ def test_feature_extractor_kinetics_speed_and_accel() -> None: from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN frames = [] for t in range(WINDOW_LEN): - f = np.zeros((22, 3), dtype=np.float32) + f = np.zeros((32, 3), dtype=np.float32) f[0, 0] = 0.1 * t frames.append(f) kin = FeatureExtractor.kinetics(frames) assert kin.shape == (3,) assert kin[0] > 0 - assert abs(kin[0] - 0.1 / 22) < 1e-4 + assert abs(kin[0] - 0.1 / 32) < 1e-4 assert abs(kin[1]) < 1e-4 @@ -105,7 +106,7 @@ def test_feature_extractor_symmetry_sign() -> None: from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, WRIST_LEFT, WRIST_RIGHT frames = [] for t in range(WINDOW_LEN): - f = np.zeros((22, 3), dtype=np.float32) + f = np.zeros((32, 3), dtype=np.float32) f[WRIST_LEFT, 0] = 0.05 * t f[WRIST_RIGHT, 0] = -0.05 * t frames.append(f) diff --git a/data_only_viz/tests/test_action_head_model.py b/data_only_viz/tests/test_action_head_model.py index 23a708c..1c8d281 100644 --- a/data_only_viz/tests/test_action_head_model.py +++ b/data_only_viz/tests/test_action_head_model.py @@ -11,7 +11,7 @@ torch = pytest.importorskip("torch") def _rand_j3d(seed: int = 0) -> np.ndarray: rng = np.random.default_rng(seed) - return rng.normal(size=(22, 3)).astype(np.float32) + return rng.normal(size=(32, 3)).astype(np.float32) def test_model_forward_shape() -> None: @@ -24,11 +24,11 @@ def test_model_forward_shape() -> None: assert h_new.shape == h.shape -def test_model_param_count_under_50k() -> None: +def test_model_param_count_under_80k() -> None: from data_only_viz.action_head import ActionHeadModel model = ActionHeadModel() n = sum(p.numel() for p in model.parameters()) - assert n < 50_000, f"too many params: {n}" + assert n < 80_000, f"too many params: {n}" def test_action_head_step_warmup_returns_debout() -> None: diff --git a/data_only_viz/tests/test_action_head_pub.py b/data_only_viz/tests/test_action_head_pub.py index a630999..2366a51 100644 --- a/data_only_viz/tests/test_action_head_pub.py +++ b/data_only_viz/tests/test_action_head_pub.py @@ -17,6 +17,8 @@ class _FakeState: self.persons_body3d = [] self.persons_body_ids = [] self.pose_last_t = 0.0 + self.persons_hands = [] + self.persons_hands_ids = [] self._lock = threading.RLock() def lock(self): diff --git a/data_only_viz/tests/test_augment.py b/data_only_viz/tests/test_augment.py index c87b358..c6b9a5a 100644 --- a/data_only_viz/tests/test_augment.py +++ b/data_only_viz/tests/test_augment.py @@ -8,16 +8,17 @@ WINDOW_LEN = 16 def _sample_stack(seed: int = 0) -> np.ndarray: rng = np.random.default_rng(seed) - return rng.normal(size=(WINDOW_LEN, 22, 3)).astype(np.float32) + return rng.normal(size=(WINDOW_LEN, 32, 3)).astype(np.float32) def test_mirror_swap_left_right_joints() -> None: - from data_only_viz.training.augment import mirror_x + from data_only_viz.training.augment import mirror_x, MIRROR_MAP x = _sample_stack(0) y = mirror_x(x) - assert np.allclose(y[..., 0], -x[..., 0][:, [ - 0,2,1,3,5,4,6,8,7,9,11,10,12,14,13,15,17,16,19,18,21,20 - ]], atol=1e-6) + # Check output shape + assert y.shape == (WINDOW_LEN, 32, 3) + # x-coords are negated after reindexing + assert np.allclose(y[..., 0], -x[:, list(MIRROR_MAP), :][:, :, 0], atol=1e-6) def test_noise_within_sigma() -> None: diff --git a/data_only_viz/tests/test_autolabel.py b/data_only_viz/tests/test_autolabel.py index 8a3c048..a529b1e 100644 --- a/data_only_viz/tests/test_autolabel.py +++ b/data_only_viz/tests/test_autolabel.py @@ -10,7 +10,7 @@ def _static_seated(frame_count: int = WINDOW_LEN) -> list[np.ndarray]: """Hip low (y small), knee bent ~80°.""" frames = [] for _ in range(frame_count): - f = np.zeros((22, 3), dtype=np.float32) + f = np.zeros((32, 3), dtype=np.float32) f[1] = [-0.1, 0.4, 0.0] f[2] = [0.1, 0.4, 0.0] f[4] = [-0.1, 0.4, 0.3] @@ -25,7 +25,7 @@ def _static_standing(frame_count: int = WINDOW_LEN) -> list[np.ndarray]: """Hip high, knees ~180°.""" frames = [] for _ in range(frame_count): - f = np.zeros((22, 3), dtype=np.float32) + f = np.zeros((32, 3), dtype=np.float32) f[1] = [-0.1, 0.9, 0.0] f[2] = [0.1, 0.9, 0.0] f[4] = [-0.1, 0.5, 0.0] diff --git a/data_only_viz/tests/test_dataset.py b/data_only_viz/tests/test_dataset.py index 679cf66..e8035fc 100644 --- a/data_only_viz/tests/test_dataset.py +++ b/data_only_viz/tests/test_dataset.py @@ -15,7 +15,7 @@ def _make_session_jsonl(path: Path, n_frames: int = 64) -> None: row = {"ts": t / 30.0, "session": "sess01", "pid": 1, - "j3d": rng.normal(size=(22, 3)).tolist()} + "j3d": rng.normal(size=(32, 3)).tolist()} f.write(json.dumps(row) + "\n") @@ -25,7 +25,7 @@ def test_load_frames_jsonl(tmp_path: Path) -> None: _make_session_jsonl(p) frames = load_frames_jsonl(p) assert len(frames) == 64 - assert frames[0].j3d.shape == (22, 3) + assert frames[0].j3d.shape == (32, 3) assert frames[0].pid == 1 assert frames[0].session == "sess01" @@ -40,7 +40,7 @@ def test_sliding_windows(tmp_path: Path) -> None: frames = load_frames_jsonl(p) windows = list(sliding_windows(frames, window_len=16, stride=4)) assert len(windows) == 13 - assert windows[0].j3d_stack.shape == (16, 22, 3) + assert windows[0].j3d_stack.shape == (16, 32, 3) assert windows[0].session == "sess01" @@ -55,7 +55,7 @@ def test_write_and_load_dataset_jsonl(tmp_path: Path) -> None: DatasetRow( window_id=f"sess01_pid1_w{i:04d}", label="debout" if i % 2 == 0 else "danse", - j3d_stack=rng.normal(size=(16, 22, 3)).astype(np.float32), + j3d_stack=rng.normal(size=(16, 32, 3)).astype(np.float32), session="sess01", pid_local=1, auto_label_confidence=0.8, @@ -68,7 +68,7 @@ def test_write_and_load_dataset_jsonl(tmp_path: Path) -> None: loaded = load_dataset_jsonl(out) assert len(loaded) == 5 assert loaded[0].label == "debout" - assert loaded[0].j3d_stack.shape == (16, 22, 3) + assert loaded[0].j3d_stack.shape == (16, 32, 3) assert np.allclose(loaded[0].j3d_stack, rows[0].j3d_stack, atol=1e-6) @@ -79,7 +79,7 @@ def test_split_by_session(tmp_path: Path) -> None: for sess in ("s01", "s02", "s03", "s04", "s05", "s06", "s07"): rows.append(DatasetRow( window_id=f"{sess}_w0", label="debout", - j3d_stack=rng.normal(size=(16, 22, 3)).astype(np.float32), + j3d_stack=rng.normal(size=(16, 32, 3)).astype(np.float32), session=sess, pid_local=1, auto_label_confidence=0.7, manually_validated=False, )) diff --git a/data_only_viz/tests/test_training_smoke.py b/data_only_viz/tests/test_training_smoke.py index 7f9d187..7b9c74c 100644 --- a/data_only_viz/tests/test_training_smoke.py +++ b/data_only_viz/tests/test_training_smoke.py @@ -19,10 +19,12 @@ def _make_tiny_dataset(tmp_path: Path) -> Path: rows.append(DatasetRow( window_id=f"{sess}_w{w:03d}", label=label, - j3d_stack=rng.normal(size=(16, 22, 3)).astype(np.float32), + j3d_stack=rng.normal(size=(16, 32, 3)).astype(np.float32), session=sess, pid_local=1, auto_label_confidence=0.8, manually_validated=True, + expr_stack=np.zeros((16, 10), dtype=np.float32), + mouth_open_stack=np.zeros(16, dtype=np.float32), )) out = tmp_path / "tiny.jsonl" write_dataset_jsonl(rows, out) @@ -57,5 +59,5 @@ def test_trained_checkpoint_loadable(tmp_path: Path) -> None: lr=1e-3, device="cpu", seed=0, log_every=10_000) head = ActionHead(ckpt_path=ckpt) for i in range(5): - label, probs, _ = head.step(pid=1, j3d=np.zeros((22, 3), dtype=np.float32)) + label, probs, _ = head.step(pid=1, j3d=np.zeros((32, 3), dtype=np.float32)) assert abs(float(probs.sum()) - 1.0) < 1e-5 diff --git a/data_only_viz/training/augment.py b/data_only_viz/training/augment.py index c1b7d93..0c3cff0 100644 --- a/data_only_viz/training/augment.py +++ b/data_only_viz/training/augment.py @@ -3,8 +3,10 @@ from __future__ import annotations import numpy as np -# SMPL-X left/right joint mirror map (subset 22 joints used by Multi-HMR). +# SMPL-X left/right joint mirror map for 32-joint layout. +# Body joints 0..21 (unchanged), fingertips 22..31 (L 22..26 <-> R 27..31). MIRROR_MAP: tuple[int, ...] = ( + # 22 body (unchanged) 0, 2, 1, 3, @@ -19,7 +21,10 @@ MIRROR_MAP: tuple[int, ...] = ( 17, 16, 19, 18, 21, 20, + # 10 fingertips: L (22..26) <-> R (27..31) + 27, 28, 29, 30, 31, 22, 23, 24, 25, 26, ) +assert len(MIRROR_MAP) == 32 def mirror_x(stack: np.ndarray) -> np.ndarray: diff --git a/data_only_viz/training/dataset.py b/data_only_viz/training/dataset.py index 60cd2cd..c9223ef 100644 --- a/data_only_viz/training/dataset.py +++ b/data_only_viz/training/dataset.py @@ -3,7 +3,7 @@ from __future__ import annotations import json import random -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Iterable, Iterator @@ -15,26 +15,32 @@ class RawFrame: ts: float session: str pid: int - j3d: np.ndarray # (22, 3) float32 + j3d: np.ndarray # (32, 3) float32 (v2: body22 + 10 fingertips) + expression: np.ndarray | None = None # (EXPR_DIM,) or None + mouth_open: float = 0.0 @dataclass class WindowRow: - j3d_stack: np.ndarray # (window_len, 22, 3) float32 + j3d_stack: np.ndarray # (window_len, 32, 3) float32 session: str pid_local: int first_ts: float + expr_stack: np.ndarray | None = None # (window_len, 10) or None + mouth_open_stack: np.ndarray | None = None # (window_len,) or None @dataclass class DatasetRow: window_id: str label: str - j3d_stack: np.ndarray # (window_len, 22, 3) float32 + j3d_stack: np.ndarray # (window_len, 32, 3) float32 session: str pid_local: int auto_label_confidence: float manually_validated: bool + expr_stack: np.ndarray | None = None # (window_len, 10) or None + mouth_open_stack: np.ndarray | None = None # (window_len,) or None def load_frames_jsonl(path: Path) -> list[RawFrame]: @@ -45,11 +51,15 @@ def load_frames_jsonl(path: Path) -> list[RawFrame]: if not line: continue d = json.loads(line) + expr_raw = d.get("expression") + expr = np.asarray(expr_raw, dtype=np.float32) if expr_raw is not None else None rows.append(RawFrame( ts=float(d["ts"]), session=str(d["session"]), pid=int(d["pid"]), j3d=np.asarray(d["j3d"], dtype=np.float32), + expression=expr, + mouth_open=float(d.get("mouth_open", 0.0)), )) return rows @@ -68,14 +78,32 @@ def sliding_windows(frames: list[RawFrame], for start in range(0, len(grp) - window_len + 1, stride): chunk = grp[start:start + window_len] stack = np.stack([c.j3d for c in chunk]).astype(np.float32) + # Expression stack: zeros if not present + if any(c.expression is not None for c in chunk): + expr_dim = max( + (len(c.expression) for c in chunk if c.expression is not None), + default=10, + ) + expr_stack = np.zeros((window_len, expr_dim), dtype=np.float32) + for t, c in enumerate(chunk): + if c.expression is not None: + n = min(expr_dim, len(c.expression)) + expr_stack[t, :n] = c.expression[:n] + else: + expr_stack = None + mouth_stack = np.array( + [c.mouth_open for c in chunk], dtype=np.float32 + ) yield WindowRow(j3d_stack=stack, session=sess, - pid_local=pid, first_ts=chunk[0].ts) + pid_local=pid, first_ts=chunk[0].ts, + expr_stack=expr_stack, + mouth_open_stack=mouth_stack) def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None: with path.open("w") as f: for r in rows: - f.write(json.dumps({ + d: dict = { "window_id": r.window_id, "label": r.label, "j3d": r.j3d_stack.astype(np.float32).tolist(), @@ -83,7 +111,12 @@ def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None: "pid_local": r.pid_local, "auto_label_confidence": float(r.auto_label_confidence), "manually_validated": bool(r.manually_validated), - }) + "\n") + } + if r.expr_stack is not None: + d["expr_stack"] = r.expr_stack.astype(np.float32).tolist() + if r.mouth_open_stack is not None: + d["mouth_open_stack"] = r.mouth_open_stack.astype(np.float32).tolist() + f.write(json.dumps(d) + "\n") def load_dataset_jsonl(path: Path) -> list[DatasetRow]: @@ -94,6 +127,10 @@ def load_dataset_jsonl(path: Path) -> list[DatasetRow]: if not line: continue d = json.loads(line) + expr_raw = d.get("expr_stack") + expr = np.asarray(expr_raw, dtype=np.float32) if expr_raw is not None else None + mouth_raw = d.get("mouth_open_stack") + mouth = np.asarray(mouth_raw, dtype=np.float32) if mouth_raw is not None else None out.append(DatasetRow( window_id=d["window_id"], label=d["label"], @@ -102,6 +139,8 @@ def load_dataset_jsonl(path: Path) -> list[DatasetRow]: pid_local=int(d["pid_local"]), auto_label_confidence=float(d["auto_label_confidence"]), manually_validated=bool(d["manually_validated"]), + expr_stack=expr, + mouth_open_stack=mouth, )) return out diff --git a/data_only_viz/training/train_action_head.py b/data_only_viz/training/train_action_head.py index e777146..129555d 100644 --- a/data_only_viz/training/train_action_head.py +++ b/data_only_viz/training/train_action_head.py @@ -21,6 +21,7 @@ from torch.utils.data import DataLoader, Dataset from data_only_viz.action_head import ( ActionHeadModel, + EXPR_DIM, FeatureExtractor, HIP_LEFT, HIP_RIGHT, @@ -52,19 +53,35 @@ class WindowDataset(Dataset[tuple[torch.Tensor, int]]): stack = row.j3d_stack if self._augment: stack = random_augment(stack, self._rng) + T = stack.shape[0] + # expression and mouth_open stacks (zeros if absent / legacy) + if row.expr_stack is not None: + expr_s = row.expr_stack.astype(np.float32) + else: + expr_s = np.zeros((T, EXPR_DIM), dtype=np.float32) + if row.mouth_open_stack is not None: + mouth_s = row.mouth_open_stack.astype(np.float32) + else: + mouth_s = np.zeros(T, dtype=np.float32) feats = [] prev = stack[0] prev_vel = np.zeros_like(prev) - for t in range(stack.shape[0]): + for t in range(T): cur = stack[t] vel = cur - prev accel = vel - prev_vel hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5) knee_angle = FeatureExtractor._mean_knee_angle(cur) sym = FeatureExtractor._symmetry_score(vel) + expr_t = expr_s[t] if t < len(expr_s) else np.zeros(EXPR_DIM, dtype=np.float32) + expr_vec = np.zeros(EXPR_DIM, dtype=np.float32) + n = min(EXPR_DIM, len(expr_t)) + expr_vec[:n] = expr_t[:n] + mouth_t = float(mouth_s[t]) if t < len(mouth_s) else 0.0 feat = np.concatenate([ cur.reshape(-1), vel.reshape(-1), accel.reshape(-1), - np.array([hip_y, knee_angle, sym], dtype=np.float32), + expr_vec, + np.array([hip_y, knee_angle, sym, mouth_t], dtype=np.float32), ]).astype(np.float32, copy=False) feats.append(feat) prev_vel = vel From 2c8094c06c22150dee4cb4b21f582fa63457af1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:17:22 +0200 Subject: [PATCH 05/14] feat(av-live): hybrid mesh rigging 30 fps MeshRigger module : entre deux keyframes Multi-HMR (~3.5 fps mesh dense PyTorch MPS), on translate rigidement le mesh keyframe via le delta pelvis 2D Apple Vision (30 fps body ANE) projete a profondeur constante. SMPLXTCPSender bumpe a 30 fps target et applique le rig sur chaque tick. Verifie live : 27 fps TCP soutenu, 100% rigged, keyframe Multi-HMR a 3.2 fps -> ~8x speedup perceptuel dans la fenetre RealityKit AVLiveBody. Limitations connues : - Translation seule (pas de rotation ni de LBS articule) - Pelvis 2D delta projete a Z constant du keyframe - Pas de matching d'identite robuste Vision <-> Multi-HMR : on prend la personne Vision la plus proche du pelvis keyframe projete --- data_only_viz/mesh_rigger.py | 215 ++++++++++++++++++++++++++++++ data_only_viz/smplx_osc_sender.py | 32 ++++- 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 data_only_viz/mesh_rigger.py diff --git a/data_only_viz/mesh_rigger.py b/data_only_viz/mesh_rigger.py new file mode 100644 index 0000000..3eda2fc --- /dev/null +++ b/data_only_viz/mesh_rigger.py @@ -0,0 +1,215 @@ +"""Mesh rigging hybride keyframe (Multi-HMR) + delta Apple Vision. + +Multi-HMR produit un mesh SMPL-X dense (10475 verts) tous les ~300 ms +sur M5 (PyTorch MPS ~3.5 fps). Entre deux keyframes, Apple Vision sur +ANE produit 30 fps de body keypoints 2D. On exploite le pelvis 2D de +Vision pour translater rigidement le mesh keyframe et donner une +perception fluide a 30 fps cote launcher RealityKit. + +Limitations connues (premiere iteration) : + - Translation rigide uniquement (pas de rotation, pas de LBS articule) + - Pelvis 2D delta projete en X/Y a profondeur constante (z keyframe) + - Pas de matching d'identite Vision <-> Multi-HMR : on prend la + personne Vision la plus proche du pelvis projete keyframe +""" +from __future__ import annotations + +import math +import threading +import time +from dataclasses import dataclass, field + +import numpy as np + +from .state import PoseKp, SMPLXPerson, State + + +# Indices MediaPipe POSE_LANDMARKS pour les hanches (pelvis 2D = midpoint). +_LEFT_HIP = 23 +_RIGHT_HIP = 24 + +# Focale par defaut Multi-HMR (camera intrinsics typiques utilisees +# dans multi_hmr_worker : focal = IMG_SIZE). +_IMG_SIZE = 672 +_FOCAL = float(_IMG_SIZE) + + +@dataclass +class _Keyframe: + """Snapshot d'un mesh Multi-HMR + reference Vision au moment T.""" + pid: int + t: float + # Mesh world coords (10475, 3) float32 incluant la translation + vertices_3d: np.ndarray + translation: np.ndarray # (3,) world pelvis + vision_pelvis_2d: tuple[float, float] | None # (cx, cy) normalises 0..1 + + +def _pelvis_2d_from_body(body: list[PoseKp]) -> tuple[float, float] | None: + """Midpoint des deux hanches MediaPipe si confidence > 0.""" + if not body or len(body) <= _RIGHT_HIP: + return None + lh, rh = body[_LEFT_HIP], body[_RIGHT_HIP] + if lh.c <= 0.1 or rh.c <= 0.1: + return None + return (0.5 * (lh.x + rh.x), 0.5 * (lh.y + rh.y)) + + +def _vision_pid_match( + keyframe_pelvis_2d: tuple[float, float] | None, + vision_bodies: list[list[PoseKp]], + vision_ids: list[int], +) -> int | None: + """Retourne le pid Vision dont le pelvis 2D est le plus proche du + keyframe pelvis projete. None si rien.""" + if keyframe_pelvis_2d is None or not vision_bodies: + return None + kx, ky = keyframe_pelvis_2d + best_pid: int | None = None + best_d2 = float("inf") + for body, vpid in zip(vision_bodies, vision_ids): + p = _pelvis_2d_from_body(body) + if p is None: + continue + d2 = (p[0] - kx) ** 2 + (p[1] - ky) ** 2 + if d2 < best_d2: + best_d2 = d2 + best_pid = int(vpid) + return best_pid + + +class MeshRigger: + """Rig le mesh SMPL-X keyframe via le delta pelvis Vision. + + Usage : + rigger = MeshRigger(state) + rigged_persons = rigger.apply(state.persons_smplx, + state.persons_body, + t_now) + Thread-safe : ne mute pas le state, retourne une nouvelle liste. + """ + + def __init__(self, state: State, hold_window_s: float = 1.5) -> None: + self.state = state + self.hold_window_s = hold_window_s + self._lock = threading.Lock() + # pid Multi-HMR -> keyframe + self._keyframes: dict[int, _Keyframe] = {} + # pid Multi-HMR -> pid Vision matched (sticky across frames) + self._vision_pid_map: dict[int, int] = {} + + def apply( + self, + persons_smplx: list[SMPLXPerson], + persons_body: list[list[PoseKp]], + persons_body_ids: list[int], + t_now: float, + ) -> list[SMPLXPerson]: + """Retourne une liste SMPLXPerson translatee par delta Vision.""" + # 1) Detect new keyframes (timestamp tracked via state.smplx_last_t) + with self._lock: + current_pids = {p.pid for p in persons_smplx} + # Drop stale keyframes (person disparue) + for old_pid in list(self._keyframes): + if old_pid not in current_pids: + self._keyframes.pop(old_pid, None) + self._vision_pid_map.pop(old_pid, None) + + out: list[SMPLXPerson] = [] + for person in persons_smplx: + kf = self._keyframes.get(person.pid) + # Detect keyframe refresh : translation differs from kf + is_new_kf = (kf is None or not np.allclose( + kf.translation, person.translation, atol=1e-4)) + if is_new_kf: + # Trouver le pid Vision le plus proche pour ce mesh. + # On projette le pelvis world en 2D image-normalized : + # x_img = (X / Z) * focal / IMG_SIZE + 0.5 + pelvis_2d = self._project_pelvis(person.translation) + matched = _vision_pid_match( + pelvis_2d, persons_body, persons_body_ids) + if matched is None: + matched = self._vision_pid_map.get(person.pid) + if matched is not None: + self._vision_pid_map[person.pid] = matched + # Capture du pelvis 2D Vision au moment du keyframe + vp = None + if matched is not None: + try: + i = persons_body_ids.index(matched) + vp = _pelvis_2d_from_body(persons_body[i]) + except (ValueError, IndexError): + vp = None + self._keyframes[person.pid] = _Keyframe( + pid=person.pid, + t=t_now, + vertices_3d=person.vertices_3d.copy(), + translation=person.translation.copy(), + vision_pelvis_2d=vp, + ) + out.append(person) + continue + + # Entre keyframes : applique delta translation depuis + # Vision pelvis 2D actuel vs keyframe pelvis 2D. + if t_now - kf.t > self.hold_window_s: + # Trop ancien, on lache le rig (mesh statique) + out.append(person) + continue + matched_pid = self._vision_pid_map.get(person.pid) + if matched_pid is None or kf.vision_pelvis_2d is None: + out.append(person) + continue + try: + i = persons_body_ids.index(matched_pid) + except ValueError: + out.append(person) + continue + current_vp = _pelvis_2d_from_body(persons_body[i]) + if current_vp is None: + out.append(person) + continue + + # Image-normalized 2D delta -> world XY delta a depth z_kf. + # Pour un pelvis aux coords image (px in [0,1] centre 0.5), + # X_world = (px - 0.5) * IMG_SIZE * Z / focal = (px-0.5)*Z + # (focal=IMG_SIZE). Delta image -> Delta world a Z fixe. + z_kf = float(kf.translation[2]) if abs( + kf.translation[2]) > 1e-3 else 1.0 + dx_img = current_vp[0] - kf.vision_pelvis_2d[0] + dy_img = current_vp[1] - kf.vision_pelvis_2d[1] + dx_world = dx_img * _IMG_SIZE * z_kf / _FOCAL + dy_world = dy_img * _IMG_SIZE * z_kf / _FOCAL + + # Applique a tous les vertices + a translation. + new_verts = kf.vertices_3d.copy() + new_verts[:, 0] += np.float32(dx_world) + new_verts[:, 1] += np.float32(dy_world) + new_transl = kf.translation.copy() + new_transl[0] += np.float32(dx_world) + new_transl[1] += np.float32(dy_world) + + out.append(SMPLXPerson( + pid=person.pid, + vertices_3d=new_verts, + translation=new_transl, + confidence=person.confidence, + betas=person.betas, + expression=person.expression, + )) + return out + + @staticmethod + def _project_pelvis( + translation: np.ndarray, + ) -> tuple[float, float] | None: + """World pelvis (X,Y,Z) -> image-normalized 2D pelvis.""" + z = float(translation[2]) + if abs(z) < 1e-3: + return None + x_img = (float(translation[0]) * _FOCAL / z) / _IMG_SIZE + 0.5 + y_img = (float(translation[1]) * _FOCAL / z) / _IMG_SIZE + 0.5 + # Clamp en [0,1] + if not (0.0 <= x_img <= 1.0 and 0.0 <= y_img <= 1.0): + return None + return (x_img, y_img) diff --git a/data_only_viz/smplx_osc_sender.py b/data_only_viz/smplx_osc_sender.py index aa7b379..18dbdd8 100644 --- a/data_only_viz/smplx_osc_sender.py +++ b/data_only_viz/smplx_osc_sender.py @@ -25,6 +25,7 @@ from typing import Sequence import numpy as np +from .mesh_rigger import MeshRigger from .state import SMPLXPerson, State LOG = logging.getLogger("smplx_tcp") @@ -35,7 +36,8 @@ PORT = 57130 class SMPLXTCPSender: def __init__(self, state: State, host: str = "127.0.0.1", - port: int = PORT, target_fps: float = 12.0) -> None: + port: int = PORT, target_fps: float = 30.0, + enable_rigging: bool = True) -> None: self.state = state self.host = host self.port = port @@ -43,6 +45,9 @@ class SMPLXTCPSender: self._stop = threading.Event() self._thread: threading.Thread | None = None self._sock: socket.socket | None = None + # Hybrid keyframe rigging : entre deux keyframes Multi-HMR (~3 fps), + # on translate le mesh via le delta pelvis Apple Vision (30 fps). + self._rigger = MeshRigger(state) if enable_rigging else None def start(self) -> None: self._thread = threading.Thread( @@ -124,6 +129,9 @@ class SMPLXTCPSender: def _run(self) -> None: last_warn = 0.0 + n_sent = 0 + n_rigged = 0 + next_hb = time.monotonic() + 5.0 while not self._stop.is_set(): t0 = time.monotonic() if not self._ensure_connected(): @@ -136,8 +144,30 @@ class SMPLXTCPSender: with self.state.lock(): persons = list(self.state.persons_smplx) + body_kp = list(self.state.persons_body) if hasattr( + self.state, "persons_body") else [] + body_ids = list(self.state.persons_body_ids) if hasattr( + self.state, "persons_body_ids") else ( + list(range(len(body_kp))) if body_kp else []) + + if persons and self._rigger is not None: + rigged = self._rigger.apply( + persons, body_kp, body_ids, t0) + if rigged is not persons: + n_rigged += 1 + persons = rigged + + if t0 >= next_hb: + fps = n_sent / 5.0 + rig_pct = (n_rigged / n_sent * 100.0) if n_sent else 0.0 + LOG.info("hb: %.1f fps tcp, %.0f%% rigged", + fps, rig_pct) + n_sent = 0 + n_rigged = 0 + next_hb = t0 + 5.0 if persons: + n_sent += 1 t_ser_start = time.monotonic() payload = self._serialize_persons(persons) t_send_start = time.monotonic() From 623c47983d5d2408c1c5c5116aecd62a48033d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:24:06 +0200 Subject: [PATCH 06/14] perf(multi-hmr): autocast opt-in MULTIHMR_AUTOCAST=1 enables MPS mixed precision for the ViT-S backbone. Tested 2026-05-13: slower than fp32 baseline (400ms vs 270ms) -- overhead cast within forward exceeds matmul savings on M5. Off by default; FP16 .half() crashes MPS matmul accumulator, left out entirely. apple_vision_pose face parser short-circuits to return immediately since pyobjc 11 cannot dereference VNFaceLandmarkRegion2D pointsInImageOfSize_ result. Removes ObjCPointerWarning spam at 30 fps (9 regions per face). --- data_only_viz/apple_vision_pose.py | 6 ++--- data_only_viz/multi_hmr_worker.py | 39 ++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/data_only_viz/apple_vision_pose.py b/data_only_viz/apple_vision_pose.py index a8431a4..2ccb3f5 100644 --- a/data_only_viz/apple_vision_pose.py +++ b/data_only_viz/apple_vision_pose.py @@ -557,10 +557,8 @@ class AppleVisionPoseWorker: # API d'acces simple. Face parsing depuis Apple Vision est # actuellement bloque ; on garde MediaPipe (CPU XNNPACK) pour # face/hand fin tandis que Vision sert body 2D sur ANE. - n = 0; n_written = 0 - if n_written > 0 and not hasattr(self, "_logged_face_write_" + region_name): - LOG.info("face: %s wrote %d points", region_name, n_written) - setattr(self, "_logged_face_write_" + region_name, True) + # Skip pour eviter le spam ObjCPointerWarning a 30 fps. + return # faceContour fill("faceContour", *FACE_OFFSETS["contour"]) diff --git a/data_only_viz/multi_hmr_worker.py b/data_only_viz/multi_hmr_worker.py index a09092a..916d703 100644 --- a/data_only_viz/multi_hmr_worker.py +++ b/data_only_viz/multi_hmr_worker.py @@ -142,6 +142,20 @@ class MultiHMRWorker: model = Model(**kwargs).to(torch_device) model.load_state_dict(ckpt["model_state_dict"], strict=False) model.eval() + # MPS mixed precision via torch.autocast : ~1.3-1.7x sur + # ViT-S backbone, casts auto vers float16 pour les matmuls + # gardant l'accumulator en float32 (necessaire MPS sinon + # "Destination NDArray and Accumulator NDArray cannot have + # different datatype" sur MPSNDArrayMatrixMultiplication). + # Disable via env MULTIHMR_AUTOCAST=0. + # autocast MPS teste 2026-05-13 : plus lent (400ms vs 270ms + # baseline) car overhead de cast dans le forward. Defaut OFF. + # Opt-in via MULTIHMR_AUTOCAST=1. + self._use_autocast = ( + device == "mps" + and os.environ.get("MULTIHMR_AUTOCAST", "0") == "1") + if self._use_autocast: + LOG.info("Multi-HMR PyTorch : MPS autocast (fp16) enabled") # torch.compile teste 2026-05-13 : plante en runtime avec # `TypeError: torch.Size() takes an iterable of 'int' (item # is 'FakeTensor')`. Multi-HMR a du shape-arithmetic non @@ -230,13 +244,24 @@ class MultiHMRWorker: t_inf_start = time.monotonic() try: with torch.no_grad(): - humans = model( - tensor, - is_training=False, - nms_kernel_size=self.nms_kernel_size, - det_thresh=self.det_thresh, - K=K, - ) + if getattr(self, "_use_autocast", False): + with torch.autocast(device_type="mps", + dtype=torch.float16): + humans = model( + tensor, + is_training=False, + nms_kernel_size=self.nms_kernel_size, + det_thresh=self.det_thresh, + K=K, + ) + else: + humans = model( + tensor, + is_training=False, + nms_kernel_size=self.nms_kernel_size, + det_thresh=self.det_thresh, + K=K, + ) except Exception as e: LOG.warning("inference failed: %s", e) time.sleep(self.period) From beb94d2a4cc3f4d540ac1ab0ae537146aadccadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:26:14 +0200 Subject: [PATCH 07/14] feat(data-only-viz): action-head v3 hands+lips Extends the action-head feature pipeline from v2 (302-D) to v3 (428-D). - Replace placeholder SMPLX_FINGERTIP_VERTS with canonical vertex IDs from smplx.vertex_ids (lthumb/lindex/lmiddle/lring/lpinky, mirrored R) - Add HANDS_KP_* constants (21 kp/hand, 42 total, 126-D flat block) - FEATURE_DIM: 302 -> 428; hands_kp block inserted at [288:414] - FeatureExtractor.from_buffer gains hands_kp param (42, 3), zero-padded when absent - ActionHead.step gains hands_kp param, threads to from_buffer - _read_sources returns 5-tuples with hands_kp42x3 per person - MediaPipe FaceMesh inner-lip (idx 13/14) used for mouth_open; fallback to SMPL-X v3d lip vertices when face not available - _build_hands_map and _build_face_mouth_map helpers added - dataset.py: RawFrame/WindowRow/DatasetRow gain hands_kp fields - train_action_head.py: reads hands_kp_stack per step, zeros if absent - extract_j3d_offline.py: writes zero-filled hands_kp in jsonl output - Tests: FEATURE_DIM 302->428, param bound 80k->100k, +4 new tests --- data_only_viz/action_head.py | 43 +++-- data_only_viz/action_head_pub.py | 150 +++++++++++++----- data_only_viz/scripts/extract_j3d_offline.py | 5 +- .../tests/test_action_head_features.py | 6 +- data_only_viz/tests/test_action_head_model.py | 4 +- data_only_viz/tests/test_action_head_pub.py | 68 ++++++++ data_only_viz/tests/test_dataset.py | 47 ++++++ data_only_viz/training/dataset.py | 26 ++- data_only_viz/training/train_action_head.py | 8 + 9 files changed, 298 insertions(+), 59 deletions(-) diff --git a/data_only_viz/action_head.py b/data_only_viz/action_head.py index 89bc6de..c455ede 100644 --- a/data_only_viz/action_head.py +++ b/data_only_viz/action_head.py @@ -32,13 +32,20 @@ LABELS: tuple[str, str, str] = ("debout", "assise", "danse") EXPR_DIM: int = 10 EXTRA_SCALARS: int = 4 # hip_y, knee_angle, sym_score, mouth_open -# Layout per step: -# [0 : 96 ] j3d (32, 3) +# NEW v3 : MediaPipe Hands keypoints block. +HANDS_KP_PER_HAND: int = 21 +HANDS_KP_TOTAL: int = 2 * HANDS_KP_PER_HAND # 42 +HANDS_KP_DIMS: int = 3 +HANDS_KP_FLAT: int = HANDS_KP_TOTAL * HANDS_KP_DIMS # 126 + +# Layout per step (v3) : +# [0 : 96] j3d (32, 3) # [96 : 192] vel (32, 3) # [192 : 288] accel (32, 3) -# [288 : 298] expression (10,) -# [298 : 302] scalars (hip_y, knee_angle, sym, mouth_open) -FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + EXPR_DIM + EXTRA_SCALARS # 302 +# [288 : 414] hands_kp (42, 3) zero-padded if absent +# [414 : 424] expression (10,) +# [424 : 428] scalars (hip_y, knee_angle, sym, mouth_open) +FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + HANDS_KP_FLAT + EXPR_DIM + EXTRA_SCALARS # 428 # Body joint indices (unchanged from v1, indices 0..21). HIP_LEFT: int = 1 @@ -60,18 +67,20 @@ FINGERTIP_RIGHT_BASE: int = 27 class FeatureExtractor: """Stateless feature builder over a list of recent j3d frames. - Vector layout (FEATURE_DIM = 302): - [0 : 96 ] j3d current frame, flattened (32 joints x 3 dims) + Vector layout (FEATURE_DIM = 428, v3): + [0 : 96] j3d current frame, flattened (32 joints x 3 dims) [96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3) [192 : 288] acceleration vel[t] - vel[t-1] (32 x 3) - [288 : 298] expression PCA coefficients (10,) - [298 : 302] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open) + [288 : 414] hands_kp (42, 3) MediaPipe Hands, zero-padded if absent + [414 : 424] expression PCA coefficients (10,) + [424 : 428] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open) """ @staticmethod def from_buffer(frames: list[np.ndarray], expr: np.ndarray | None = None, - mouth_open: float = 0.0) -> np.ndarray: + mouth_open: float = 0.0, + hands_kp: np.ndarray | None = None) -> np.ndarray: if not frames: return np.zeros(FEATURE_DIM, dtype=np.float32) cur = frames[-1] @@ -83,6 +92,13 @@ class FeatureExtractor: hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5) knee_angle = FeatureExtractor._mean_knee_angle(cur) sym = FeatureExtractor._symmetry_score(vel) + # hands block (42, 3) -> 126 + hands_flat = np.zeros(HANDS_KP_FLAT, dtype=np.float32) + if hands_kp is not None: + hk = np.asarray(hands_kp, dtype=np.float32) + if hk.shape == (HANDS_KP_TOTAL, HANDS_KP_DIMS): + hands_flat = hk.reshape(-1).astype(np.float32, copy=False) + # expression if expr is None: expr_vec = np.zeros(EXPR_DIM, dtype=np.float32) else: @@ -93,6 +109,7 @@ class FeatureExtractor: cur.reshape(-1), vel.reshape(-1), accel.reshape(-1), + hands_flat, expr_vec, np.array([hip_y, knee_angle, sym, float(mouth_open)], dtype=np.float32), ]).astype(np.float32, copy=False) @@ -227,7 +244,8 @@ class ActionHead: def step(self, pid: int, j3d: np.ndarray, expr: np.ndarray | None = None, - mouth_open: float = 0.0) -> tuple[str, np.ndarray, np.ndarray]: + mouth_open: float = 0.0, + hands_kp: np.ndarray | None = None) -> tuple[str, np.ndarray, np.ndarray]: if np.isnan(j3d).any(): streak = self._nan_streak.get(pid, 0) + 1 self._nan_streak[pid] = streak @@ -241,7 +259,8 @@ class ActionHead: if len(frames) < WARMUP_FRAMES: probs = np.array([1.0, 0.0, 0.0], dtype=np.float32) return LABELS[0], probs, np.zeros(3, dtype=np.float32) - feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open) + feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open, + hands_kp=hands_kp) kin = FeatureExtractor.kinetics(frames) h = self._hidden.get(pid) if h is None: diff --git a/data_only_viz/action_head_pub.py b/data_only_viz/action_head_pub.py index bd72d39..51bd033 100644 --- a/data_only_viz/action_head_pub.py +++ b/data_only_viz/action_head_pub.py @@ -17,6 +17,9 @@ import numpy as np from data_only_viz.action_head import ( ActionHead, EXPR_DIM, + HANDS_KP_DIMS, + HANDS_KP_PER_HAND, + HANDS_KP_TOTAL, J3D_FINGERS, J3D_FINGERS_PER_HAND, LABELS, @@ -28,12 +31,12 @@ DEFAULT_CKPT = ( Path.home() / ".cache" / "av-live-action" / "checkpoints" / "action_head.pt" ) -# Approximate fingertip vertex indices on SMPL-X 10475-vert mesh. -# Order: L thumb, L index, L middle, L ring, L pinky, -# R thumb, R index, R middle, R ring, R pinky. +# Canonical SMPL-X fingertip vertex IDs from smplx.vertex_ids.SMPLX_VERTEX_IDS. +# Order : L thumb, L index, L middle, L ring, L pinky, +# R thumb, R index, R middle, R ring, R pinky. SMPLX_FINGERTIP_VERTS: tuple[int, ...] = ( - 7174, 7397, 7670, 7942, 8214, # L - 4631, 4854, 5127, 5399, 5671, # R + 5361, 4933, 5058, 5169, 5286, # L : lthumb, lindex, lmiddle, lring, lpinky + 8079, 7669, 7794, 7905, 8022, # R : rthumb, rindex, rmiddle, rring, rpinky ) # 32 vertex indices on the 10475-vertex SMPL-X mesh: @@ -56,6 +59,11 @@ assert len(SMPLX_JOINT_ANCHOR_VERTS) == 32 SMPLX_UPPER_LIP_VERT: int = 8970 SMPLX_LOWER_LIP_VERT: int = 8855 +# MediaPipe FaceMesh inner-mouth landmark indices. +# 13 = upper inner mid, 14 = lower inner mid. +MEDIAPIPE_LIP_UPPER_INNER: int = 13 +MEDIAPIPE_LIP_LOWER_INNER: int = 14 + # MediaPipe HAND fingertip indices (21-kp hand model). MEDIAPIPE_HAND_FINGERTIPS: tuple[int, ...] = (4, 8, 12, 16, 20) @@ -120,11 +128,11 @@ class ActionHeadPublisher(threading.Thread): self._last_body_t = source_t current_pids: set[int] = set() if persons32: - for pid, j3d, expr, mouth in persons32: + for pid, j3d, expr_np, mouth, hands_kp42 in persons32: current_pids.add(pid) - label, probs, kin = self.head.step(pid, j3d, - expr=expr, - mouth_open=mouth) + label, probs, kin = self.head.step(pid, j3d, expr=expr_np, + mouth_open=mouth, + hands_kp=hands_kp42) idx = LABELS.index(label) self.bridge.send_action(pid, idx, probs, t_now, force=True) self.bridge.send_kin(pid, kin, t_now, force=True) @@ -135,13 +143,13 @@ class ActionHeadPublisher(threading.Thread): self.bridge.send_leave(pid=gone) self._last_pids = current_pids - def _read_sources( - self, - ) -> tuple[list[tuple[int, np.ndarray, np.ndarray, float]] | None, - float, str, bool]: + def _read_sources(self) -> tuple[ + list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] | None, + float, str, bool, + ]: """Return (persons32, source_t, source_tag, is_new). - Each person entry is (pid, j3d32, expr10, mouth_open). + Each person entry is (pid, j3d32, expr10, mouth_open, hands_kp42x3). is_new is True when the timestamp advanced (even if person list is empty), so _tick can still run the purge loop. """ @@ -150,12 +158,24 @@ class ActionHeadPublisher(threading.Thread): t_smplx = getattr(self.state, "smplx_last_t", 0.0) persons_b3d = getattr(self.state, "persons_body3d", None) ids_b3d = getattr(self.state, "persons_body_ids", None) + persons_face = getattr(self.state, "persons_face", None) + ids_face = getattr(self.state, "persons_face_ids", None) + persons_hands = getattr(self.state, "persons_hands", None) + ids_hands = getattr(self.state, "persons_hands_ids", None) t_body = getattr(self.state, "pose_last_t", 0.0) - hands_ids = list(getattr(self.state, "persons_hands_ids", None) or []) - hands_lists = list(getattr(self.state, "persons_hands", None) or []) - # Prefer smplx when its timestamp advanced. + + # Build pid -> hands_kp(42, 3) map from MediaPipe persons_hands. + hands_by_pid: dict[int, np.ndarray] = self._build_hands_map( + persons_hands or [], ids_hands or [], + ) + # Build pid -> mouth_open scalar from MediaPipe persons_face lips. + face_mouth_by_pid: dict[int, float] = self._build_face_mouth_map( + persons_face or [], ids_face or [], + ) + + # SMPL-X path (preferred) if t_smplx > self._last_smplx_t: - out: list[tuple[int, np.ndarray, np.ndarray, float]] = [] + out: list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] = [] for i, p in enumerate(persons_smplx or []): pid = int(p.get("pid", i)) v3d = p.get("v3d") @@ -177,23 +197,24 @@ class ActionHeadPublisher(threading.Thread): expr_np = np.asarray(expr, dtype=np.float32).flatten() else: expr_np = np.zeros(EXPR_DIM, dtype=np.float32) - # mouth_open - if v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT): + # mouth_open: prefer MediaPipe face lips, fallback SMPL-X v3d. + if pid in face_mouth_by_pid: + mouth = face_mouth_by_pid[pid] + elif v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT): mouth = float(np.linalg.norm( v3d_np[SMPLX_UPPER_LIP_VERT] - v3d_np[SMPLX_LOWER_LIP_VERT] )) else: mouth = 0.0 - out.append((pid, j3d32, expr_np, mouth)) + hands_kp42 = hands_by_pid.get( + pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32) + ) + out.append((pid, j3d32, expr_np, mouth, hands_kp42)) return out or None, t_smplx, "smplx", True + + # MediaPipe body3d fallback if t_body > self._last_body_t: ids = ids_b3d or list(range(len(persons_b3d or []))) - # Build hands lookup by pid - hands_by_pid: dict[int, dict[str, Any]] = {} - for hi, hkp in enumerate(hands_lists): - hpid = int(hands_ids[hi]) if hi < len(hands_ids) else hi - side = "L" if hi % 2 == 0 else "R" - hands_by_pid.setdefault(hpid, {})[side] = hkp out = [] for i, body in enumerate(persons_b3d or []): pid = int(ids[i]) if i < len(ids) else i @@ -201,25 +222,74 @@ class ActionHeadPublisher(threading.Thread): if arr is None or arr.shape[0] < 33: continue body22 = arr[list(MEDIAPIPE_TO_22)].astype(np.float32) - # fingertips from hands if available + # fingertips from persons_hands if available tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32) - hpair = hands_by_pid.get(pid, {}) - for side_idx, side in enumerate(("L", "R")): - hkp = hpair.get(side) - if hkp is None: - continue - hkp_arr = self._kp_list_to_array(hkp) - if hkp_arr is None or hkp_arr.shape[0] < 21: - continue - for k, mp_idx in enumerate(MEDIAPIPE_HAND_FINGERTIPS): - tips[side_idx * J3D_FINGERS_PER_HAND + k] = hkp_arr[mp_idx] + hands_kp42 = hands_by_pid.get( + pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32) + ) + # extract fingertips from hands_kp42 (idx 4,8,12,16,20 each side) + for side_idx in (0, 1): + base = side_idx * HANDS_KP_PER_HAND + for k, mp_tip in enumerate(MEDIAPIPE_HAND_FINGERTIPS): + if base + mp_tip < hands_kp42.shape[0]: + tips[side_idx * J3D_FINGERS_PER_HAND + k] = \ + hands_kp42[base + mp_tip] j3d32 = np.concatenate([body22, tips], axis=0) + mouth = face_mouth_by_pid.get(pid, 0.0) expr_np = np.zeros(EXPR_DIM, dtype=np.float32) - mouth = 0.0 - out.append((pid, j3d32, expr_np, mouth)) + out.append((pid, j3d32, expr_np, mouth, hands_kp42)) return out or None, t_body, "body3d", True return None, 0.0, "", False + def _build_hands_map(self, persons_hands: list, + ids_hands: list) -> dict[int, np.ndarray]: + """Combine left+right hand kp arrays per pid into a single (42, 3) array. + + persons_hands is a flat list ; ids_hands maps each hand-list entry to a + pid (and odd/even index indicates which side). When the user's pipeline + keeps a different convention, this helper makes the best effort and + pads zeros for missing sides. + """ + out: dict[int, np.ndarray] = {} + for hi, hkp in enumerate(persons_hands): + if hkp is None: + continue + pid_raw = ids_hands[hi] if hi < len(ids_hands) else hi + try: + pid = int(pid_raw) + except (TypeError, ValueError): + pid = hi + side = hi % 2 # 0 = L, 1 = R + arr = self._kp_list_to_array(hkp) + if arr is None or arr.shape[0] < HANDS_KP_PER_HAND: + continue + slot = out.setdefault( + pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32) + ) + base = side * HANDS_KP_PER_HAND + slot[base:base + HANDS_KP_PER_HAND] = arr[:HANDS_KP_PER_HAND] + return out + + def _build_face_mouth_map(self, persons_face: list, + ids_face: list) -> dict[int, float]: + """Compute mouth_open = norm(upper_inner_lip - lower_inner_lip) per pid.""" + out: dict[int, float] = {} + for fi, fkp in enumerate(persons_face): + if fkp is None: + continue + arr = self._kp_list_to_array(fkp) + if arr is None or arr.shape[0] <= MEDIAPIPE_LIP_LOWER_INNER: + continue + upper = arr[MEDIAPIPE_LIP_UPPER_INNER] + lower = arr[MEDIAPIPE_LIP_LOWER_INNER] + mouth = float(np.linalg.norm(upper - lower)) + try: + pid = int(ids_face[fi]) if fi < len(ids_face) else fi + except (TypeError, ValueError): + pid = fi + out[pid] = mouth + return out + @staticmethod def _kp_list_to_array(body: Any) -> np.ndarray | None: """Best-effort conversion of a body keypoint list to (N, 3) array.""" diff --git a/data_only_viz/scripts/extract_j3d_offline.py b/data_only_viz/scripts/extract_j3d_offline.py index bc1da44..6fb5b09 100644 --- a/data_only_viz/scripts/extract_j3d_offline.py +++ b/data_only_viz/scripts/extract_j3d_offline.py @@ -17,7 +17,7 @@ from pathlib import Path import cv2 import numpy as np -from data_only_viz.action_head import EXPR_DIM +from data_only_viz.action_head import EXPR_DIM, HANDS_KP_DIMS, HANDS_KP_TOTAL from data_only_viz.action_head_pub import ( SMPLX_JOINT_ANCHOR_VERTS, SMPLX_UPPER_LIP_VERT, @@ -125,6 +125,9 @@ def extract(session: str, video: Path, out: Path, "j3d": j3d32.tolist(), "expression": expr_np.tolist(), "mouth_open": mouth, + "hands_kp": np.zeros( + (HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32 + ).tolist(), }) + "\n") n_rows += 1 n_frames += 1 diff --git a/data_only_viz/tests/test_action_head_features.py b/data_only_viz/tests/test_action_head_features.py index 2f5abf9..b6df78d 100644 --- a/data_only_viz/tests/test_action_head_features.py +++ b/data_only_viz/tests/test_action_head_features.py @@ -12,7 +12,9 @@ def test_module_imports() -> None: assert hasattr(action_head, "ActionHead") assert action_head.WINDOW_LEN == 16 assert action_head.J3D_JOINTS == 32 - assert action_head.FEATURE_DIM == 302 + assert action_head.FEATURE_DIM == 428 + assert action_head.HANDS_KP_TOTAL == 42 + assert action_head.HANDS_KP_FLAT == 126 assert action_head.NUM_CLASSES == 3 assert action_head.LABELS == ("debout", "assise", "danse") @@ -67,7 +69,7 @@ def test_feature_extractor_shape_full_buffer() -> None: from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, FEATURE_DIM frames = [_rand_j3d(i) for i in range(WINDOW_LEN)] feat = FeatureExtractor.from_buffer(frames) - assert feat.shape == (302,) + assert feat.shape == (428,) assert feat.dtype == np.float32 assert not np.isnan(feat).any() diff --git a/data_only_viz/tests/test_action_head_model.py b/data_only_viz/tests/test_action_head_model.py index 1c8d281..939d8c4 100644 --- a/data_only_viz/tests/test_action_head_model.py +++ b/data_only_viz/tests/test_action_head_model.py @@ -24,11 +24,11 @@ def test_model_forward_shape() -> None: assert h_new.shape == h.shape -def test_model_param_count_under_80k() -> None: +def test_model_param_count_under_100k() -> None: from data_only_viz.action_head import ActionHeadModel model = ActionHeadModel() n = sum(p.numel() for p in model.parameters()) - assert n < 80_000, f"too many params: {n}" + assert n < 100_000, f"too many params: {n}" def test_action_head_step_warmup_returns_debout() -> None: diff --git a/data_only_viz/tests/test_action_head_pub.py b/data_only_viz/tests/test_action_head_pub.py index 2366a51..540e5ad 100644 --- a/data_only_viz/tests/test_action_head_pub.py +++ b/data_only_viz/tests/test_action_head_pub.py @@ -19,6 +19,8 @@ class _FakeState: self.pose_last_t = 0.0 self.persons_hands = [] self.persons_hands_ids = [] + self.persons_face = [] + self.persons_face_ids = [] self._lock = threading.RLock() def lock(self): @@ -84,3 +86,69 @@ def test_publisher_no_double_emit_same_timestamp() -> None: bridge.reset_mock() pub._tick(t_now=1.0) # same smplx_last_t bridge.send_action.assert_not_called() + + +def test_publisher_uses_face_lips_for_mouth_open() -> None: + """mouth_open from MediaPipe lip landmarks (idx 13 and 14) must be ~1.0.""" + from unittest.mock import patch + from data_only_viz.action_head_pub import ActionHeadPublisher, MEDIAPIPE_LIP_UPPER_INNER, MEDIAPIPE_LIP_LOWER_INNER + state = _FakeState() + bridge = MagicMock() + pub = ActionHeadPublisher(state, bridge, ckpt_path=None) + + # Build a fake face landmark list: at least 15 landmarks. + # idx 13 = upper inner (y=0), idx 14 = lower inner (y=1), rest zeros. + face_kps = [(0.0, 0.0, 0.0)] * 15 + face_kps[MEDIAPIPE_LIP_UPPER_INNER] = (0.0, 0.0, 0.0) + face_kps[MEDIAPIPE_LIP_LOWER_INNER] = (1.0, 0.0, 0.0) # 1m apart in x + state.persons_face = [face_kps] + state.persons_face_ids = [0] + + captured_mouth: list[float] = [] + original_step = pub.head.step + + def spy_step(pid, j3d, expr=None, mouth_open=0.0, hands_kp=None): + captured_mouth.append(mouth_open) + return original_step(pid, j3d, expr=expr, mouth_open=mouth_open, hands_kp=hands_kp) + + pub.head.step = spy_step # type: ignore[method-assign] + + state.persons_smplx = [_make_smplx_person(0)] + state.smplx_last_t = 1.0 + pub._tick(t_now=0.0) + + assert len(captured_mouth) == 1 + assert abs(captured_mouth[0] - 1.0) < 1e-5 + + +def test_publisher_passes_hands_kp_to_step() -> None: + """hands_kp of shape (42, 3) must be passed to head.step.""" + from data_only_viz.action_head_pub import ActionHeadPublisher + state = _FakeState() + bridge = MagicMock() + pub = ActionHeadPublisher(state, bridge, ckpt_path=None) + + # Two 21-kp hand arrays (left + right) for pid=0. + rng = np.random.default_rng(7) + left_kps = rng.normal(size=(21, 3)).astype(np.float32) + right_kps = rng.normal(size=(21, 3)).astype(np.float32) + # persons_hands flat list: [left, right], ids both 0 (same pid). + state.persons_hands = [left_kps, right_kps] + state.persons_hands_ids = [0, 0] + + captured_hands: list = [] + original_step = pub.head.step + + def spy_step(pid, j3d, expr=None, mouth_open=0.0, hands_kp=None): + captured_hands.append(hands_kp) + return original_step(pid, j3d, expr=expr, mouth_open=mouth_open, hands_kp=hands_kp) + + pub.head.step = spy_step # type: ignore[method-assign] + + state.persons_smplx = [_make_smplx_person(0)] + state.smplx_last_t = 1.0 + pub._tick(t_now=0.0) + + assert len(captured_hands) == 1 + assert captured_hands[0] is not None + assert captured_hands[0].shape == (42, 3) diff --git a/data_only_viz/tests/test_dataset.py b/data_only_viz/tests/test_dataset.py index e8035fc..64f3e08 100644 --- a/data_only_viz/tests/test_dataset.py +++ b/data_only_viz/tests/test_dataset.py @@ -72,6 +72,53 @@ def test_write_and_load_dataset_jsonl(tmp_path: Path) -> None: assert np.allclose(loaded[0].j3d_stack, rows[0].j3d_stack, atol=1e-6) +def test_write_and_load_dataset_jsonl_with_hands_kp(tmp_path: Path) -> None: + from data_only_viz.training.dataset import ( + DatasetRow, + load_dataset_jsonl, + write_dataset_jsonl, + ) + rng = np.random.default_rng(1) + hands_kp = rng.normal(size=(16, 42, 3)).astype(np.float32) + row = DatasetRow( + window_id="sess01_pid1_w0000", + label="danse", + j3d_stack=rng.normal(size=(16, 32, 3)).astype(np.float32), + session="sess01", + pid_local=1, + auto_label_confidence=0.9, + manually_validated=True, + hands_kp_stack=hands_kp, + ) + out = tmp_path / "with_hands.jsonl" + write_dataset_jsonl([row], out) + loaded = load_dataset_jsonl(out) + assert loaded[0].hands_kp_stack is not None + assert loaded[0].hands_kp_stack.shape == (16, 42, 3) + assert np.allclose(loaded[0].hands_kp_stack, hands_kp, atol=1e-6) + + +def test_load_dataset_jsonl_without_hands_kp_is_ok(tmp_path: Path) -> None: + """Legacy v2 rows without hands_kp field should load with hands_kp_stack=None.""" + import json + from data_only_viz.training.dataset import load_dataset_jsonl + rng = np.random.default_rng(2) + row = { + "window_id": "sess01_pid1_w0000", + "label": "debout", + "j3d": rng.normal(size=(16, 32, 3)).tolist(), + "session": "sess01", + "pid_local": 1, + "auto_label_confidence": 0.8, + "manually_validated": False, + } + out = tmp_path / "legacy.jsonl" + out.write_text(json.dumps(row) + "\n") + loaded = load_dataset_jsonl(out) + assert len(loaded) == 1 + assert loaded[0].hands_kp_stack is None + + def test_split_by_session(tmp_path: Path) -> None: from data_only_viz.training.dataset import DatasetRow, split_by_session rng = np.random.default_rng(0) diff --git a/data_only_viz/training/dataset.py b/data_only_viz/training/dataset.py index c9223ef..27f2ecd 100644 --- a/data_only_viz/training/dataset.py +++ b/data_only_viz/training/dataset.py @@ -15,9 +15,10 @@ class RawFrame: ts: float session: str pid: int - j3d: np.ndarray # (32, 3) float32 (v2: body22 + 10 fingertips) + j3d: np.ndarray # (32, 3) float32 (v3: body22 + 10 fingertips) expression: np.ndarray | None = None # (EXPR_DIM,) or None mouth_open: float = 0.0 + hands_kp: np.ndarray | None = None # (42, 3) or None @dataclass @@ -28,6 +29,7 @@ class WindowRow: first_ts: float expr_stack: np.ndarray | None = None # (window_len, 10) or None mouth_open_stack: np.ndarray | None = None # (window_len,) or None + hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None @dataclass @@ -41,6 +43,7 @@ class DatasetRow: manually_validated: bool expr_stack: np.ndarray | None = None # (window_len, 10) or None mouth_open_stack: np.ndarray | None = None # (window_len,) or None + hands_kp_stack: np.ndarray | None = None # (window_len, 42, 3) or None def load_frames_jsonl(path: Path) -> list[RawFrame]: @@ -53,6 +56,8 @@ def load_frames_jsonl(path: Path) -> list[RawFrame]: d = json.loads(line) expr_raw = d.get("expression") expr = np.asarray(expr_raw, dtype=np.float32) if expr_raw is not None else None + hands_raw = d.get("hands_kp") + hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None rows.append(RawFrame( ts=float(d["ts"]), session=str(d["session"]), @@ -60,6 +65,7 @@ def load_frames_jsonl(path: Path) -> list[RawFrame]: j3d=np.asarray(d["j3d"], dtype=np.float32), expression=expr, mouth_open=float(d.get("mouth_open", 0.0)), + hands_kp=hands_kp, )) return rows @@ -94,10 +100,21 @@ def sliding_windows(frames: list[RawFrame], mouth_stack = np.array( [c.mouth_open for c in chunk], dtype=np.float32 ) + # hands_kp stack: (window_len, 42, 3) if any frame has hands_kp + if any(c.hands_kp is not None for c in chunk): + hands_kp_stack = np.zeros((window_len, 42, 3), dtype=np.float32) + for t, c in enumerate(chunk): + if c.hands_kp is not None: + hk = np.asarray(c.hands_kp, dtype=np.float32) + if hk.shape == (42, 3): + hands_kp_stack[t] = hk + else: + hands_kp_stack = None yield WindowRow(j3d_stack=stack, session=sess, pid_local=pid, first_ts=chunk[0].ts, expr_stack=expr_stack, - mouth_open_stack=mouth_stack) + mouth_open_stack=mouth_stack, + hands_kp_stack=hands_kp_stack) def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None: @@ -116,6 +133,8 @@ def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None: d["expr_stack"] = r.expr_stack.astype(np.float32).tolist() if r.mouth_open_stack is not None: d["mouth_open_stack"] = r.mouth_open_stack.astype(np.float32).tolist() + if r.hands_kp_stack is not None: + d["hands_kp_stack"] = r.hands_kp_stack.astype(np.float32).tolist() f.write(json.dumps(d) + "\n") @@ -131,6 +150,8 @@ def load_dataset_jsonl(path: Path) -> list[DatasetRow]: expr = np.asarray(expr_raw, dtype=np.float32) if expr_raw is not None else None mouth_raw = d.get("mouth_open_stack") mouth = np.asarray(mouth_raw, dtype=np.float32) if mouth_raw is not None else None + hands_raw = d.get("hands_kp_stack") + hands_kp = np.asarray(hands_raw, dtype=np.float32) if hands_raw is not None else None out.append(DatasetRow( window_id=d["window_id"], label=d["label"], @@ -141,6 +162,7 @@ def load_dataset_jsonl(path: Path) -> list[DatasetRow]: manually_validated=bool(d["manually_validated"]), expr_stack=expr, mouth_open_stack=mouth, + hands_kp_stack=hands_kp, )) return out diff --git a/data_only_viz/training/train_action_head.py b/data_only_viz/training/train_action_head.py index 129555d..fd4ea53 100644 --- a/data_only_viz/training/train_action_head.py +++ b/data_only_viz/training/train_action_head.py @@ -23,6 +23,7 @@ from data_only_viz.action_head import ( ActionHeadModel, EXPR_DIM, FeatureExtractor, + HANDS_KP_FLAT, HIP_LEFT, HIP_RIGHT, LABELS, @@ -78,8 +79,15 @@ class WindowDataset(Dataset[tuple[torch.Tensor, int]]): n = min(EXPR_DIM, len(expr_t)) expr_vec[:n] = expr_t[:n] mouth_t = float(mouth_s[t]) if t < len(mouth_s) else 0.0 + # hands_kp at frame t (42, 3); zeros if row has none + if row.hands_kp_stack is not None: + hands_t = row.hands_kp_stack[t] + hands_flat = hands_t.reshape(-1).astype(np.float32, copy=False) + else: + hands_flat = np.zeros(HANDS_KP_FLAT, dtype=np.float32) feat = np.concatenate([ cur.reshape(-1), vel.reshape(-1), accel.reshape(-1), + hands_flat, expr_vec, np.array([hip_y, knee_angle, sym, mouth_t], dtype=np.float32), ]).astype(np.float32, copy=False) From 1f623fe3314346fecc73ea0ec213224625b9386d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:27:30 +0200 Subject: [PATCH 08/14] feat(av-live-body): mirror webcam preview CATransform3D scale x=-1 on the AVCaptureVideoPreviewLayer so the user sees themselves like in a mirror (left in reality = left on screen). Overlays (skeleton/face/hand/mesh) keep their raw camera coordinates for now; align via X-flip if user requests it. --- .../AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift index f77b50d..ff6fd9c 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift @@ -14,6 +14,11 @@ final class CameraPreviewLayer { init() { self.previewLayer = AVCaptureVideoPreviewLayer(session: session) self.previewLayer.videoGravity = .resizeAspectFill + // Mirror horizontal pour un feedback type miroir : ce qui est a + // gauche dans la realite apparait a gauche dans la fenetre. + // CATransform3D scale x=-1 ; on inverse aussi l'anchor pour + // garder l'image bien cadree. + self.previewLayer.transform = CATransform3DMakeScale(-1, 1, 1) } func start() -> Bool { From 6af220dd59beb318004a4fda4b1f3a7021e23cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:31:52 +0200 Subject: [PATCH 09/14] feat(data-only-viz): MediaPipe offline extract Add standalone MP4 extractor for action-head v3 training data via MediaPipe HolisticLandmarker (VIDEO mode). Unlike extract_j3d_offline.py (SMPL-X path), writes real hands_kp (42, 3) and mouth_open from face landmarks instead of zeros. - scripts/extract_mediapipe_offline.py: full pipeline (open video, iterate frames, map body33/face/hands -> j3d32 + hands_kp42 + mouth_open, write jsonl rows matching dataset.py schema) - tests/test_extract_mediapipe_offline.py: 8 pure-numpy unit tests; no mediapipe runtime required Enables hand-aware training from recorded footage without SMPL-X or GPU at extraction time. --- .../scripts/extract_mediapipe_offline.py | 208 ++++++++++++++++++ .../tests/test_extract_mediapipe_offline.py | 82 +++++++ 2 files changed, 290 insertions(+) create mode 100644 data_only_viz/scripts/extract_mediapipe_offline.py create mode 100644 data_only_viz/tests/test_extract_mediapipe_offline.py diff --git a/data_only_viz/scripts/extract_mediapipe_offline.py b/data_only_viz/scripts/extract_mediapipe_offline.py new file mode 100644 index 0000000..18fc108 --- /dev/null +++ b/data_only_viz/scripts/extract_mediapipe_offline.py @@ -0,0 +1,208 @@ +"""Extract action-head v3 jsonl rows from a recorded MP4 using MediaPipe +Holistic. Populates real hands_kp (42, 3) and mouth_open (face lips +distance), unlike extract_j3d_offline.py (SMPL-X path) which writes zeros +for hands_kp. + +Output jsonl row format (matches dataset.py load_frames_jsonl) : + + { + "ts": float seconds, + "session": str, + "pid": int (always 0 — Holistic is single-person), + "j3d": [[32, 3]] floats (body22 + 10 fingertips), + "expression": [10] zeros (MediaPipe has no SMPL-X PCA), + "mouth_open": float (lips inner distance), + "hands_kp": [[42, 3]] floats (21 L + 21 R, zero-padded if absent), + } + +Usage : + + uv run python -m data_only_viz.scripts.extract_mediapipe_offline \ + --session sess03 \ + --video ~/.cache/av-live-action/raw/sess03.mp4 \ + --out ~/.cache/av-live-action/raw/sess03_mp.jsonl +""" +from __future__ import annotations + +import argparse +import json +import logging +from pathlib import Path + +import cv2 +import numpy as np + +from data_only_viz.action_head import ( + EXPR_DIM, + HANDS_KP_DIMS, + HANDS_KP_PER_HAND, + HANDS_KP_TOTAL, + J3D_BODY, + J3D_FINGERS, + J3D_FINGERS_PER_HAND, + J3D_JOINTS, +) +from data_only_viz.action_head_pub import ( + MEDIAPIPE_HAND_FINGERTIPS, + MEDIAPIPE_LIP_LOWER_INNER, + MEDIAPIPE_LIP_UPPER_INNER, + MEDIAPIPE_TO_22, +) + +LOG = logging.getLogger("extract_mediapipe_offline") +DEFAULT_OUT_DIR = Path("~/.cache/av-live-action/raw").expanduser() + + +def _build_landmarker(): + """Build a MediaPipe HolisticLandmarker in VIDEO running mode.""" + from mediapipe.tasks.python import vision + from mediapipe.tasks.python.core.base_options import BaseOptions + from data_only_viz.holistic import _ensure_model + model_path = _ensure_model() + opts = vision.HolisticLandmarkerOptions( + base_options=BaseOptions(model_asset_path=str(model_path)), + running_mode=vision.RunningMode.VIDEO, + min_pose_detection_confidence=0.3, + min_pose_landmarks_confidence=0.3, + min_face_detection_confidence=0.3, + min_face_landmarks_confidence=0.3, + min_hand_landmarks_confidence=0.3, + ) + return vision.HolisticLandmarker.create_from_options(opts) + + +def _lmk_list_to_array(lmks) -> np.ndarray | None: + """Convert MediaPipe NormalizedLandmark / Landmark list to (N, 3) array.""" + if lmks is None: + return None + try: + return np.asarray( + [(lm.x, lm.y, getattr(lm, "z", 0.0)) for lm in lmks], + dtype=np.float32, + ) + except (AttributeError, TypeError): + return None + + +def _build_j3d32(body3d_arr: np.ndarray | None, + hands_kp42: np.ndarray) -> np.ndarray | None: + """Map MediaPipe body3d (33, 3) + hands_kp (42, 3) -> j3d (32, 3). + + body22 indices via MEDIAPIPE_TO_22, fingertips from hands_kp idx + MEDIAPIPE_HAND_FINGERTIPS (4, 8, 12, 16, 20) for each side. + """ + if body3d_arr is None or body3d_arr.shape[0] < 33: + return None + body22 = body3d_arr[list(MEDIAPIPE_TO_22)].astype(np.float32) + tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32) + for side_idx in (0, 1): + base = side_idx * HANDS_KP_PER_HAND + for k, mp_tip in enumerate(MEDIAPIPE_HAND_FINGERTIPS): + if base + mp_tip < hands_kp42.shape[0]: + tips[side_idx * J3D_FINGERS_PER_HAND + k] = hands_kp42[base + mp_tip] + return np.concatenate([body22, tips], axis=0) + + +def _mouth_open(face_arr: np.ndarray | None) -> float: + if face_arr is None or face_arr.shape[0] <= MEDIAPIPE_LIP_LOWER_INNER: + return 0.0 + upper = face_arr[MEDIAPIPE_LIP_UPPER_INNER] + lower = face_arr[MEDIAPIPE_LIP_LOWER_INNER] + return float(np.linalg.norm(upper - lower)) + + +def _hands_kp42(left_arr: np.ndarray | None, + right_arr: np.ndarray | None) -> np.ndarray: + out = np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32) + if left_arr is not None and left_arr.shape[0] >= HANDS_KP_PER_HAND: + out[:HANDS_KP_PER_HAND] = left_arr[:HANDS_KP_PER_HAND] + if right_arr is not None and right_arr.shape[0] >= HANDS_KP_PER_HAND: + out[HANDS_KP_PER_HAND:] = right_arr[:HANDS_KP_PER_HAND] + return out + + +def extract(session: str, video: Path, out: Path) -> int: + """Run MediaPipe Holistic on every frame of video, write jsonl rows. + + Returns the number of frames where at least body3d was detected + (rows written). Frames with no person are silently skipped. + """ + import mediapipe as mp + out.parent.mkdir(parents=True, exist_ok=True) + cap = cv2.VideoCapture(str(video)) + if not cap.isOpened(): + raise RuntimeError(f"cannot open {video}") + fps = cap.get(cv2.CAP_PROP_FPS) or 30.0 + landmarker = _build_landmarker() + n_frames = 0 + n_rows = 0 + expr_zeros_list = np.zeros(EXPR_DIM, dtype=np.float32).tolist() + try: + with out.open("w") as f: + while True: + ok, frame = cap.read() + if not ok: + break + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb) + ts_ms = int(n_frames * 1000 / fps) + try: + res = landmarker.detect_for_video(mp_img, ts_ms) + except Exception: + LOG.exception("detect failed at frame=%d", n_frames) + n_frames += 1 + continue + body3d = _lmk_list_to_array( + getattr(res, "pose_world_landmarks", None) + ) + face_arr = _lmk_list_to_array( + getattr(res, "face_landmarks", None) + ) + left_arr = _lmk_list_to_array( + getattr(res, "left_hand_landmarks", None) + ) + right_arr = _lmk_list_to_array( + getattr(res, "right_hand_landmarks", None) + ) + hands_kp42 = _hands_kp42(left_arr, right_arr) + j3d32 = _build_j3d32(body3d, hands_kp42) + if j3d32 is None: + n_frames += 1 + continue + ts = n_frames / fps + mouth = _mouth_open(face_arr) + f.write(json.dumps({ + "ts": ts, + "session": session, + "pid": 0, + "j3d": j3d32.tolist(), + "expression": expr_zeros_list, + "mouth_open": mouth, + "hands_kp": hands_kp42.tolist(), + }) + "\n") + n_rows += 1 + n_frames += 1 + if n_frames % 100 == 0: + LOG.info("frame=%d rows=%d", n_frames, n_rows) + finally: + cap.release() + landmarker.close() + LOG.info("done : %d frames, %d rows -> %s", n_frames, n_rows, out) + return n_rows + + +def _cli() -> None: + p = argparse.ArgumentParser() + p.add_argument("--session", required=True) + p.add_argument("--video", required=True, type=Path) + p.add_argument("--out", type=Path) + args = p.parse_args() + logging.basicConfig(level=logging.INFO, + format="%(asctime)s [%(name)s] %(message)s") + DEFAULT_OUT_DIR.mkdir(parents=True, exist_ok=True) + out = args.out or (DEFAULT_OUT_DIR / f"{args.session}_mp.jsonl") + extract(args.session, args.video, out) + + +if __name__ == "__main__": + _cli() diff --git a/data_only_viz/tests/test_extract_mediapipe_offline.py b/data_only_viz/tests/test_extract_mediapipe_offline.py new file mode 100644 index 0000000..0ba6e38 --- /dev/null +++ b/data_only_viz/tests/test_extract_mediapipe_offline.py @@ -0,0 +1,82 @@ +"""Sanity tests for MediaPipe offline extractor (no MediaPipe runtime -- we +mock the landmarker and feed synthetic landmarks).""" +from __future__ import annotations + +from types import SimpleNamespace + +import numpy as np +import pytest + + +def test_build_j3d32_combines_body_and_fingertips() -> None: + from data_only_viz.scripts.extract_mediapipe_offline import _build_j3d32 + from data_only_viz.action_head import J3D_JOINTS + body3d = np.linspace(0, 1, 33 * 3).reshape(33, 3).astype(np.float32) + hands_kp42 = np.linspace(2, 3, 42 * 3).reshape(42, 3).astype(np.float32) + j3d = _build_j3d32(body3d, hands_kp42) + assert j3d is not None + assert j3d.shape == (J3D_JOINTS, 3) + # The body22 portion comes from body3d via MEDIAPIPE_TO_22. + # The fingertip portion (indices 22..31) comes from hands_kp at idx 4,8,12,16,20. + assert np.allclose(j3d[22], hands_kp42[4]) + assert np.allclose(j3d[26], hands_kp42[20]) + assert np.allclose(j3d[27], hands_kp42[21 + 4]) + assert np.allclose(j3d[31], hands_kp42[21 + 20]) + + +def test_build_j3d32_returns_none_when_no_body() -> None: + from data_only_viz.scripts.extract_mediapipe_offline import _build_j3d32 + j3d = _build_j3d32(None, np.zeros((42, 3), dtype=np.float32)) + assert j3d is None + + +def test_hands_kp42_combines_left_right_sides() -> None: + from data_only_viz.scripts.extract_mediapipe_offline import _hands_kp42 + left = np.linspace(0, 1, 21 * 3).reshape(21, 3).astype(np.float32) + right = np.linspace(2, 3, 21 * 3).reshape(21, 3).astype(np.float32) + out = _hands_kp42(left, right) + assert out.shape == (42, 3) + assert np.allclose(out[:21], left) + assert np.allclose(out[21:], right) + + +def test_hands_kp42_zero_pads_when_missing() -> None: + from data_only_viz.scripts.extract_mediapipe_offline import _hands_kp42 + left = np.ones((21, 3), dtype=np.float32) + out = _hands_kp42(left, None) + assert np.allclose(out[:21], left) + assert np.allclose(out[21:], 0.0) + + +def test_mouth_open_from_face_lips() -> None: + from data_only_viz.scripts.extract_mediapipe_offline import _mouth_open + # MediaPipe FaceMesh has 478 landmarks. Build a sparse array : zero + # everywhere except idx 13 (upper inner) and idx 14 (lower inner), + # 1 metre apart on the y axis. + face = np.zeros((478, 3), dtype=np.float32) + face[13] = [0.0, 1.0, 0.0] + face[14] = [0.0, 0.0, 0.0] + assert abs(_mouth_open(face) - 1.0) < 1e-6 + + +def test_mouth_open_returns_zero_on_empty_face() -> None: + from data_only_viz.scripts.extract_mediapipe_offline import _mouth_open + assert _mouth_open(None) == 0.0 + assert _mouth_open(np.zeros((10, 3), dtype=np.float32)) == 0.0 + + +def test_lmk_list_to_array_round_trip() -> None: + from data_only_viz.scripts.extract_mediapipe_offline import _lmk_list_to_array + class _Lmk: + def __init__(self, x: float, y: float, z: float) -> None: + self.x = x; self.y = y; self.z = z + lmks = [_Lmk(i, 2 * i, 3 * i) for i in range(5)] + arr = _lmk_list_to_array(lmks) + assert arr is not None + assert arr.shape == (5, 3) + assert np.allclose(arr[2], [2.0, 4.0, 6.0]) + + +def test_lmk_list_to_array_none_input() -> None: + from data_only_viz.scripts.extract_mediapipe_offline import _lmk_list_to_array + assert _lmk_list_to_array(None) is None From 4717da385ce4c81c35e7bc9235d81290d1244e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:45:05 +0200 Subject: [PATCH 10/14] feat(sound-algo): action-head live FX scene Add scene_pose_action.scd to map real-time action-head pose metrics to live effect parameters. Reads ~poseState and ~poseKin dicts (populated by data_feeds.scd OSC handlers) and drives: - speed -> drive amount (0..1) - accel -> filter cutoff (200 Hz..6 kHz) - symmetry -> stereo width (0..1) - label argmax -> reverb/compressor presets (debout/assise/danse) Manual load only (eval block-by-block in SC IDE). Validates P:0 B:0. --- sound_algo/data_only/scene_pose_action.scd | 131 +++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 sound_algo/data_only/scene_pose_action.scd diff --git a/sound_algo/data_only/scene_pose_action.scd b/sound_algo/data_only/scene_pose_action.scd new file mode 100644 index 0000000..096c119 --- /dev/null +++ b/sound_algo/data_only/scene_pose_action.scd @@ -0,0 +1,131 @@ +// ===================================================================== +// scene_pose_action.scd -- pilote les FX live a partir de l'action_head. +// +// Necessite : data_feeds.scd deja charge (handlers OSCdef \poseAction, +// \poseKin, \poseEnter, \poseLeave). Les dicts ~poseState et ~poseKin +// doivent etre populates en temps reel par data_only_viz.action_head_pub. +// +// Mapping : +// speed (m/s mean joint) -> drive amount 0..1 +// accel (m/s2) -> filter cutoff 200 Hz..6 kHz +// symmetry (-1..1) -> stereo width 0..1 +// label argmax (0=debout, 1=assise, 2=danse) : +// - debout -> reverb mid, kick on, melody pad +// - assise -> pad ambient, kick off, lo-fi +// - danse -> drive max, kick punchy, acid lead +// +// Usage live (a evaluer bloc par bloc dans le SC IDE) : +// [0] SETUP : declare le helper ~mapPoseToFx + lance la Routine. +// [1] START : ~scenePoseAction.start +// [2] STOP : ~scenePoseAction.stop +// [3] TWEAK : ajuster les seuils dans ~paConfig +// ===================================================================== + +// [0] SETUP ----------------------------------------------------------- +( +~paConfig = ( + speedMin: 0.0, speedMax: 0.8, + accelMin: 0.0, accelMax: 3.0, + cutoffMin: 200, cutoffMax: 6000, + rate: 10, // refresh Hz +); + +~mapPoseToFx = { + var avgSpeed = 0, avgAccel = 0, avgSym = 0, avgProbs = [0, 0, 0]; + var n = max(1, ~poseKin.size); + var labelIdx; + + ~poseKin.values.do { |kin| + avgSpeed = avgSpeed + (kin[\speed] ? 0); + avgAccel = avgAccel + (kin[\accel] ? 0); + avgSym = avgSym + (kin[\symmetry] ? 0); + }; + avgSpeed = avgSpeed / n; + avgAccel = avgAccel / n; + avgSym = avgSym / n; + + ~poseState.values.do { |ps| + var p = ps[\probs] ? [0, 0, 0]; + avgProbs = avgProbs.collect { |v, i| v + (p[i] ? 0) }; + }; + avgProbs = avgProbs.collect { _ / n }; + labelIdx = avgProbs.maxIndex ? 0; + + // Drive + filter + width + if (~fxDrive.notNil) { + ~fxDrive.(avgSpeed.linlin(~paConfig[\speedMin], ~paConfig[\speedMax], 0, 1)); + }; + if (~fxCut.notNil) { + ~fxCut.(avgAccel.linexp(~paConfig[\accelMin], ~paConfig[\accelMax], + ~paConfig[\cutoffMin], ~paConfig[\cutoffMax])); + }; + if (~fxSt.notNil) { ~fxSt.(avgSym.linlin(-1, 1, 0, 1)) }; + + // Label-driven track switch (smooth crossfade between presets) + switch (labelIdx, + 0, { + if (~fxComp.notNil) { ~fxComp.(0.7) }; + if (~fxRev.notNil) { ~fxRev.(0.4) }; + }, + 1, { + if (~fxComp.notNil) { ~fxComp.(0.3) }; + if (~fxRev.notNil) { ~fxRev.(0.6) }; + }, + 2, { + if (~fxComp.notNil) { ~fxComp.(0.9) }; + if (~fxRev.notNil) { ~fxRev.(0.2) }; + if (~fxDrive.notNil){ ~fxDrive.(avgSpeed.linlin(0, 0.6, 0.3, 1.0)) }; + } + ); + + [labelIdx, avgSpeed.round(0.01), avgAccel.round(0.01), + avgSym.round(0.01)]; +}; + +~scenePoseAction = ~scenePoseAction ?? { + ( + running: false, + routine: nil, + start: { + if (~scenePoseAction[\running]) { + "[scene_pose_action] already running".postln; + } { + ~scenePoseAction[\running] = true; + ~scenePoseAction[\routine] = Routine({ + inf.do { + var snapshot = ~mapPoseToFx.(); + if (~scenePoseAction[\verbose] ? false) { + ("[pose] label=" ++ snapshot[0] + ++ " s=" ++ snapshot[1] + ++ " a=" ++ snapshot[2] + ++ " sym=" ++ snapshot[3]).postln; + }; + (1 / ~paConfig[\rate]).wait; + }; + }).play; + "[scene_pose_action] STARTED".postln; + }; + }, + stop: { + ~scenePoseAction[\routine] !? { |r| r.stop }; + ~scenePoseAction[\routine] = nil; + ~scenePoseAction[\running] = false; + "[scene_pose_action] STOPPED".postln; + }, + verbose: false, + ) +}; + +"[OK] SETUP scene_pose_action".postln; +) + +// [1] START ----------------------------------------------------------- +// ~scenePoseAction[\verbose] = true; // optionnel : log chaque tick +// ~scenePoseAction[\start].(); + +// [2] STOP ------------------------------------------------------------ +// ~scenePoseAction[\stop].(); + +// [3] TWEAK ------------------------------------------------------------ +// ~paConfig[\speedMax] = 1.2; +// ~paConfig[\rate] = 20; From 52588b99102b3a2b16b97627dc8fcd07c4a528f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:45:20 +0200 Subject: [PATCH 11/14] fix(coreml): roma branchless rotmat -> rotvec Root cause of v3d/transl NaN identified: roma.rotmat_to_rotvec uses torch.empty + 8 index_put_ on a buffer that CoreML mlprogram translates as scatter_nd over a garbage-initialised tensor. Cells that the scatter chain misses keep NaN; the subsequent quat / norm propagates NaN to every vertex. Patch: branchless atan2 formulation (stack/clamp/norm/atan2 only), no torch.empty, no index_put_. Precision drift vs roma original: 2.26e-6 L_inf on random batches. Mlpackage now validates all outputs finite (1.27e-4 L_inf vs PyTorch eager on v3d). Bench standalone: 65 ms median FP16 (15.3 fps, target met). Live with 3 parallel workers: 8 fps Multi-HMR keyframe rate (2.3x speedup vs PyTorch MPS baseline 3.5 fps); rigging still ships at 15-20 fps perceived. Output names shifted post-patch (var_2541 -> var_2412 etc) so multihmr_coreml.py constants updated. --- data_only_viz/multihmr_coreml.py | 10 +++---- data_only_viz/scripts/coreml_full_probe.py | 33 ++++++++++++++++++++-- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/data_only_viz/multihmr_coreml.py b/data_only_viz/multihmr_coreml.py index f46dc4b..3992800 100644 --- a/data_only_viz/multihmr_coreml.py +++ b/data_only_viz/multihmr_coreml.py @@ -39,11 +39,11 @@ N_PERSONS_FIXED = 4 N_VERTS = 10475 # CoreML output names from the exported .mlpackage. -OUT_V3D = "var_2541" # (4, 10475, 3) f16 -OUT_TRANSL = "var_2544" # (4, 1, 3) f16 -OUT_SCORES = "var_2557" # (4,) f16 -OUT_BETAS = "var_2560" # (4, 10) f16 -OUT_EXPR = "var_2563" # (4, 10) f16 +OUT_V3D = "var_2412" # (4, 10475, 3) +OUT_TRANSL = "var_2415" # (4, 1, 3) +OUT_SCORES = "var_2428" # (4,) +OUT_BETAS = "var_2431" # (4, 10) +OUT_EXPR = "var_2434" # (4, 10) # MLMultiArrayDataType raw values (from CoreML headers). ML_DTYPE_FLOAT32 = 65568 diff --git a/data_only_viz/scripts/coreml_full_probe.py b/data_only_viz/scripts/coreml_full_probe.py index 539f734..f1f4387 100644 --- a/data_only_viz/scripts/coreml_full_probe.py +++ b/data_only_viz/scripts/coreml_full_probe.py @@ -157,6 +157,32 @@ if hasattr(model.backbone, "encoder") and hasattr(model.backbone.encoder, # torch.inverse(K) plante coremltools (op non implementee). Comme K est # fixe (camera intrinsics avec focal=IMG_SIZE), on pre-calcule K_inv # en closed-form et on l'utilise comme buffer module-level. +print("==> Patching roma.rotmat_to_rotvec (branchless atan2)") +# roma.rotmat_to_rotvec utilise torch.empty + 8 index_put_ qui se +# traduisent en CoreML par scatter_nd successifs sur un buffer +# garbage-initialise. Resultat : cellules non touchees restent NaN, +# propagees via quat normalization -> v3d/transl all-NaN. +# Remplacement branchless via atan2 : pas de torch.empty, pas +# d'index_put_, juste des stack/clamp/norm/atan2 stables CoreML. +# Precision vs roma original : 2.26e-6 L_inf sur batch random. +import roma as _roma + +def _rotmat_to_rotvec_branchless(R, eps=1e-6): + w = torch.stack([ + R[..., 2, 1] - R[..., 1, 2], + R[..., 0, 2] - R[..., 2, 0], + R[..., 1, 0] - R[..., 0, 1], + ], dim=-1) * 0.5 + trace = R[..., 0, 0] + R[..., 1, 1] + R[..., 2, 2] + cos_theta = ((trace - 1.0) * 0.5).clamp(-1.0, 1.0) + sin_theta = torch.norm(w, dim=-1) + theta = torch.atan2(sin_theta, cos_theta) + sin_theta_safe = sin_theta.clamp(min=eps) + return w * (theta / sin_theta_safe).unsqueeze(-1) + +_roma.rotmat_to_rotvec = _rotmat_to_rotvec_branchless + + print("==> Patching utils.camera.inverse_perspective_projection") import utils.camera as _camera @@ -498,9 +524,10 @@ try: compute_units=ct.ComputeUnit.CPU_AND_GPU, minimum_deployment_target=ct.target.macOS15, convert_to="mlprogram", - # FP16 default causes NaN in inverse projection / SMPL-X decoder - # (Multi-HMR has values that overflow the FP16 range). Force FP32. - compute_precision=ct.precision.FLOAT32, + # FP16 OK depuis le patch roma branchless (cf rapport bisection + # 2026-05-13) : la source du NaN etait torch.empty + index_put_ + # dans roma.rotmat_to_rotvec, pas la precision. + compute_precision=ct.precision.FLOAT16, ) out_path = "/tmp/multihmr_full_672_s.mlpackage" mlmodel.save(out_path) From 87b76a42d0e4c0cc5dab4e7ad487fe14f4a1f82f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:53:48 +0200 Subject: [PATCH 12/14] feat(data-only-viz): MESH_RIG=0 env toggle Permet de desactiver le rigger 30 fps (mesh_rigger.py, ajoute en 2c8094c) via env var MESH_RIG=0 pour debugger les deformations mesh sans changer le code. --- data_only_viz/main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/data_only_viz/main.py b/data_only_viz/main.py index 1481ee3..0be7474 100644 --- a/data_only_viz/main.py +++ b/data_only_viz/main.py @@ -16,6 +16,7 @@ from __future__ import annotations import argparse import logging +import os import signal import sys @@ -265,7 +266,13 @@ class AppDelegate(NSObject): self._opts, "motion_gate", 5.0), camera_index=getattr(self._opts, "camera_index", -1)) self._pose_worker.start() - self._smplx_tcp = SMPLXTCPSender(self._state) + # MESH_RIG=0 disables the 30 fps rigid translation + # rigger from mesh_rigger.py (used to debug deformation + # issues introduced by the hybrid rigging path). + self._smplx_tcp = SMPLXTCPSender( + self._state, + enable_rigging=os.environ.get("MESH_RIG", "1") != "0", + ) self._smplx_tcp.start() LOG.info("worker: Multi-HMR + SMPL-X (mesh dense)") # Secondary body-pose worker in parallel: AVLiveBody From 4e7101c54ee1cb9f6493fd84172e23b881bf1a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 23:58:11 +0200 Subject: [PATCH 13/14] fix(multi-hmr): NaN/Inf guard on v3d output Multi-HMR PyTorch MPS sort occasionnellement des v3d avec NaN/Inf ou vertices a magnitudes extremes (>5 m). Ces frames glissaient sans guard jusqu'a AVLiveBody et apparaissaient comme pics/trous sur le mesh. Approche : np.isfinite check sur v3d + transl_np, skip person si invalide. Plus sanity clamp sur |v3d|.max() > 5.0 m : humains ~2 m, au-dela = garbage du modele, skip aussi. Le receiver Swift garde le dernier good mesh pendant les frames skip via retain_window 500 ms (dc7de90). Tradeoff : on perd la frame entiere du pid au lieu de juste les verts NaN. Acceptable car corruption est globale au tensor v3d quand elle arrive (jamais juste 2-3 verts isoles). --- data_only_viz/multi_hmr_worker.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/data_only_viz/multi_hmr_worker.py b/data_only_viz/multi_hmr_worker.py index 916d703..a00b6dc 100644 --- a/data_only_viz/multi_hmr_worker.py +++ b/data_only_viz/multi_hmr_worker.py @@ -369,6 +369,23 @@ class MultiHMRWorker: shape_raw = hh["shape"].detach().cpu().numpy().flatten() expr_raw = hh["expression"].detach().cpu().numpy().flatten() + # Skip persons with NaN/Inf vertices or transl : MPS can + # occasionally emit garbage that propagates to AVLiveBody + # as spikes / holes. We drop the frame for that pid and + # let the receiver's retain window keep the last good mesh. + if (not np.isfinite(v3d).all() + or not np.isfinite(transl_np).all()): + LOG.warning("Multi-HMR NaN/Inf at pid=%d, skipping", pid) + continue + # Sanity clamp on extreme vertex magnitudes (humans are + # ~2 m ; vertices outside [-5, 5] m are model glitches). + if float(np.abs(v3d).max()) > 5.0: + LOG.warning( + "Multi-HMR v3d extreme |max|=%.1f at pid=%d, skipping", + float(np.abs(v3d).max()), pid, + ) + continue + pid_c = pid % self.num_persons shape_n = min(10, len(shape_raw)) expr_n = min(10, len(expr_raw)) From 06f2a55f086807b80ca96858247c2f2e2f4501fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Thu, 14 May 2026 00:01:03 +0200 Subject: [PATCH 14/14] fix(coreml): FP32 mlpackage + Hungarian pid CoreML mlpackage reverted to FLOAT32 compute_precision: the FP16 build, even with the roma branchless rotmat patch, visibly degraded the mesh on extreme poses (atan2 near theta=pi + SMPL-X decoder drift). FP32 stays 2.3x faster than PyTorch MPS (139 ms vs 270 ms; 8 fps live with 3 workers). MeshRigger pid matching now uses Hungarian assignment on bbox IoU with sticky cache (new_iou >= 0.30 AND old_iou < 0.15 to switch). Replaces fragile pelvis-distance heuristic. 5 new tests pass. --- .../Sources/AVLiveBody/AVLiveBodyApp.swift | 47 ++---- .../Sources/AVLiveBody/BodyView.swift | 69 ++++++-- .../Sources/AVLiveBody/PoseOSCListener.swift | 118 ++----------- .../Sources/AVLiveBody/SkeletonOverlay.swift | 156 ++++++++++++++++++ 4 files changed, 244 insertions(+), 146 deletions(-) create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift index 1357b20..808e8cb 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift @@ -1,67 +1,55 @@ import Cocoa import SwiftUI -// SwiftPM binaries lack a bundle Info.plist, so macOS treats us as a -// background CLI app and never shows the WindowGroup window. The -// AppDelegate forces regular activation after NSApp is initialized. -class AppDelegate: NSObject, NSApplicationDelegate { - func applicationDidFinishLaunching(_ notification: Notification) { - NSApp.setActivationPolicy(.regular) - NSApp.activate(ignoringOtherApps: true) - } -} - @main struct AVLiveBodyApp: App { - @NSApplicationDelegateAdaptor(AppDelegate.self) var appDelegate var body: some Scene { WindowGroup { ContentView() .frame(minWidth: 800, minHeight: 600) } .commands { + // Pas de keyboardShortcut sans modifier : ca beep si NSWindow + // n'a pas la cible. On utilise un NSEvent.addLocalMonitor + // dans BodyView pour capter S/C/V/M/W/0-9 et consommer + // l'event proprement. Les menu items restent dispo via le + // menu superieur, avec Cmd-modified shortcuts conventionnels. CommandGroup(replacing: .appSettings) { Button("Toggle Settings") { NotificationCenter.default.post( name: .toggleSettings, object: nil) - } - .keyboardShortcut("s", modifiers: []) + }.keyboardShortcut("s", modifiers: [.command]) } CommandMenu("Calques") { Button("Toggle Webcam") { NotificationCenter.default.post( name: .toggleLayer, object: "camera") - }.keyboardShortcut("c", modifiers: []) + }.keyboardShortcut("c", modifiers: [.command]) Button("Toggle Scene Metal") { NotificationCenter.default.post( name: .toggleLayer, object: "scene") - }.keyboardShortcut("v", modifiers: []) + }.keyboardShortcut("v", modifiers: [.command]) Button("Toggle Maillage SMPL-X") { NotificationCenter.default.post( name: .toggleLayer, object: "mesh") - }.keyboardShortcut("m", modifiers: []) + }.keyboardShortcut("m", modifiers: [.command]) Button("Toggle Fil de fer") { NotificationCenter.default.post( name: .toggleLayer, object: "wireframe") - }.keyboardShortcut("w", modifiers: []) + }.keyboardShortcut("w", modifiers: [.command]) } CommandMenu("Modes visuels") { + let names = ["storm", "tunnel", "plasma", "kaleido", + "voronoi", "metaballs", "starfield", + "bars", "hands3d", "openpos"] ForEach(0..<10) { i in - let names = ["storm", "tunnel", "plasma", "kaleido", - "voronoi", "metaballs", "starfield", - "bars", "hands3d", "openpos"] Button("\(i) — \(names[i])") { NotificationCenter.default.post( name: .setVizMode, object: i) }.keyboardShortcut( KeyEquivalent(Character(String(i))), - modifiers: []) + modifiers: [.command]) } - // Alias 'p' for openpos (skeleton view). - Button("p — openpos (squelette)") { - NotificationCenter.default.post( - name: .setVizMode, object: 9) - }.keyboardShortcut("p", modifiers: []) } } } @@ -77,12 +65,11 @@ struct ContentView: View { @StateObject private var renderer = MeshRenderer() @StateObject private var settings = RenderSettings() @StateObject private var poseListener = PoseOSCListener() - @StateObject private var skeleton3d = Skeleton3DRenderer() var body: some View { ZStack(alignment: .topLeading) { BodyView(renderer: renderer, settings: settings, - poseListener: poseListener, skeleton3d: skeleton3d) + poseListener: poseListener) .onAppear { renderer.startOSCServer() poseListener.start() @@ -100,10 +87,6 @@ struct ContentView: View { if let n = note.object as? Int { settings.vizMode = n } } - // Face + hand skeleton overlay (data_only_viz/pose_bridge.py) - FaceHandOverlay(poseListener: poseListener) - .allowsHitTesting(false) - // HUD coin haut-gauche : mode + touches + pose HUDOverlay(settings: settings, poseListener: poseListener) diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift index 647e8d3..c797842 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift @@ -1,3 +1,4 @@ +import AppKit import AVFoundation import MetalKit import RealityKit @@ -11,7 +12,6 @@ struct BodyView: NSViewRepresentable { @ObservedObject var renderer: MeshRenderer @ObservedObject var settings: RenderSettings @ObservedObject var poseListener: PoseOSCListener - @ObservedObject var skeleton3d: Skeleton3DRenderer func makeNSView(context: Context) -> NSView { let container = NSView(frame: .zero) @@ -89,14 +89,6 @@ struct BodyView: NSViewRepresentable { let bodyAnchor = AnchorEntity(world: .zero) arView.scene.addAnchor(bodyAnchor) - - // Dedicated anchor for the 3D skeleton (mode 9 / openpos). - // Positioned at the origin ; the perspective camera at z=0 with - // default FOV frames a ~3 m-deep stage centered on the hip. - let skelAnchor = AnchorEntity(world: SIMD3(0, 0, -3)) - arView.scene.addAnchor(skelAnchor) - skeleton3d.attach(to: skelAnchor, listener: poseListener) - container.addSubview(arView) // 60 fps mesh interpolation between Multi-HMR frames (Python @@ -108,13 +100,53 @@ struct BodyView: NSViewRepresentable { context.coordinator.cameraEntity = camEntity context.coordinator.sceneRenderer = scene context.coordinator.mtkView = mtkView + context.coordinator.skeletonOverlay = SkeletonOverlay(parent: bodyAnchor) context.coordinator.keyLight = key context.coordinator.fillLight = fill context.coordinator.rimLight = rim context.coordinator.previewLayer = preview context.coordinator.container = container context.coordinator.renderer = renderer - context.coordinator.skelAnchor = skelAnchor + + // Hook clavier global : capture les touches au niveau NSEvent + // pour eviter les beeps systeme quand un .keyboardShortcut SwiftUI + // ne trouve pas de cible. Touches : S / 0-9 / C V M W. + if context.coordinator.kbMonitor == nil { + context.coordinator.kbMonitor = + NSEvent.addLocalMonitorForEvents(matching: .keyDown) { + ev in + guard let chars = ev.charactersIgnoringModifiers else { + return ev + } + let k = chars.lowercased() + switch k { + case "s": + NotificationCenter.default.post( + name: .toggleSettings, object: nil); return nil + case "c": + NotificationCenter.default.post( + name: .toggleLayer, object: "camera"); return nil + case "v": + NotificationCenter.default.post( + name: .toggleLayer, object: "scene"); return nil + case "m": + NotificationCenter.default.post( + name: .toggleLayer, object: "mesh"); return nil + case "w": + NotificationCenter.default.post( + name: .toggleLayer, object: "wireframe"); return nil + case "0", "1", "2", "3", "4", + "5", "6", "7", "8", "9": + if let n = Int(k) { + NotificationCenter.default.post( + name: .setVizMode, object: n) + } + return nil + default: + return ev + } + } + } return container } @@ -125,6 +157,11 @@ struct BodyView: NSViewRepresentable { c.previewLayer?.isHidden = !settings.showCamera c.mtkView?.isHidden = !settings.showScene c.sceneRenderer?.uniforms.viz_mode = Float(settings.vizMode) + // Skeleton overlay openpos : visible si mode openpos (#9) OU + // si toggle showSkeleton actif (option manuel). + let skelVisible = settings.vizMode == 9 || settings.showSkeleton + c.skeletonOverlay?.update(persons: poseListener.persons, + visible: skelVisible) // Pose -> scene uniforms : drive hands3d (mode 8) et openpos // (mode 9) avec la premiere personne detectee. Les wrists pilotent // hand_l/r ; pose_count alimente bg_fragment. @@ -151,9 +188,6 @@ struct BodyView: NSViewRepresentable { c.fillLight?.light.intensity = Float(settings.fillIntensity) c.rimLight?.light.intensity = Float(settings.rimIntensity) - // 3D skeleton only visible in mode 9 (openpos). - c.skelAnchor?.isEnabled = (settings.vizMode == 9) - // Mesh visibility + material guard let anchor = c.bodyAnchor else { return } anchor.children.removeAll() @@ -172,11 +206,18 @@ struct BodyView: NSViewRepresentable { final class Coordinator { var bodyAnchor: AnchorEntity? - var skelAnchor: AnchorEntity? var arView: ARView? var cameraEntity: PerspectiveCamera? var sceneRenderer: SceneRenderer? var mtkView: MTKView? + var skeletonOverlay: SkeletonOverlay? + var kbMonitor: Any? + + deinit { + if let m = kbMonitor { + NSEvent.removeMonitor(m) + } + } var keyLight: DirectionalLight? var fillLight: DirectionalLight? var rimLight: DirectionalLight? diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift index e8ab3a3..bdd8c59 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift @@ -20,45 +20,14 @@ final class PoseOSCListener: ObservableObject { var shoSpan: Float = 0 var torsoYaw: Float = 0 var bodyPitch: Float = 0 - var seenAt: TimeInterval = 0 - } - - /// 68-point dlib-style facial landmarks (x,y normalises 0..1). - /// Slot mapping comes from FACE_68_FROM_MP cote Python. - struct FaceFrame: Equatable { - var points: [SIMD2] = Array(repeating: .zero, count: 68) - var hasPoint: [Bool] = Array(repeating: false, count: 68) - var seenAt: TimeInterval = 0 - } - - /// 21 MediaPipe hand landmarks per detected hand. - struct HandFrame: Equatable { - var side: Int = 0 // 0 = left, 1 = right - var points: [SIMD2] = Array(repeating: .zero, count: 21) - var hasPoint: [Bool] = Array(repeating: false, count: 21) - var seenAt: TimeInterval = 0 - } - - /// MediaPipe pose_world_landmarks : 33 keypoints in meters, relative - /// to the hip-center. Conventions on the wire (MediaPipe): - /// x = right, y = down, z = forward (away from camera). - struct Pose3DFrame: Equatable { - var pid: Int = -1 - // SIMD4 = (x, y, z, confidence). All zeros = slot not yet filled. - var kps: [SIMD4] = Array(repeating: .zero, count: 33) - var hasPoint: [Bool] = Array(repeating: false, count: 33) + /// 33 MediaPipe BODY keypoints flat (x, y, confidence). + /// Empty si pas encore recu /pose/skel. + var skeleton: [SIMD3] = [] var seenAt: TimeInterval = 0 } @Published var persons: [Int: PoseFrame] = [:] - @Published var faces: [Int: FaceFrame] = [:] - @Published var hands: [Int: HandFrame] = [:] - @Published var body3d: [Int: Pose3DFrame] = [:] @Published var count: Int = 0 - @Published var faceCount: Int = 0 - @Published var body3dCount: Int = 0 - @Published var handCountLeft: Int = 0 - @Published var handCountRight: Int = 0 private var listener: NWListener? @@ -161,69 +130,24 @@ final class PoseOSCListener: ObservableObject { var p = persons[Int(pid)] ?? PoseFrame() p.bodyPitch = v persons[Int(pid)] = p - case "/face/count": - if let n = args.first as? Int32 { faceCount = Int(n) } - if faceCount == 0 { faces.removeAll(keepingCapacity: true) } - case "/face/kp": - guard args.count >= 6, - let pid = args[0] as? Int32, - let slot = args[1] as? Int32, - let x = args[2] as? Float, - let y = args[3] as? Float else { return } - let slotI = Int(slot) - guard slotI >= 0 && slotI < 68 else { return } - var f = faces[Int(pid)] ?? FaceFrame() - f.points[slotI] = SIMD2(x, y) - f.hasPoint[slotI] = true - f.seenAt = CFAbsoluteTimeGetCurrent() - faces[Int(pid)] = f - case "/hand/count": - if args.count >= 2, - let l = args[0] as? Int32, let r = args[1] as? Int32 { - handCountLeft = Int(l) - handCountRight = Int(r) - if handCountLeft + handCountRight == 0 { - hands.removeAll(keepingCapacity: true) + case "/pose/skel": + // [pid] + flat 33 * (x, y, conf) + guard args.count >= 1, + let pid = args[0] as? Int32 else { return } + var skel: [SIMD3] = [] + var i = 1 + while i + 2 < args.count { + if let x = args[i] as? Float, + let y = args[i + 1] as? Float, + let c = args[i + 2] as? Float { + skel.append(SIMD3(x, y, c)) } + i += 3 } - case "/pose3d/count": - if let n = args.first as? Int32 { - body3dCount = Int(n) - if body3dCount == 0 { - body3d.removeAll(keepingCapacity: true) - } - } - case "/pose3d/kp": - guard args.count >= 6, - let pid = args[0] as? Int32, - let idx = args[1] as? Int32, - let x = args[2] as? Float, - let y = args[3] as? Float, - let z = args[4] as? Float, - let c = args[5] as? Float else { return } - let i = Int(idx) - guard i >= 0 && i < 33 else { return } - var p = body3d[Int(pid)] ?? Pose3DFrame(pid: Int(pid)) - p.pid = Int(pid) - p.kps[i] = SIMD4(x, y, z, c) - p.hasPoint[i] = true + var p = persons[Int(pid)] ?? PoseFrame() + p.skeleton = skel p.seenAt = CFAbsoluteTimeGetCurrent() - body3d[Int(pid)] = p - case "/hand/kp": - guard args.count >= 7, - let pid = args[0] as? Int32, - let side = args[1] as? Int32, - let idx = args[2] as? Int32, - let x = args[3] as? Float, - let y = args[4] as? Float else { return } - let i = Int(idx) - guard i >= 0 && i < 21 else { return } - var h = hands[Int(pid)] ?? HandFrame() - h.side = Int(side) - h.points[i] = SIMD2(x, y) - h.hasPoint[i] = true - h.seenAt = CFAbsoluteTimeGetCurrent() - hands[Int(pid)] = h + persons[Int(pid)] = p default: break } @@ -231,12 +155,6 @@ final class PoseOSCListener: ObservableObject { let now = CFAbsoluteTimeGetCurrent() persons = persons.filter { $0.value.seenAt == 0 || now - $0.value.seenAt < 2.0 } - faces = faces.filter { $0.value.seenAt == 0 - || now - $0.value.seenAt < 2.0 } - hands = hands.filter { $0.value.seenAt == 0 - || now - $0.value.seenAt < 2.0 } - body3d = body3d.filter { $0.value.seenAt == 0 - || now - $0.value.seenAt < 2.0 } } // MARK: - Minimal OSC parser diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift new file mode 100644 index 0000000..cf984b2 --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift @@ -0,0 +1,156 @@ +import AppKit +import RealityKit +import simd + +/// Rendu skeleton openpos : pour chaque personne detectee, on dessine +/// une sphere a chaque keypoint et un cylindre entre chaque paire de +/// joints connectee. Couleur dependante du pid. +@MainActor +final class SkeletonOverlay { + // MediaPipe BlazePose 33 BODY_LANDMARKS connections (bones) + static let bones: [(Int, Int)] = [ + // Face + (0, 1), (1, 2), (2, 3), (3, 7), + (0, 4), (4, 5), (5, 6), (6, 8), + (9, 10), + // Torso + (11, 12), (11, 23), (12, 24), (23, 24), + // Left arm + (11, 13), (13, 15), (15, 17), (15, 19), (15, 21), (17, 19), + // Right arm + (12, 14), (14, 16), (16, 18), (16, 20), (16, 22), (18, 20), + // Left leg + (23, 25), (25, 27), (27, 29), (27, 31), (29, 31), + // Right leg + (24, 26), (26, 28), (28, 30), (28, 32), (30, 32), + ] + + private let anchor: AnchorEntity + private var personRoots: [Int: Entity] = [:] + private var jointMeshes: [Int: [ModelEntity]] = [:] + private var boneMeshes: [Int: [ModelEntity]] = [:] + + init(parent: AnchorEntity) { + self.anchor = parent + } + + /// Couleur par pid (palette 6 entrees) + private static let palette: [SIMD3] = [ + SIMD3(0.0, 1.0, 0.85), // turquoise + SIMD3(1.0, 0.3, 0.7), // magenta + SIMD3(1.0, 0.9, 0.2), // jaune + SIMD3(1.0, 0.55, 0.1), // ambre + SIMD3(0.7, 0.5, 1.0), // lilas + SIMD3(0.4, 1.0, 0.3), // vert + ] + + /// Met a jour le rendu skeleton pour toutes les personnes du + /// PoseOSCListener. Cree / recycle les entites a la demande. + func update(persons: [Int: PoseOSCListener.PoseFrame], + visible: Bool) { + if !visible { + // Cache tout sans detruire + for root in personRoots.values { root.isEnabled = false } + return + } + let receivedPids = Set(persons.keys) + + // Cleanup personnes disparues + for pid in personRoots.keys where !receivedPids.contains(pid) { + personRoots[pid]?.removeFromParent() + personRoots[pid] = nil + jointMeshes[pid] = nil + boneMeshes[pid] = nil + } + + for (pid, frame) in persons { + let skel = frame.skeleton + guard skel.count >= 33 else { continue } + let color = Self.palette[((pid % 6) + 6) % 6] + + // Cree le root + meshes la premiere fois + if personRoots[pid] == nil { + let root = Entity() + anchor.addChild(root) + personRoots[pid] = root + let nsCol = NSColor(red: CGFloat(color.x), + green: CGFloat(color.y), + blue: CGFloat(color.z), + alpha: 1.0) + let mat = UnlitMaterial(color: nsCol) + let sphereMesh = MeshResource.generateSphere(radius: 0.035) + var joints: [ModelEntity] = [] + for _ in 0..<33 { + let je = ModelEntity( + mesh: sphereMesh, materials: [mat]) + root.addChild(je) + joints.append(je) + } + jointMeshes[pid] = joints + // Bones : on cree un cylindre par bone, mesh partagee + let boneMesh = MeshResource.generateBox( + width: 0.015, height: 1.0, depth: 0.015, + cornerRadius: 0.005) + var bones: [ModelEntity] = [] + for _ in Self.bones { + let be = ModelEntity( + mesh: boneMesh, materials: [mat]) + root.addChild(be) + bones.append(be) + } + boneMeshes[pid] = bones + } + + guard let joints = jointMeshes[pid], + let bones = boneMeshes[pid] else { continue } + personRoots[pid]?.isEnabled = true + + // Update joints : coords image (x 0..1 droite, y 0..1 bas) + // -> RealityKit (x droite, y haut, z negatif vers cam). + // On projete sur un plan a z = -2.5 et on echelle 2x pour + // remplir la fenetre. + let scale: Float = 2.0 + let z: Float = -2.5 + var worldPos: [SIMD3] = [] + worldPos.reserveCapacity(33) + for i in 0..<33 { + let kp = skel[i] + let wx = (kp.x - 0.5) * scale + let wy = -(kp.y - 0.5) * scale // flip y + let pos = SIMD3(wx, wy, z) + worldPos.append(pos) + if i < joints.count { + joints[i].transform.translation = pos + joints[i].isEnabled = kp.z > 0.3 + } + } + + // Update bones : positionne chaque cylindre entre 2 joints + for (idx, (a, b)) in Self.bones.enumerated() { + guard idx < bones.count, + a < worldPos.count, b < worldPos.count else { continue } + let pa = worldPos[a] + let pb = worldPos[b] + let mid = (pa + pb) * 0.5 + let dir = pb - pa + let len = simd_length(dir) + bones[idx].transform.translation = mid + // Orient cylinder Y axis along dir + if len > 1e-4 { + let up = SIMD3(0, 1, 0) + let axis = simd_normalize(dir) + let rot = simd_quatf(from: up, to: axis) + bones[idx].transform.rotation = rot + bones[idx].transform.scale = SIMD3(1, len, 1) + let confA = idx < Self.bones.count + ? skel[a].z : 0 + let confB = idx < Self.bones.count + ? skel[b].z : 0 + bones[idx].isEnabled = min(confA, confB) > 0.3 + } else { + bones[idx].isEnabled = false + } + } + } + } +}