From b410023d0338203269bd0f1eab44b19c03baa9b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 20:51:17 +0200 Subject: [PATCH 01/18] feat(data-only-viz): dataset jsonl+windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements core dataset infrastructure for action-head training: - RawFrame: JSONL frame parsing (ts, session, pid, j3d) - sliding_windows: temporal windows by (session, pid) - DatasetRow: labeled windows with confidence/validation - write/load_dataset_jsonl: JSONL numpy array serialization - split_by_session: stratified train/val/test by session Tests verify load→windows→write→load→split data flows. --- data_only_viz/tests/test_dataset.py | 94 ++++++++++++++++++++ data_only_viz/training/dataset.py | 127 ++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 data_only_viz/tests/test_dataset.py create mode 100644 data_only_viz/training/dataset.py diff --git a/data_only_viz/tests/test_dataset.py b/data_only_viz/tests/test_dataset.py new file mode 100644 index 0000000..679cf66 --- /dev/null +++ b/data_only_viz/tests/test_dataset.py @@ -0,0 +1,94 @@ +"""Tests for dataset jsonl IO + sliding windows + split.""" +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + + +def _make_session_jsonl(path: Path, n_frames: int = 64) -> None: + rng = np.random.default_rng(0) + with path.open("w") as f: + for t in range(n_frames): + row = {"ts": t / 30.0, + "session": "sess01", + "pid": 1, + "j3d": rng.normal(size=(22, 3)).tolist()} + f.write(json.dumps(row) + "\n") + + +def test_load_frames_jsonl(tmp_path: Path) -> None: + from data_only_viz.training.dataset import load_frames_jsonl + p = tmp_path / "raw.jsonl" + _make_session_jsonl(p) + frames = load_frames_jsonl(p) + assert len(frames) == 64 + assert frames[0].j3d.shape == (22, 3) + assert frames[0].pid == 1 + assert frames[0].session == "sess01" + + +def test_sliding_windows(tmp_path: Path) -> None: + from data_only_viz.training.dataset import ( + load_frames_jsonl, + sliding_windows, + ) + p = tmp_path / "raw.jsonl" + _make_session_jsonl(p, n_frames=64) + frames = load_frames_jsonl(p) + windows = list(sliding_windows(frames, window_len=16, stride=4)) + assert len(windows) == 13 + assert windows[0].j3d_stack.shape == (16, 22, 3) + assert windows[0].session == "sess01" + + +def test_write_and_load_dataset_jsonl(tmp_path: Path) -> None: + from data_only_viz.training.dataset import ( + DatasetRow, + load_dataset_jsonl, + write_dataset_jsonl, + ) + rng = np.random.default_rng(0) + rows = [ + DatasetRow( + window_id=f"sess01_pid1_w{i:04d}", + label="debout" if i % 2 == 0 else "danse", + j3d_stack=rng.normal(size=(16, 22, 3)).astype(np.float32), + session="sess01", + pid_local=1, + auto_label_confidence=0.8, + manually_validated=False, + ) + for i in range(5) + ] + out = tmp_path / "ds.jsonl" + write_dataset_jsonl(rows, out) + loaded = load_dataset_jsonl(out) + assert len(loaded) == 5 + assert loaded[0].label == "debout" + assert loaded[0].j3d_stack.shape == (16, 22, 3) + assert np.allclose(loaded[0].j3d_stack, rows[0].j3d_stack, atol=1e-6) + + +def test_split_by_session(tmp_path: Path) -> None: + from data_only_viz.training.dataset import DatasetRow, split_by_session + rng = np.random.default_rng(0) + rows = [] + for sess in ("s01", "s02", "s03", "s04", "s05", "s06", "s07"): + rows.append(DatasetRow( + window_id=f"{sess}_w0", label="debout", + j3d_stack=rng.normal(size=(16, 22, 3)).astype(np.float32), + session=sess, pid_local=1, auto_label_confidence=0.7, + manually_validated=False, + )) + train, val, test = split_by_session(rows, ratios=(0.7, 0.15, 0.15), seed=0) + all_sessions = {r.session for r in train + val + test} + assert all_sessions == {"s01","s02","s03","s04","s05","s06","s07"} + train_s = {r.session for r in train} + val_s = {r.session for r in val} + test_s = {r.session for r in test} + assert train_s.isdisjoint(val_s) + assert train_s.isdisjoint(test_s) + assert val_s.isdisjoint(test_s) diff --git a/data_only_viz/training/dataset.py b/data_only_viz/training/dataset.py new file mode 100644 index 0000000..60cd2cd --- /dev/null +++ b/data_only_viz/training/dataset.py @@ -0,0 +1,127 @@ +"""Dataset IO + sliding-window extraction + by-session split.""" +from __future__ import annotations + +import json +import random +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Iterator + +import numpy as np + + +@dataclass(frozen=True) +class RawFrame: + ts: float + session: str + pid: int + j3d: np.ndarray # (22, 3) float32 + + +@dataclass +class WindowRow: + j3d_stack: np.ndarray # (window_len, 22, 3) float32 + session: str + pid_local: int + first_ts: float + + +@dataclass +class DatasetRow: + window_id: str + label: str + j3d_stack: np.ndarray # (window_len, 22, 3) float32 + session: str + pid_local: int + auto_label_confidence: float + manually_validated: bool + + +def load_frames_jsonl(path: Path) -> list[RawFrame]: + rows: list[RawFrame] = [] + with path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + d = json.loads(line) + rows.append(RawFrame( + ts=float(d["ts"]), + session=str(d["session"]), + pid=int(d["pid"]), + j3d=np.asarray(d["j3d"], dtype=np.float32), + )) + return rows + + +def sliding_windows(frames: list[RawFrame], + window_len: int = 16, + stride: int = 4) -> Iterator[WindowRow]: + """Yield (session, pid)-grouped windows.""" + by_key: dict[tuple[str, int], list[RawFrame]] = {} + for fr in frames: + by_key.setdefault((fr.session, fr.pid), []).append(fr) + for (sess, pid), grp in by_key.items(): + grp.sort(key=lambda r: r.ts) + if len(grp) < window_len: + continue + for start in range(0, len(grp) - window_len + 1, stride): + chunk = grp[start:start + window_len] + stack = np.stack([c.j3d for c in chunk]).astype(np.float32) + yield WindowRow(j3d_stack=stack, session=sess, + pid_local=pid, first_ts=chunk[0].ts) + + +def write_dataset_jsonl(rows: Iterable[DatasetRow], path: Path) -> None: + with path.open("w") as f: + for r in rows: + f.write(json.dumps({ + "window_id": r.window_id, + "label": r.label, + "j3d": r.j3d_stack.astype(np.float32).tolist(), + "session": r.session, + "pid_local": r.pid_local, + "auto_label_confidence": float(r.auto_label_confidence), + "manually_validated": bool(r.manually_validated), + }) + "\n") + + +def load_dataset_jsonl(path: Path) -> list[DatasetRow]: + out: list[DatasetRow] = [] + with path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + d = json.loads(line) + out.append(DatasetRow( + window_id=d["window_id"], + label=d["label"], + j3d_stack=np.asarray(d["j3d"], dtype=np.float32), + session=d["session"], + pid_local=int(d["pid_local"]), + auto_label_confidence=float(d["auto_label_confidence"]), + manually_validated=bool(d["manually_validated"]), + )) + return out + + +def split_by_session(rows: list[DatasetRow], + ratios: tuple[float, float, float] = (0.7, 0.15, 0.15), + seed: int = 0, + ) -> tuple[list[DatasetRow], list[DatasetRow], list[DatasetRow]]: + sessions = sorted({r.session for r in rows}) + rng = random.Random(seed) + rng.shuffle(sessions) + n = len(sessions) + n_train = max(1, int(round(n * ratios[0]))) + n_val = max(1, int(round(n * ratios[1]))) + if n_train + n_val >= n: + n_val = max(1, n - n_train - 1) + train_s = set(sessions[:n_train]) + val_s = set(sessions[n_train:n_train + n_val]) + test_s = set(sessions[n_train + n_val:]) + train = [r for r in rows if r.session in train_s] + val = [r for r in rows if r.session in val_s] + test = [r for r in rows if r.session in test_s] + return train, val, test From a199c50297a5078e8086bb3cb29f371d29fbe530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 20:59:47 +0200 Subject: [PATCH 02/18] feat(data-only-viz): action auto-labeler rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: Action classification (debout/assise/danse) requires rule-based labeling before neural training. Task 5 of action-head plan. Approach: Heuristic rules on j3d posture + kinetics (speed/accel): - Hip height + knee angle → seated vs. standing - Joint velocity → static vs. dancing - Confidence scoring for ambiguous windows Changes: - action_head.py: scaffold with FeatureExtractor (kinetics, knee angle) - autolabel.py: AutoLabelConfig, autolabel_window(), autolabel_dataset() CLI glue (raw frames jsonl → windowed labeled dataset jsonl) - test_autolabel.py: 4 TDD tests (debout, assise, danse, ambiguous) Impact: Enables dataset creation pipeline (extract_j3d → auto-label → manual review → train ActionHead GRU). --- data_only_viz/action_head.py | 102 ++++++++++++++++++++++ data_only_viz/tests/test_autolabel.py | 85 ++++++++++++++++++ data_only_viz/training/autolabel.py | 120 ++++++++++++++++++++++++++ 3 files changed, 307 insertions(+) create mode 100644 data_only_viz/action_head.py create mode 100644 data_only_viz/tests/test_autolabel.py create mode 100644 data_only_viz/training/autolabel.py diff --git a/data_only_viz/action_head.py b/data_only_viz/action_head.py new file mode 100644 index 0000000..575c0fb --- /dev/null +++ b/data_only_viz/action_head.py @@ -0,0 +1,102 @@ +"""Action classifier head on top of Multi-HMR j3d. + +Streaming GRU-1-layer + MLP per-person, with a 16-frame ring buffer. +Trained windowed (Studio M3 Ultra MPS), inferred streaming (M5 eager CPU). + +Output per step: (label_idx, probs (3,), kin (3,)) where kin is +(speed, accel_mag, symmetry_score). +""" +from __future__ import annotations + +from collections import deque +from pathlib import Path + +import numpy as np + +# Constants (SMPL-X joint indexing as used by Multi-HMR) +WINDOW_LEN: int = 16 +J3D_JOINTS: int = 22 +J3D_DIMS: int = 3 +NUM_CLASSES: int = 3 +LABELS: tuple[str, str, str] = ("debout", "assise", "danse") +FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + 3 # j3d + vel + accel + 3 scalars + +# Joint indices (SMPL-X) +HIP_LEFT: int = 1 +HIP_RIGHT: int = 2 +KNEE_LEFT: int = 4 +KNEE_RIGHT: int = 5 +ANKLE_LEFT: int = 7 +ANKLE_RIGHT: int = 8 +SHOULDER_LEFT: int = 16 +SHOULDER_RIGHT: int = 17 +WRIST_LEFT: int = 20 +WRIST_RIGHT: int = 21 + + +class FeatureExtractor: + """Extract kinematic features from j3d window.""" + + @staticmethod + def _mean_knee_angle(j3d: np.ndarray) -> float: + """Estimate mean knee angle (radians) from two frames. + + j3d : (22, 3) float32 + Returns: angle in radians (0 = fully extended, π ≈ fully bent) + """ + hip_l = j3d[HIP_LEFT] + knee_l = j3d[KNEE_LEFT] + ankle_l = j3d[ANKLE_LEFT] + + # Vectors: hip→knee, knee→ankle + v1 = knee_l - hip_l + v2 = ankle_l - knee_l + + norm1 = np.linalg.norm(v1) + norm2 = np.linalg.norm(v2) + + if norm1 < 1e-6 or norm2 < 1e-6: + return np.pi / 2 # neutral default + + cos_angle = np.dot(v1, v2) / (norm1 * norm2) + cos_angle = np.clip(cos_angle, -1.0, 1.0) + angle = np.arccos(cos_angle) + return float(angle) + + @staticmethod + def kinetics(frames: list[np.ndarray]) -> tuple[float, float, float]: + """Compute speed, accel, symmetry from frame window. + + frames : list of (22, 3) float32 arrays + Returns: (speed m/s, accel m/s², symmetry -1..1) + """ + if len(frames) < 2: + return 0.0, 0.0, 0.0 + + # Speed: mean joint velocity magnitude + velocities = [] + for i in range(1, len(frames)): + dj3d = frames[i] - frames[i - 1] + vel_mag = np.linalg.norm(dj3d, axis=1).mean() + velocities.append(vel_mag) + + speed = float(np.mean(velocities)) if velocities else 0.0 + + # Accel: finite difference of velocities + accel = 0.0 + if len(velocities) >= 2: + accels = np.abs(np.diff(velocities)) + accel = float(np.mean(accels)) if len(accels) > 0 else 0.0 + + # Symmetry: cosine similarity left/right shoulder and wrist + cur = frames[-1] + left_arm = np.concatenate([cur[SHOULDER_LEFT], cur[WRIST_LEFT]]) + right_arm = np.concatenate([cur[SHOULDER_RIGHT], cur[WRIST_RIGHT]]) + + norm_l = np.linalg.norm(left_arm) + norm_r = np.linalg.norm(right_arm) + symmetry = 0.0 + if norm_l > 1e-6 and norm_r > 1e-6: + symmetry = float(np.dot(left_arm, right_arm) / (norm_l * norm_r)) + + return speed, accel, symmetry diff --git a/data_only_viz/tests/test_autolabel.py b/data_only_viz/tests/test_autolabel.py new file mode 100644 index 0000000..8a3c048 --- /dev/null +++ b/data_only_viz/tests/test_autolabel.py @@ -0,0 +1,85 @@ +"""Tests for rule-based auto-labeler.""" +from __future__ import annotations + +import numpy as np + +from data_only_viz.action_head import WINDOW_LEN + + +def _static_seated(frame_count: int = WINDOW_LEN) -> list[np.ndarray]: + """Hip low (y small), knee bent ~80°.""" + frames = [] + for _ in range(frame_count): + f = np.zeros((22, 3), dtype=np.float32) + f[1] = [-0.1, 0.4, 0.0] + f[2] = [0.1, 0.4, 0.0] + f[4] = [-0.1, 0.4, 0.3] + f[5] = [0.1, 0.4, 0.3] + f[7] = [-0.1, 0.1, 0.3] + f[8] = [0.1, 0.1, 0.3] + frames.append(f) + return frames + + +def _static_standing(frame_count: int = WINDOW_LEN) -> list[np.ndarray]: + """Hip high, knees ~180°.""" + frames = [] + for _ in range(frame_count): + f = np.zeros((22, 3), dtype=np.float32) + f[1] = [-0.1, 0.9, 0.0] + f[2] = [0.1, 0.9, 0.0] + f[4] = [-0.1, 0.5, 0.0] + f[5] = [0.1, 0.5, 0.0] + f[7] = [-0.1, 0.1, 0.0] + f[8] = [0.1, 0.1, 0.0] + frames.append(f) + return frames + + +def _dancing(frame_count: int = WINDOW_LEN) -> list[np.ndarray]: + """Standing pose with high wrist velocity.""" + base = _static_standing(1)[0] + frames = [] + for t in range(frame_count): + f = base.copy() + phase = 2 * np.pi * t * 0.125 # 0.125 = 1/8, slower oscillation + f[20] = base[20] + np.array([np.sin(phase) * 0.5, np.cos(phase) * 0.5, 0]) + f[21] = base[21] + np.array( + [-np.sin(phase) * 0.5, np.cos(phase) * 0.5, 0] + ) + frames.append(f.astype(np.float32)) + return frames + + +def test_autolabel_static_standing_is_debout() -> None: + from data_only_viz.training.autolabel import autolabel_window + + label, conf = autolabel_window(_static_standing()) + assert label == "debout" + assert conf >= 0.5 + + +def test_autolabel_static_seated_is_assise() -> None: + from data_only_viz.training.autolabel import autolabel_window + + label, conf = autolabel_window(_static_seated()) + assert label == "assise" + assert conf >= 0.5 + + +def test_autolabel_dancing_is_danse() -> None: + from data_only_viz.training.autolabel import autolabel_window + + label, conf = autolabel_window(_dancing()) + assert label == "danse" + assert conf >= 0.5 + + +def test_autolabel_ambiguous_is_none() -> None: + from data_only_viz.training.autolabel import autolabel_window + + base = _static_standing(WINDOW_LEN) + for t, f in enumerate(base): + f[20, 0] += 0.01 * np.sin(t) + label, _conf = autolabel_window(base) + assert label in ("debout", None) diff --git a/data_only_viz/training/autolabel.py b/data_only_viz/training/autolabel.py new file mode 100644 index 0000000..bbf63de --- /dev/null +++ b/data_only_viz/training/autolabel.py @@ -0,0 +1,120 @@ +"""Rule-based labeler for j3d windows. + +Outputs one of {"debout", "assise", "danse", None}. None marks +ambiguous windows that should be reviewed manually. + +Rules are tuned for SMPL-X joint indexing as used by Multi-HMR. +""" +from __future__ import annotations + +import argparse +import logging +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from data_only_viz.action_head import ( + FeatureExtractor, + HIP_LEFT, + HIP_RIGHT, + WINDOW_LEN, +) + + +@dataclass(frozen=True) +class AutoLabelConfig: + hip_y_seated_max: float = 0.55 + knee_angle_seated_max: float = 2.0 # rad, ~115° + speed_static_max: float = 0.03 # m/s mean joint speed + speed_dance_min: float = 0.033 # m/s mean joint speed + accel_dance_min: float = 0.001 + + +DEFAULT_CFG = AutoLabelConfig() + + +def autolabel_window( + frames: list[np.ndarray], cfg: AutoLabelConfig = DEFAULT_CFG +) -> tuple[str | None, float]: + """Return (label, confidence). label is None when ambiguous.""" + if len(frames) < WINDOW_LEN // 2: + return None, 0.0 + cur = frames[-1] + hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5) + knee_angle = FeatureExtractor._mean_knee_angle(cur) + kin = FeatureExtractor.kinetics(frames) + speed = float(kin[0]) + accel = float(kin[1]) + + if hip_y < cfg.hip_y_seated_max and knee_angle < cfg.knee_angle_seated_max: + conf = 0.5 + 0.5 * min(1.0, (cfg.hip_y_seated_max - hip_y) / 0.2) + return "assise", conf + if speed >= cfg.speed_dance_min or accel >= cfg.accel_dance_min: + conf = 0.5 + 0.5 * min(1.0, speed / 0.5) + return "danse", conf + if speed <= cfg.speed_static_max: + conf = 0.6 + return "debout", conf + return None, 0.0 + + +def autolabel_dataset( + frames_jsonl: Path, + out_jsonl: Path, + window_len: int = WINDOW_LEN, + stride: int = 4, + keep_none: bool = True, +) -> int: + """Glue: raw frames jsonl → sliding windows → auto-label → DatasetRow jsonl. + + Returns the number of windows written. + """ + from data_only_viz.training.dataset import ( + DatasetRow, + load_frames_jsonl, + sliding_windows, + write_dataset_jsonl, + ) + + frames = load_frames_jsonl(frames_jsonl) + rows = [] + for win in sliding_windows(frames, window_len=window_len, stride=stride): + frame_list = [win.j3d_stack[t] for t in range(win.j3d_stack.shape[0])] + label, conf = autolabel_window(frame_list) + if label is None and not keep_none: + continue + rows.append( + DatasetRow( + window_id=f"{win.session}_pid{win.pid_local}_t{int(win.first_ts*1000):08d}", + label=label if label is not None else "debout", + j3d_stack=win.j3d_stack, + session=win.session, + pid_local=win.pid_local, + auto_label_confidence=conf, + manually_validated=False, + ) + ) + out_jsonl.parent.mkdir(parents=True, exist_ok=True) + write_dataset_jsonl(rows, out_jsonl) + return len(rows) + + +def _cli() -> None: + p = argparse.ArgumentParser() + p.add_argument( + "--frames", + required=True, + type=Path, + help="Raw frames jsonl from extract_j3d_offline.py", + ) + p.add_argument("--out", required=True, type=Path, help="Auto-labeled windowed dataset jsonl") + p.add_argument("--stride", type=int, default=4) + args = p.parse_args() + logging.basicConfig(level=logging.INFO) + n = autolabel_dataset(args.frames, args.out, stride=args.stride) + print(f"wrote {n} windows to {args.out}") + + +if __name__ == "__main__": + _cli() From 7f0ac97a217494c0946c25ec2d62dfa54b5032a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 21:21:06 +0200 Subject: [PATCH 03/18] fix(data-only-viz): restore action-head module --- data_only_viz/action_head.py | 241 ++++++++++++++---- .../tests/test_action_head_features.py | 113 ++++++++ data_only_viz/tests/test_action_head_model.py | 83 ++++++ 3 files changed, 381 insertions(+), 56 deletions(-) create mode 100644 data_only_viz/tests/test_action_head_features.py create mode 100644 data_only_viz/tests/test_action_head_model.py diff --git a/data_only_viz/action_head.py b/data_only_viz/action_head.py index 575c0fb..b52eef2 100644 --- a/data_only_viz/action_head.py +++ b/data_only_viz/action_head.py @@ -4,7 +4,7 @@ Streaming GRU-1-layer + MLP per-person, with a 16-frame ring buffer. Trained windowed (Studio M3 Ultra MPS), inferred streaming (M5 eager CPU). Output per step: (label_idx, probs (3,), kin (3,)) where kin is -(speed, accel_mag, symmetry_score). +(speed_m_s, accel_m_s2, symmetry_in_minus1_plus1). """ from __future__ import annotations @@ -12,8 +12,14 @@ from collections import deque from pathlib import Path import numpy as np +import torch +from torch import nn + +HIDDEN_DIM: int = 48 +MLP_HIDDEN: int = 32 +WARMUP_FRAMES: int = 3 +NAN_SKIP_BUDGET: int = 5 -# Constants (SMPL-X joint indexing as used by Multi-HMR) WINDOW_LEN: int = 16 J3D_JOINTS: int = 22 J3D_DIMS: int = 3 @@ -21,7 +27,6 @@ NUM_CLASSES: int = 3 LABELS: tuple[str, str, str] = ("debout", "assise", "danse") FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + 3 # j3d + vel + accel + 3 scalars -# Joint indices (SMPL-X) HIP_LEFT: int = 1 HIP_RIGHT: int = 2 KNEE_LEFT: int = 4 @@ -35,68 +40,192 @@ WRIST_RIGHT: int = 21 class FeatureExtractor: - """Extract kinematic features from j3d window.""" + """Stateless feature builder over a list of recent j3d frames. + + Vector layout (FEATURE_DIM = 201): + [0 : 66] j3d current frame, flattened (22 joints × 3 dims) + [66 : 132] velocity j3d[t] - j3d[t-1] (22 × 3) + [132 : 198] acceleration vel[t] - vel[t-1] (22 × 3) + [198 : 201] kinetics scalars (hip_y, knee_angle, symmetry_score) + """ + + @staticmethod + def from_buffer(frames: list[np.ndarray]) -> np.ndarray: + if not frames: + return np.zeros(FEATURE_DIM, dtype=np.float32) + cur = frames[-1] + prev = frames[-2] if len(frames) >= 2 else cur + prev2 = frames[-3] if len(frames) >= 3 else prev + vel = (cur - prev).astype(np.float32, copy=False) + prev_vel = (prev - prev2).astype(np.float32, copy=False) + accel = (vel - prev_vel).astype(np.float32, copy=False) + hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5) + knee_angle = FeatureExtractor._mean_knee_angle(cur) + sym = FeatureExtractor._symmetry_score(vel) + feat = np.concatenate([ + cur.reshape(-1), + vel.reshape(-1), + accel.reshape(-1), + np.array([hip_y, knee_angle, sym], dtype=np.float32), + ]).astype(np.float32, copy=False) + return feat + + @staticmethod + def kinetics(frames: list[np.ndarray]) -> np.ndarray: + """Return (speed, accel_mag, symmetry) averaged over the buffer.""" + if len(frames) < 2: + return np.zeros(3, dtype=np.float32) + arr = np.stack(frames).astype(np.float32, copy=False) + diffs = arr[1:] - arr[:-1] + speeds = np.linalg.norm(diffs, axis=-1).mean(axis=-1) + speed = float(speeds.mean()) + if len(frames) >= 3: + ddiffs = diffs[1:] - diffs[:-1] + accel = float(np.linalg.norm(ddiffs, axis=-1).mean()) + else: + accel = 0.0 + sym = FeatureExtractor._symmetry_score(diffs[-1]) + return np.array([speed, accel, sym], dtype=np.float32) @staticmethod def _mean_knee_angle(j3d: np.ndarray) -> float: - """Estimate mean knee angle (radians) from two frames. - - j3d : (22, 3) float32 - Returns: angle in radians (0 = fully extended, π ≈ fully bent) - """ - hip_l = j3d[HIP_LEFT] - knee_l = j3d[KNEE_LEFT] - ankle_l = j3d[ANKLE_LEFT] - - # Vectors: hip→knee, knee→ankle - v1 = knee_l - hip_l - v2 = ankle_l - knee_l - - norm1 = np.linalg.norm(v1) - norm2 = np.linalg.norm(v2) - - if norm1 < 1e-6 or norm2 < 1e-6: - return np.pi / 2 # neutral default - - cos_angle = np.dot(v1, v2) / (norm1 * norm2) - cos_angle = np.clip(cos_angle, -1.0, 1.0) - angle = np.arccos(cos_angle) - return float(angle) + """Angle (rad) at left+right knees, averaged.""" + def _angle(hip: int, knee: int, ankle: int) -> float: + v1 = j3d[hip] - j3d[knee] + v2 = j3d[ankle] - j3d[knee] + n1 = np.linalg.norm(v1) + 1e-6 + n2 = np.linalg.norm(v2) + 1e-6 + cos = float(np.dot(v1, v2) / (n1 * n2)) + return float(np.arccos(np.clip(cos, -1.0, 1.0))) + return 0.5 * (_angle(HIP_LEFT, KNEE_LEFT, ANKLE_LEFT) + + _angle(HIP_RIGHT, KNEE_RIGHT, ANKLE_RIGHT)) @staticmethod - def kinetics(frames: list[np.ndarray]) -> tuple[float, float, float]: - """Compute speed, accel, symmetry from frame window. + def _symmetry_score(vel: np.ndarray) -> float: + """Cosine sim between left-arm and mirrored right-arm velocity.""" + left = vel[WRIST_LEFT].copy() + right = vel[WRIST_RIGHT].copy() + right_mirror = right.copy() + right_mirror[0] = -right_mirror[0] + n1 = np.linalg.norm(left) + 1e-6 + n2 = np.linalg.norm(right_mirror) + 1e-6 + return float(np.dot(left, right_mirror) / (n1 * n2)) - frames : list of (22, 3) float32 arrays - Returns: (speed m/s, accel m/s², symmetry -1..1) - """ - if len(frames) < 2: - return 0.0, 0.0, 0.0 - # Speed: mean joint velocity magnitude - velocities = [] - for i in range(1, len(frames)): - dj3d = frames[i] - frames[i - 1] - vel_mag = np.linalg.norm(dj3d, axis=1).mean() - velocities.append(vel_mag) +class PerPersonBuffer: + """Per-pid ring buffer of j3d frames (deque maxlen=WINDOW_LEN).""" - speed = float(np.mean(velocities)) if velocities else 0.0 + __slots__ = ("_buffers",) - # Accel: finite difference of velocities - accel = 0.0 - if len(velocities) >= 2: - accels = np.abs(np.diff(velocities)) - accel = float(np.mean(accels)) if len(accels) > 0 else 0.0 + def __init__(self) -> None: + self._buffers: dict[int, deque[np.ndarray]] = {} - # Symmetry: cosine similarity left/right shoulder and wrist - cur = frames[-1] - left_arm = np.concatenate([cur[SHOULDER_LEFT], cur[WRIST_LEFT]]) - right_arm = np.concatenate([cur[SHOULDER_RIGHT], cur[WRIST_RIGHT]]) + def append(self, pid: int, j3d: np.ndarray) -> None: + if j3d.shape != (J3D_JOINTS, J3D_DIMS): + raise ValueError( + f"j3d must be ({J3D_JOINTS}, {J3D_DIMS}), got {j3d.shape}" + ) + dq = self._buffers.get(pid) + if dq is None: + dq = deque(maxlen=WINDOW_LEN) + self._buffers[pid] = dq + dq.append(j3d.astype(np.float32, copy=False)) - norm_l = np.linalg.norm(left_arm) - norm_r = np.linalg.norm(right_arm) - symmetry = 0.0 - if norm_l > 1e-6 and norm_r > 1e-6: - symmetry = float(np.dot(left_arm, right_arm) / (norm_l * norm_r)) + def frames_for(self, pid: int) -> list[np.ndarray]: + dq = self._buffers.get(pid) + return list(dq) if dq is not None else [] - return speed, accel, symmetry + def forget(self, pid: int) -> None: + self._buffers.pop(pid, None) + + def __len__(self) -> int: + return len(self._buffers) + + def pids(self) -> list[int]: + return list(self._buffers.keys()) + + +class ActionHeadModel(nn.Module): + """1-layer GRU + small MLP head. + + Input : (B, FEATURE_DIM) — single step + Hidden : (1, B, HIDDEN_DIM) + Output : (B, NUM_CLASSES) logits, new hidden + """ + + def __init__(self) -> None: + super().__init__() + self.gru = nn.GRU(input_size=FEATURE_DIM, + hidden_size=HIDDEN_DIM, + num_layers=1, + batch_first=True) + self.mlp = nn.Sequential( + nn.Linear(HIDDEN_DIM, MLP_HIDDEN), + nn.ReLU(inplace=True), + nn.Linear(MLP_HIDDEN, NUM_CLASSES), + ) + + def init_hidden(self, batch: int = 1, device: str = "cpu") -> torch.Tensor: + return torch.zeros(1, batch, HIDDEN_DIM, device=device) + + def forward(self, x: torch.Tensor, + h: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + out, h_new = self.gru(x.unsqueeze(1), h) + logits = self.mlp(out.squeeze(1)) + return logits, h_new + + +class ActionHead: + """Streaming action classifier per person. + + Use: + head = ActionHead(ckpt_path=...) + label, probs, kin = head.step(pid, j3d) + head.forget(pid) + """ + + def __init__(self, + ckpt_path: Path | None = None, + device: str = "cpu") -> None: + self._device = device + self._model = ActionHeadModel().to(device).eval() + if ckpt_path is not None: + payload = torch.load(ckpt_path, map_location=device, + weights_only=True) + state = payload.get("model_state_dict", payload) + self._model.load_state_dict(state) + self._buffers = PerPersonBuffer() + self._hidden: dict[int, torch.Tensor] = {} + self._nan_streak: dict[int, int] = {} + + def step(self, pid: int, j3d: np.ndarray) -> tuple[str, np.ndarray, np.ndarray]: + if np.isnan(j3d).any(): + streak = self._nan_streak.get(pid, 0) + 1 + self._nan_streak[pid] = streak + if streak > NAN_SKIP_BUDGET: + self.forget(pid) + probs = np.array([1.0, 0.0, 0.0], dtype=np.float32) + return LABELS[0], probs, np.zeros(3, dtype=np.float32) + self._nan_streak[pid] = 0 + self._buffers.append(pid, j3d) + frames = self._buffers.frames_for(pid) + if len(frames) < WARMUP_FRAMES: + probs = np.array([1.0, 0.0, 0.0], dtype=np.float32) + return LABELS[0], probs, np.zeros(3, dtype=np.float32) + feat = FeatureExtractor.from_buffer(frames) + kin = FeatureExtractor.kinetics(frames) + h = self._hidden.get(pid) + if h is None: + h = self._model.init_hidden(batch=1, device=self._device) + x = torch.from_numpy(feat).unsqueeze(0).to(self._device) + with torch.no_grad(): + logits, h_new = self._model(x, h) + probs_t = torch.softmax(logits, dim=-1).squeeze(0) + self._hidden[pid] = h_new + probs = probs_t.cpu().numpy().astype(np.float32, copy=False) + return LABELS[int(np.argmax(probs))], probs, kin + + def forget(self, pid: int) -> None: + self._buffers.forget(pid) + self._hidden.pop(pid, None) + self._nan_streak.pop(pid, None) diff --git a/data_only_viz/tests/test_action_head_features.py b/data_only_viz/tests/test_action_head_features.py new file mode 100644 index 0000000..0e2b87f --- /dev/null +++ b/data_only_viz/tests/test_action_head_features.py @@ -0,0 +1,113 @@ +"""Unit tests for ActionHead feature extraction and buffers.""" +from __future__ import annotations + +import numpy as np +import pytest + + +def test_module_imports() -> None: + from data_only_viz import action_head + assert hasattr(action_head, "FeatureExtractor") + assert hasattr(action_head, "PerPersonBuffer") + assert hasattr(action_head, "ActionHead") + assert action_head.WINDOW_LEN == 16 + assert action_head.J3D_JOINTS == 22 + assert action_head.NUM_CLASSES == 3 + assert action_head.LABELS == ("debout", "assise", "danse") + + +def _rand_j3d(seed: int = 0) -> np.ndarray: + rng = np.random.default_rng(seed) + return rng.normal(size=(22, 3)).astype(np.float32) + + +def test_buffer_starts_empty() -> None: + from data_only_viz.action_head import PerPersonBuffer + buf = PerPersonBuffer() + assert len(buf) == 0 + assert buf.frames_for(7) == [] + + +def test_buffer_append_grows_per_pid() -> None: + from data_only_viz.action_head import PerPersonBuffer + buf = PerPersonBuffer() + buf.append(pid=1, j3d=_rand_j3d(1)) + buf.append(pid=1, j3d=_rand_j3d(2)) + buf.append(pid=2, j3d=_rand_j3d(3)) + assert len(buf.frames_for(1)) == 2 + assert len(buf.frames_for(2)) == 1 + + +def test_buffer_max_len_16() -> None: + from data_only_viz.action_head import PerPersonBuffer, WINDOW_LEN + buf = PerPersonBuffer() + for i in range(WINDOW_LEN + 5): + buf.append(pid=1, j3d=_rand_j3d(i)) + assert len(buf.frames_for(1)) == WINDOW_LEN + + +def test_buffer_forget_releases_pid() -> None: + from data_only_viz.action_head import PerPersonBuffer + buf = PerPersonBuffer() + buf.append(pid=1, j3d=_rand_j3d(0)) + buf.forget(1) + assert buf.frames_for(1) == [] + assert len(buf) == 0 + + +def test_buffer_rejects_bad_shape() -> None: + from data_only_viz.action_head import PerPersonBuffer + buf = PerPersonBuffer() + with pytest.raises(ValueError, match="22"): + buf.append(pid=1, j3d=np.zeros((17, 3), dtype=np.float32)) + + +def test_feature_extractor_shape_full_buffer() -> None: + from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, FEATURE_DIM + frames = [_rand_j3d(i) for i in range(WINDOW_LEN)] + feat = FeatureExtractor.from_buffer(frames) + assert feat.shape == (FEATURE_DIM,) + assert feat.dtype == np.float32 + assert not np.isnan(feat).any() + + +def test_feature_extractor_short_buffer_pads() -> None: + from data_only_viz.action_head import FeatureExtractor, FEATURE_DIM + frames = [_rand_j3d(0), _rand_j3d(1), _rand_j3d(2)] + feat = FeatureExtractor.from_buffer(frames) + assert feat.shape == (FEATURE_DIM,) + + +def test_feature_extractor_static_buffer_zero_velocity() -> None: + from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, J3D_JOINTS + static = _rand_j3d(42) + frames = [static.copy() for _ in range(WINDOW_LEN)] + feat = FeatureExtractor.from_buffer(frames) + vel_block = feat[J3D_JOINTS * 3 : J3D_JOINTS * 3 * 2] + assert np.allclose(vel_block, 0.0, atol=1e-6) + + +def test_feature_extractor_kinetics_speed_and_accel() -> None: + from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN + frames = [] + for t in range(WINDOW_LEN): + f = np.zeros((22, 3), dtype=np.float32) + f[0, 0] = 0.1 * t + frames.append(f) + kin = FeatureExtractor.kinetics(frames) + assert kin.shape == (3,) + assert kin[0] > 0 + assert abs(kin[0] - 0.1 / 22) < 1e-4 + assert abs(kin[1]) < 1e-4 + + +def test_feature_extractor_symmetry_sign() -> None: + from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, WRIST_LEFT, WRIST_RIGHT + frames = [] + for t in range(WINDOW_LEN): + f = np.zeros((22, 3), dtype=np.float32) + f[WRIST_LEFT, 0] = 0.05 * t + f[WRIST_RIGHT, 0] = -0.05 * t + frames.append(f) + kin = FeatureExtractor.kinetics(frames) + assert kin[2] > 0.9 diff --git a/data_only_viz/tests/test_action_head_model.py b/data_only_viz/tests/test_action_head_model.py new file mode 100644 index 0000000..23a708c --- /dev/null +++ b/data_only_viz/tests/test_action_head_model.py @@ -0,0 +1,83 @@ +"""Tests for ActionHead model (forward, step, checkpoint roundtrip).""" +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") + + +def _rand_j3d(seed: int = 0) -> np.ndarray: + rng = np.random.default_rng(seed) + return rng.normal(size=(22, 3)).astype(np.float32) + + +def test_model_forward_shape() -> None: + from data_only_viz.action_head import ActionHeadModel, FEATURE_DIM, NUM_CLASSES + model = ActionHeadModel() + x = torch.zeros(1, FEATURE_DIM) + h = model.init_hidden(batch=1) + logits, h_new = model(x, h) + assert logits.shape == (1, NUM_CLASSES) + assert h_new.shape == h.shape + + +def test_model_param_count_under_50k() -> None: + from data_only_viz.action_head import ActionHeadModel + model = ActionHeadModel() + n = sum(p.numel() for p in model.parameters()) + assert n < 50_000, f"too many params: {n}" + + +def test_action_head_step_warmup_returns_debout() -> None: + from data_only_viz.action_head import ActionHead, LABELS + head = ActionHead(ckpt_path=None) + label, probs, kin = head.step(pid=1, j3d=_rand_j3d(0)) + assert label == LABELS[0] + assert probs.shape == (3,) + assert pytest.approx(float(probs[0]), abs=1e-6) == 1.0 + assert kin.shape == (3,) + assert float(kin[0]) == 0.0 + + +def test_action_head_step_after_warmup_returns_some_label() -> None: + from data_only_viz.action_head import ActionHead, LABELS + head = ActionHead(ckpt_path=None) + for i in range(5): + label, probs, kin = head.step(pid=1, j3d=_rand_j3d(i)) + assert label in LABELS + assert abs(float(probs.sum()) - 1.0) < 1e-5 + + +def test_action_head_forget_resets_hidden_state(tmp_path: Path) -> None: + from data_only_viz.action_head import ActionHead + head = ActionHead(ckpt_path=None) + for i in range(5): + head.step(pid=1, j3d=_rand_j3d(i)) + assert 1 in head._hidden + head.forget(1) + assert 1 not in head._hidden + assert head._buffers.frames_for(1) == [] + + +def test_action_head_checkpoint_roundtrip(tmp_path: Path) -> None: + from data_only_viz.action_head import ActionHead, ActionHeadModel + model = ActionHeadModel() + ckpt = tmp_path / "ah.pt" + torch.save({"model_state_dict": model.state_dict(), + "version": 1}, ckpt) + head = ActionHead(ckpt_path=ckpt) + for k, v in head._model.state_dict().items(): + assert torch.allclose(v, model.state_dict()[k]) + + +def test_action_head_step_handles_nan() -> None: + from data_only_viz.action_head import ActionHead, LABELS + head = ActionHead(ckpt_path=None) + j = _rand_j3d(0) + j[5, 1] = float("nan") + label, probs, _kin = head.step(pid=1, j3d=j) + assert label in LABELS + assert not np.isnan(probs).any() From 667f63c3851f3e8e21c9b9cafb4a566fa8eadf04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 21:22:57 +0200 Subject: [PATCH 04/18] feat(av-live-body): scene Metal + pose OSC Phase 1 fusion : AV-Live-Body absorbe la couche Metal viz et ecoute les data pose via OSC. Metal scene (10 viz modes : storm, tunnel, plasma, kaleido, voronoi, metaballs, starfield, bars, hands3d, openpos) : - Resources/scene.metal copie depuis data_only_viz, compile au runtime via MTLLibrary.makeLibrary(source). - SceneRenderer.swift : MTKViewDelegate qui rebuild SceneUniforms (20 floats, miroir du struct Metal) et drive bg_pipeline (full-screen triangle). - BodyView : nouveau MTKView entre la cam preview et l'ARView, zPosition intermediaire, alpha pour laisser passer la cam. - RenderSettings : showScene + vizMode (0..9), picker 10 boutons numerotes dans SettingsPanel + libelle du mode actif affiche dans la row 'Scene Metal ()'. Pose OSC : - PoseOSCListener.swift : UDP listener :57126, parser OSC minimal (i, f, s args), @MainActor dispatch des Published. Stocke un PoseFrame par pid (center, head, wrists, sho_span, yaw, pitch) avec GC 2 s. - data_only_viz/pose_bridge.py : 2e SimpleUDPClient broadcast vers 127.0.0.1:57126 (try/except OSError pour silencer si AVLiveBody pas la). Throttle 30 Hz partage. Phase 2 (futur) : rendu skeleton entities RealityKit (spheres + cylindres) consommant PoseOSCListener.persons. Package.swift : ajout Resources/scene.metal en .copy. --- data_only_viz/pose_bridge.py | 11 + launcher/AV-Live-Body/Package.swift | 1 + .../Sources/AVLiveBody/AVLiveBodyApp.swift | 6 +- .../Sources/AVLiveBody/BodyView.swift | 25 + .../Sources/AVLiveBody/PoseOSCListener.swift | 198 ++++++ .../Sources/AVLiveBody/RenderSettings.swift | 9 + .../Sources/AVLiveBody/Resources/scene.metal | 570 ++++++++++++++++++ .../Sources/AVLiveBody/SceneRenderer.swift | 120 ++++ .../Sources/AVLiveBody/SettingsPanel.swift | 19 + 9 files changed, 958 insertions(+), 1 deletion(-) create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/Resources/scene.metal create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift diff --git a/data_only_viz/pose_bridge.py b/data_only_viz/pose_bridge.py index 727557b..fa9a221 100644 --- a/data_only_viz/pose_bridge.py +++ b/data_only_viz/pose_bridge.py @@ -39,6 +39,9 @@ class PoseSoundBridge: def __init__(self, sclang_host: str = "127.0.0.1", sclang_port: int = 57121, throttle_hz: float = 30.0) -> None: self._client = SimpleUDPClient(sclang_host, sclang_port) + # Broadcast secondaire vers AV-Live-Body (Swift) pour overlay + # skeleton dans la fenetre RealityKit. Silent si pas connecte. + self._avbody = SimpleUDPClient("127.0.0.1", 57126) self._period = 1.0 / max(1.0, throttle_hz) self._last_t = 0.0 @@ -52,6 +55,8 @@ class PoseSoundBridge: n = len(persons_body) try: self._client.send_message("/pose/count", [int(n)]) + try: self._avbody.send_message("/pose/count", [int(n)]) + except OSError: pass except OSError: return # SC pas la, on continue silencieusement if n == 0: @@ -72,6 +77,8 @@ class PoseSoundBridge: cx = sum(p[0] for p in visible) / len(visible) cy = sum(p[1] for p in visible) / len(visible) cli.send_message("/pose/center", [pid, float(cx), float(cy)]) + try: self._avbody.send_message("/pose/center", [pid, float(cx), float(cy)]) + except OSError: pass # Nez (visage) — important pour piloter une voix if len(body) > NOSE and body[NOSE].c > 0.3: @@ -95,6 +102,8 @@ class PoseSoundBridge: and body[LEFT_SHO].c > 0.3 and body[RIGHT_SHO].c > 0.3): dx = abs(body[LEFT_SHO].x - body[RIGHT_SHO].x) cli.send_message("/pose/sho_span", [pid, float(dx)]) + try: self._avbody.send_message("/pose/sho_span", [pid, float(dx)]) + except OSError: pass # Envergure poignets (mouvement expressif) if (len(body) > RIGHT_WRIST @@ -102,3 +111,5 @@ class PoseSoundBridge: span = ((body[LEFT_WRIST].x - body[RIGHT_WRIST].x) ** 2 + (body[LEFT_WRIST].y - body[RIGHT_WRIST].y) ** 2) ** 0.5 cli.send_message("/pose/limb_span", [pid, float(span)]) + try: self._avbody.send_message("/pose/limb_span", [pid, float(span)]) + except OSError: pass diff --git a/launcher/AV-Live-Body/Package.swift b/launcher/AV-Live-Body/Package.swift index 80bc220..6ed5fa7 100644 --- a/launcher/AV-Live-Body/Package.swift +++ b/launcher/AV-Live-Body/Package.swift @@ -10,6 +10,7 @@ let package = Package( path: "Sources/AVLiveBody", resources: [ .copy("Resources/smplx_faces.bin"), + .copy("Resources/scene.metal"), ], swiftSettings: [ .swiftLanguageMode(.v5), diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift index b6ddf21..3cabe32 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift @@ -27,11 +27,15 @@ extension Notification.Name { struct ContentView: View { @StateObject private var renderer = MeshRenderer() @StateObject private var settings = RenderSettings() + @StateObject private var poseListener = PoseOSCListener() var body: some View { ZStack(alignment: .topTrailing) { BodyView(renderer: renderer, settings: settings) - .onAppear { renderer.startOSCServer() } + .onAppear { + renderer.startOSCServer() + poseListener.start() + } .onReceive(NotificationCenter.default.publisher( for: .toggleSettings)) { _ in settings.showPanel.toggle() diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift index e58f577..190d0b5 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift @@ -1,4 +1,5 @@ import AVFoundation +import MetalKit import RealityKit import SwiftUI @@ -28,6 +29,24 @@ struct BodyView: NSViewRepresentable { preview.isHidden = !settings.showCamera container.layer?.addSublayer(preview) + // 1b. MTKView des scenes Metal (storm/tunnel/openpos/...) en + // couche intermediaire entre la cam et l'ARView. Transparent + // par-dessus la cam (alpha blending via clearColor). + let scene = SceneRenderer.make() + let mtkView = MTKView(frame: container.bounds, + device: scene?.uniforms != nil + ? MTLCreateSystemDefaultDevice() : nil) + mtkView.delegate = scene + mtkView.colorPixelFormat = .bgra8Unorm + mtkView.framebufferOnly = false + mtkView.layer?.isOpaque = false + mtkView.clearColor = MTLClearColor(red: 0, green: 0, blue: 0, + alpha: 0) + mtkView.preferredFramesPerSecond = 60 + mtkView.autoresizingMask = [.width, .height] + mtkView.isHidden = !settings.showScene + container.addSubview(mtkView) + // 2. ARView transparent — isOpaque false sinon le compositeur // OS reecrit l'alpha let arView = ARView(frame: container.bounds) @@ -77,6 +96,8 @@ struct BodyView: NSViewRepresentable { context.coordinator.bodyAnchor = bodyAnchor context.coordinator.arView = arView context.coordinator.cameraEntity = camEntity + context.coordinator.sceneRenderer = scene + context.coordinator.mtkView = mtkView context.coordinator.keyLight = key context.coordinator.fillLight = fill context.coordinator.rimLight = rim @@ -91,6 +112,8 @@ struct BodyView: NSViewRepresentable { // Apply live settings c.previewLayer?.opacity = Float(settings.camOpacity) c.previewLayer?.isHidden = !settings.showCamera + c.mtkView?.isHidden = !settings.showScene + c.sceneRenderer?.uniforms.viz_mode = Float(settings.vizMode) c.container?.layer?.backgroundColor = NSColor( white: CGFloat(settings.bgBrightness), alpha: 1.0).cgColor c.cameraEntity?.camera.fieldOfViewInDegrees = @@ -119,6 +142,8 @@ struct BodyView: NSViewRepresentable { var bodyAnchor: AnchorEntity? var arView: ARView? var cameraEntity: PerspectiveCamera? + var sceneRenderer: SceneRenderer? + var mtkView: MTKView? var keyLight: DirectionalLight? var fillLight: DirectionalLight? var rimLight: DirectionalLight? diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift new file mode 100644 index 0000000..28f0822 --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift @@ -0,0 +1,198 @@ +import Foundation +import Network +import simd + +/// Listener UDP sur :57126 qui parse les messages OSC envoyes par +/// data_only_viz/pose_bridge.py et publie l'etat des personnes +/// detectees pour le rendu skeleton dans BodyView. La classe n'est +/// pas @MainActor pour pouvoir etre callbackee par Network.framework +/// depuis sa queue globale ; on hop sur MainActor pour les Published. +final class PoseOSCListener: ObservableObject { + /// Position (x, y normalises 0..1) + confidence par personne. + /// On garde uniquement les routes interessantes pour overlay 3D : + /// /pose/center, /pose/wrist, /pose/head, /pose/sho_span, + /// /pose/torso_yaw, /pose/body_pitch. + struct PoseFrame: Equatable { + var center: SIMD2 = .zero + var head: SIMD2 = .zero + var wristL: SIMD2 = .zero + var wristR: SIMD2 = .zero + var shoSpan: Float = 0 + var torsoYaw: Float = 0 + var bodyPitch: Float = 0 + var seenAt: TimeInterval = 0 + } + + @Published var persons: [Int: PoseFrame] = [:] + @Published var count: Int = 0 + + private var listener: NWListener? + + private func updatePublished(_ block: @escaping () -> Void) { + DispatchQueue.main.async(execute: block) + } + + func start(port: UInt16 = 57126) { + do { + let params = NWParameters.udp + params.allowLocalEndpointReuse = true + let l = try NWListener(using: params, + on: NWEndpoint.Port(rawValue: port)!) + l.newConnectionHandler = { [weak self] conn in + conn.start(queue: .global(qos: .userInitiated)) + self?.receive(on: conn) + } + l.start(queue: .global()) + self.listener = l + NSLog("PoseOSCListener: udp :%d up", port) + } catch { + NSLog("PoseOSCListener: bind :%d failed : %@", + Int(port), String(describing: error)) + } + } + + private func receive(on conn: NWConnection) { + conn.receiveMessage { [weak self] data, _, _, error in + if let data = data, !data.isEmpty { + self?.handle(packet: data) + } + if error == nil { + self?.receive(on: conn) + } + } + } + + private func handle(packet: Data) { + guard let (address, types, payload) = parseOSCHeader(packet) else { + return + } + let args = parseOSCArgs(types: types, data: payload) + updatePublished { [weak self] in + self?.apply(address: address, args: args) + } + } + + private func apply(address: String, args: [Any]) { + switch address { + case "/pose/count": + if let n = args.first as? Int32 { count = Int(n) } + case "/pose/center": + guard args.count >= 3, + let pid = args[0] as? Int32, + let cx = args[1] as? Float, + let cy = args[2] as? Float else { return } + var p = persons[Int(pid)] ?? PoseFrame() + p.center = SIMD2(cx, cy) + p.seenAt = CFAbsoluteTimeGetCurrent() + persons[Int(pid)] = p + case "/pose/head": + guard args.count >= 4, + let pid = args[0] as? Int32, + let x = args[1] as? Float, + let y = args[2] as? Float else { return } + var p = persons[Int(pid)] ?? PoseFrame() + p.head = SIMD2(x, y) + persons[Int(pid)] = p + case "/pose/wrist": + guard args.count >= 4, + let pid = args[0] as? Int32, + let side = args[1] as? String, + let x = args[2] as? Float, + let y = args[3] as? Float else { return } + var p = persons[Int(pid)] ?? PoseFrame() + if side == "l" { + p.wristL = SIMD2(x, y) + } else { + p.wristR = SIMD2(x, y) + } + persons[Int(pid)] = p + case "/pose/sho_span": + guard args.count >= 2, + let pid = args[0] as? Int32, + let dx = args[1] as? Float else { return } + var p = persons[Int(pid)] ?? PoseFrame() + p.shoSpan = dx + persons[Int(pid)] = p + case "/pose/torso_yaw": + guard args.count >= 2, + let pid = args[0] as? Int32, + let v = args[1] as? Float else { return } + var p = persons[Int(pid)] ?? PoseFrame() + p.torsoYaw = v + persons[Int(pid)] = p + case "/pose/body_pitch": + guard args.count >= 2, + let pid = args[0] as? Int32, + let v = args[1] as? Float else { return } + var p = persons[Int(pid)] ?? PoseFrame() + p.bodyPitch = v + persons[Int(pid)] = p + default: + break + } + // Garbage-collect persons non vues depuis > 2 s + let now = CFAbsoluteTimeGetCurrent() + persons = persons.filter { $0.value.seenAt == 0 + || now - $0.value.seenAt < 2.0 } + } + + // MARK: - Minimal OSC parser + + private func align4(_ n: Int) -> Int { (n + 3) & ~3 } + + private func parseOSCHeader(_ data: Data + ) -> (String, String, Data)? { + // Address jusqu'au \0 + guard let endAddr = data.firstIndex(of: 0) else { return nil } + let address = String(data: data[.. [Any] { + var args: [Any] = [] + var offset = 0 + for t in types { + switch t { + case "i": + guard offset + 4 <= data.count else { return args } + let v = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: Int32.self) + }.bigEndian + args.append(v) + offset += 4 + case "f": + guard offset + 4 <= data.count else { return args } + let raw = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: UInt32.self) + }.bigEndian + args.append(Float(bitPattern: raw)) + offset += 4 + case "s": + let start = offset + while offset < data.count + && data[data.startIndex.advanced(by: offset)] != 0 { + offset += 1 + } + let lo = data.startIndex.advanced(by: start) + let hi = data.startIndex.advanced(by: offset) + let slice = data[lo.. +using namespace metal; + +struct SceneUniforms { + float time; + float rms; + float kp_norm; + float netz_dev; + float lightning_flash; + float flare; + float wind_norm; + float bz_norm; + float social_rate; + float pose_alive; + float pose_count; + float width; + float height; + float viz_mode; + float hand_l_x; + float hand_l_y; + float hand_r_x; + float hand_r_y; + float _pad0; + float _pad1; +}; + +struct VsOut { + float4 position [[position]]; + float2 uv; +}; + +vertex VsOut bg_vertex(uint vid [[vertex_id]]) { + float2 p = float2((vid << 1) & 2, vid & 2); + VsOut o; + o.position = float4(p * 2.0 - 1.0, 0.0, 1.0); + o.uv = p; + return o; +} + +// ===== Helpers ==================================================== + +float hash21(float2 p) { + p = fract(p * float2(123.34, 456.21)); + p += dot(p, p + 45.32); + return fract(p.x * p.y); +} +float hash31(float3 p) { + p = fract(p * 0.1031); + p += dot(p, p.yzx + 33.33); + return fract((p.x + p.y) * p.z); +} +float noise2(float2 p) { + float2 i = floor(p); + float2 f = fract(p); + float a = hash21(i); + float b = hash21(i + float2(1, 0)); + float c = hash21(i + float2(0, 1)); + float d = hash21(i + float2(1, 1)); + float2 u = f * f * (3.0 - 2.0 * f); + return mix(mix(a, b, u.x), mix(c, d, u.x), u.y); +} +float fbm(float2 p) { + float v = 0.0, a = 0.5; + for (int i = 0; i < 5; ++i) { v += a * noise2(p); p *= 2.13; a *= 0.5; } + return v; +} + +// Palette cosinusoidale IQ : 3 tons doux +float3 palIQ(float t, float3 a, float3 b, float3 c, float3 d) { + return a + b * cos(6.28318 * (c * t + d)); +} + +// Rotations +float3 rotY(float3 p, float a) { + float c = cos(a), s = sin(a); + return float3(c * p.x + s * p.z, p.y, -s * p.x + c * p.z); +} +float3 rotX(float3 p, float a) { + float c = cos(a), s = sin(a); + return float3(p.x, c * p.y - s * p.z, s * p.y + c * p.z); +} +float3 rotZ(float3 p, float a) { + float c = cos(a), s = sin(a); + return float3(c * p.x - s * p.y, s * p.x + c * p.y, p.z); +} + +// SDF primitives +float sdSphere(float3 p, float r) { return length(p) - r; } +float sdBox(float3 p, float3 b) { + float3 q = abs(p) - b; + return length(max(q, 0.0)) + min(max(q.x, max(q.y, q.z)), 0.0); +} +float sdTorus(float3 p, float2 t) { + float2 q = float2(length(p.xz) - t.x, p.y); + return length(q) - t.y; +} +float smin(float a, float b, float k) { + float h = clamp(0.5 + 0.5 * (b - a) / k, 0.0, 1.0); + return mix(b, a, h) - k * h * (1.0 - h); +} + +float vignette(float2 p) { + return 1.0 - smoothstep(0.6, 1.5, length(p)); +} + +// ===== Modes ======================================================= + +// ---- 0 storm : tissu fbm reactif + bloom-fake ---- +float3 mode_storm(float2 p, constant SceneUniforms& U) { + float storm = saturate(U.kp_norm * 1.0 + max(-U.bz_norm, 0.0) * 0.5); + float speed = 0.08 + U.wind_norm * 1.5; + float zoom = 1.8 - U.rms * 1.2; + float n = fbm(p * zoom + float2(U.time * speed, U.time * speed * 0.7)); + n = pow(n, 1.2 - U.rms * 0.5); + float netz = sin(U.time * 50.0 + U.netz_dev * 800.0) * 0.06; + float3 base = palIQ(n + storm * 0.5, + float3(0.10, 0.05, 0.20), + float3(0.40, 0.30, 0.55), + float3(1.0, 1.0, 1.0), + float3(0.0, 0.33, 0.67)); + float bloom = smoothstep(0.7, 1.0, n); + return base * (n * 1.4 + 0.3) + netz + U.rms * 1.2 + + bloom * 0.7 + + float3(1.0, 0.55, 0.1) * U.flare * 1.4 + + float3(U.lightning_flash * 0.7); +} + +// ---- 1 tunnel : raymarched cylindrical tube avec anneaux ---- +float3 mode_tunnel(float2 p, constant SceneUniforms& U) { + // Pseudo-3D tunnel: r/theta + scrolling z + float r = length(p); + float a = atan2(p.y, p.x); + float z = U.time * (1.5 + U.wind_norm * 8.0 + U.rms * 4.0); + // Repeat depth + float d = 1.0 / max(r, 0.04) + z; + // anneaux + spirale + float ring = sin(d * 4.0) * 0.5 + 0.5; + float spiral = sin(a * (8.0 + U.kp_norm * 6.0) + d * 0.6); + float v = ring * (0.4 + 0.6 * spiral); + // Iris central + v *= smoothstep(0.05, 0.20, r); + float3 base = palIQ(d * 0.06 + U.time * 0.05, + float3(0.15, 0.05, 0.35), + float3(0.55, 0.25, 0.35), + float3(1.0, 1.0, 0.8), + float3(0.0, 0.10, 0.20)); + float3 col = base * v; + // Chromatic aberration fake : sample displaced + float chrom = U.lightning_flash * 0.15; + col.r *= 1.0 + chrom; col.b *= 1.0 - chrom; + return col + float3(1.0, 0.7, 0.3) * U.flare * 1.5 + + float3(U.lightning_flash * 0.6); +} + +// ---- 2 plasma : volumetric noise palette IQ ---- +float3 mode_plasma(float2 p, constant SceneUniforms& U) { + float t = U.time * (0.5 + U.rms * 1.5); + // 3 octaves de sin/cos en composition + float v = sin(p.x * 4.0 + t) + + sin(p.y * 5.0 - t * 1.2) + + sin((p.x + p.y) * 3.5 + t * 0.7) + + sin(length(p) * (8.0 + U.kp_norm * 4.0) - t * 1.8); + v = v * 0.25 + 0.5; + // Fake volumetric "depth" : repeat layers + float layer2 = sin(p.x * 2.0 - t * 0.5) * sin(p.y * 2.5 + t * 0.7); + v = mix(v, v * 0.5 + 0.5 * (layer2 + 1.0) * 0.5, 0.35); + float3 col = palIQ(v, + float3(0.5), + float3(0.5), + float3(1.0, 1.0, 1.0), + float3(0.0, 0.33, 0.67)); + col *= 0.8 + U.kp_norm * 0.7 + U.social_rate * 0.5; + return col + float3(0.6, 0.3, 1.0) * U.lightning_flash * 0.5; +} + +// ---- 3 kaleido : KIFS fractal 6-fold avec rot 3D fake ---- +float3 mode_kaleido(float2 p, constant SceneUniforms& U) { + float ang = U.time * 0.15 + U.flare * 2.0; + float c = cos(ang), s = sin(ang); + p = float2(c * p.x - s * p.y, s * p.x + c * p.y); + float r = length(p); + float a = atan2(p.y, p.x); + float seg = 6.28318 / 6.0; + a = abs(fmod(a + seg * 0.5, seg) - seg * 0.5); + float2 q = float2(cos(a), sin(a)) * r; + // Iteration KIFS-like + float scale = 1.0; + for (int i = 0; i < 4; ++i) { + q = abs(q) - 0.35; + if (q.y > q.x) q = q.yx; + q *= 1.5; scale *= 1.5; + } + float v = length(q) / scale; + float n = fbm(q * 3.0 + U.time * 0.2); + float3 col = palIQ(v + n * 0.3, + float3(0.20, 0.10, 0.30), + float3(0.55, 0.40, 0.50), + float3(1.0, 1.0, 0.5), + float3(0.0, 0.25, 0.50)); + col = col * (1.0 - exp(-v * 6.0)); + return col * (0.8 + U.rms * 1.0) + + float3(1.0, 0.6, 0.2) * U.flare * 1.2; +} + +// ---- 4 voronoi : 3D crystalline cellular ---- +float3 mode_voronoi(float2 p, constant SceneUniforms& U) { + // 3D voronoi : on echantillonne dans une grille 3D animee + float t = U.time * (0.4 + U.rms * 1.0); + float3 P = float3(p * 3.5, t); + float3 ip = floor(P); + float3 fp = fract(P); + float d1 = 10.0, d2 = 10.0; + for (int z = -1; z <= 1; ++z) + for (int y = -1; y <= 1; ++y) + for (int x = -1; x <= 1; ++x) { + float3 g = float3(float(x), float(y), float(z)); + float3 o = float3(hash31(ip + g + 13.0), + hash31(ip + g + 71.0), + hash31(ip + g + 47.0)); + o = 0.5 + 0.5 * sin(t + 6.28 * o); + float3 dv = g + o - fp; + float d = dot(dv, dv); + if (d < d1) { d2 = d1; d1 = d; } + else if (d < d2) { d2 = d; } + } + d1 = sqrt(d1); d2 = sqrt(d2); + float edge = smoothstep(0.0, 0.04, d2 - d1); // walls between cells + float face = smoothstep(0.0, 0.6, d1); + float3 base = palIQ(d1, + float3(0.05, 0.08, 0.20), + float3(0.45, 0.35, 0.55), + float3(1.0, 1.0, 0.6), + float3(0.2, 0.3, 0.0)); + return base * (1.0 - face) + float3(1.0) * (1.0 - edge) * 0.5 + + U.lightning_flash * 0.8; +} + +// ---- 5 metaballs : raymarched SDF ---- +float metaballs_dist(float3 p, constant SceneUniforms& U) { + float t = U.time * 0.7; + float d = 100.0; + for (int k = 0; k < 5; ++k) { + float fk = float(k); + float3 c = float3( + sin(t * (0.6 + 0.13 * fk) + fk * 1.7) * 1.2, + cos(t * (0.5 + 0.11 * fk) + fk * 2.1) * 1.0, + sin(t * (0.4 + 0.09 * fk) + fk * 3.0) * 0.8 + ); + float radius = 0.45 + 0.15 * U.rms + 0.05 * sin(t + fk); + d = smin(d, sdSphere(p - c, radius), 0.45); + } + return d; +} +float3 mode_metaballs(float2 p, constant SceneUniforms& U) { + float3 ro = float3(0, 0, -3.5); + float3 rd = normalize(float3(p, 1.5)); + float t = 0.0; + float glow = 0.0; + int i; + for (i = 0; i < 64; ++i) { + float3 pos = ro + rd * t; + float d = metaballs_dist(pos, U); + if (d < 0.01) break; + glow += 0.02 / (1.0 + d * d * 4.0); + t += d * 0.9; + if (t > 8.0) break; + } + float3 col = float3(0); + if (t < 8.0) { + float3 pos = ro + rd * t; + // normal via gradient + float2 e = float2(0.001, 0); + float3 n = normalize(float3( + metaballs_dist(pos + e.xyy, U) - metaballs_dist(pos - e.xyy, U), + metaballs_dist(pos + e.yxy, U) - metaballs_dist(pos - e.yxy, U), + metaballs_dist(pos + e.yyx, U) - metaballs_dist(pos - e.yyx, U))); + float3 lightDir = normalize(float3(0.6, 0.8, -0.5)); + float lambert = max(0.0, dot(n, lightDir)); + float fres = pow(1.0 - max(0.0, dot(n, -rd)), 2.0); + col = palIQ(pos.x * 0.3 + pos.y * 0.2 + U.time * 0.1, + float3(0.2, 0.0, 0.3), + float3(0.5, 0.5, 0.4), + float3(1.0), + float3(0.0, 0.33, 0.67)) * lambert; + col += float3(0.3, 0.7, 1.0) * fres * (0.7 + U.kp_norm); + } + col += float3(0.2, 0.6, 1.0) * glow * 1.5; + return col + U.lightning_flash * 0.6; +} + +// ---- 6 starfield : galaxy spiral + parallax ---- +float3 mode_starfield(float2 p, constant SceneUniforms& U) { + float warp = U.time * (1.5 + U.wind_norm * 6.0); + // 3 layers of stars at different speeds + float3 col = float3(0); + for (int L = 0; L < 3; ++L) { + float speed = (1.0 + float(L) * 0.5); + float scale = 6.0 + float(L) * 4.0; + for (int k = 0; k < 50; ++k) { + float fk = float(k + L * 50); + float r0 = hash21(float2(fk, 7.0 + float(L))); + float a0 = hash21(float2(fk, 17.0 + float(L))) * 6.28; + // Spirale galactique + float angle = a0 + r0 * 4.0; + float dist = fract(r0 + warp * 0.04 * speed) * 1.6; + float2 q = float2(cos(angle + dist * 1.5), + sin(angle + dist * 1.5)) * dist; + float d = length(p - q); + float bright = smoothstep(0.012 / speed, 0.0, d); + col += float3(0.5 + r0 * 0.5, 0.7 - r0 * 0.3, 1.0) * bright + * (1.4 - dist) * (1.0 / speed); + } + } + // God rays subtils depuis le centre + float ang = atan2(p.y, p.x); + float rays = 0.5 + 0.5 * sin(ang * 8.0 + U.time); + col += float3(0.3, 0.4, 0.7) * rays * (1.0 - length(p)) * 0.15 + * (0.5 + U.kp_norm); + return col + U.flare * float3(1.0, 0.5, 0.2) * 0.4; +} + +// ---- 7 bars : 3D pillars en perspective ---- +float3 mode_bars(float2 p, constant SceneUniforms& U) { + // Pseudo-3D : barres "horizontales" qui s'eloignent + int nbars = 24; + float t = U.time * 0.4; + float3 col = float3(0); + // Sky gradient + float3 sky = mix(float3(0.05, 0.0, 0.15), float3(0.25, 0.1, 0.35), + p.y * 0.5 + 0.5); + col = sky; + for (int i = 0; i < nbars; ++i) { + float fi = float(i) / float(nbars); + // Position en profondeur (z = 0 proche, 1 loin) + float z = fract(fi + t * (0.15 + U.rms * 0.3)); + float perspective = 1.0 / (z + 0.1); + float y_base = -0.6 + z * 1.2; // ligne d'horizon + // Hauteur barre depend du bin "i" via hash + RMS + float h0 = hash21(float2(float(i), 0.0)); + float h = sin(t * (0.5 + h0 * 4.0) + float(i)) * 0.5 + 0.5; + h = h * (0.3 + U.rms * 1.5 + U.social_rate * 0.4); + h = clamp(h, 0.02, 0.85); + float bar_top = y_base + h * perspective * 0.3; + // Largeur = 1 / nbars perspective + float bx = (fi - 0.5) * perspective * 1.5; + float bw = 0.5 / float(nbars) * perspective; + if (abs(p.x - bx) < bw && + p.y > y_base && p.y < bar_top) { + float3 c = palIQ(fi, + float3(0.5), float3(0.5), + float3(1.0, 1.0, 0.5), + float3(0.0, 0.33, 0.67)); + // Fog selon z + c *= 1.0 - z * 0.6; + col = mix(col, c, 1.0 - z * 0.3); + } + } + // Grille du sol scanline + float floor_y = -0.6; + if (p.y < floor_y) { + float depth = (floor_y - p.y) * 4.0; + float grid = step(0.95, fract(p.x * 8.0 / max(depth, 0.1))); + grid += step(0.95, fract(depth * 4.0 + t)); + col += float3(0.2, 0.3, 0.6) * grid * 0.4; + } + return col + U.flare * float3(1.0, 0.5, 0.2) * 0.3; +} + +// ---- 8 hands3d : voyage 3D pilote par les mains ---- +float map_hands(float3 p, constant SceneUniforms& U) { + float3 q = fmod(p + 2.0, 4.0) - 2.0; + float d = length(q) - 0.6; + float pulse = 0.8 + U.rms * 0.6; + d = min(d, length(p) - pulse); + d += sin(p.x * 2.0 + U.time) * 0.15 * U.kp_norm; + return d; +} +float3 mode_hands3d(float2 p, constant SceneUniforms& U) { + float hl_active = (abs(U.hand_l_x) + abs(U.hand_l_y)) > 0.01 ? 1.0 : 0.0; + float hr_active = (abs(U.hand_r_x) + abs(U.hand_r_y)) > 0.01 ? 1.0 : 0.0; + float3 cam_pos = float3( + U.hand_l_x * 5.0, + U.hand_l_y * 3.0, + -U.time * (1.5 + U.hand_l_y * 4.0 * hl_active) + ); + float yaw = U.hand_r_x * 1.2 * hr_active; + float pitch = -U.hand_r_y * 0.8 * hr_active; + float3 rd = normalize(float3(p.x, p.y, 1.5)); + rd = rotX(rd, pitch); + rd = rotY(rd, yaw); + float t = 0.0, glow = 0.0; + for (int i = 0; i < 64; ++i) { + float3 pos = cam_pos + rd * t; + float d = map_hands(pos, U); + if (d < 0.005) break; + glow += 0.02 / (1.0 + d * d * 8.0); + t += d * 0.85; + if (t > 30.0) break; + } + float3 col = float3(0); + if (t < 30.0) { + float3 pos = cam_pos + rd * t; + float fog = 1.0 - saturate(t / 30.0); + col = float3( + 0.5 + 0.5 * sin(pos.x * 0.4 + U.time), + 0.5 + 0.5 * sin(pos.y * 0.5 + U.time * 1.3), + 0.5 + 0.5 * sin(pos.z * 0.3 + U.time * 0.7) + ) * fog; + } + col += float3(0.2, 0.6, 1.0) * glow * 1.5; + col += float3(1.0, 0.5, 0.0) * U.flare * 0.8; + return col; +} + +// ---- 9 openpos : fond minimal radial pour faire ressortir le squelette ---- +// Le rendu des joints + bones se fait par le skel_pipeline rendu PAR-DESSUS +// (cf renderer.py). On laisse juste un degrade radial sombre pour le contraste. +float3 mode_openpos(float2 p, constant SceneUniforms& U) { + float r = length(p); + // Centre legerement plus clair, bords sombres. Touche de couleur + // chaude au centre selon rms pour reagir a la musique. + float3 inner = float3(0.05, 0.05, 0.10) + float3(0.30, 0.12, 0.18) * U.rms; + float3 outer = float3(0.01, 0.01, 0.02); + float3 col = mix(inner, outer, smoothstep(0.0, 1.4, r)); + // Grille de points discrete pour donner une ref de profondeur + float2 g = fmod(p * 12.0, 2.0) - 1.0; + float dot_grid = exp(-dot(g, g) * 6.0) * 0.04; + col += float3(dot_grid); + // Pulsation legere sur le kick / drop + col *= 1.0 + U.rms * 0.4; + return col; +} + +// ===== Fragment dispatcher ========================================= + +fragment float4 bg_fragment(VsOut in [[stage_in]], + constant SceneUniforms& U [[buffer(0)]]) { + float2 uv = in.uv; + float2 p = uv * 2.0 - 1.0; + p.x *= U.width / U.height; + + int mode = int(U.viz_mode + 0.5); + float3 color; + if (mode == 1) color = mode_tunnel(p, U); + else if (mode == 2) color = mode_plasma(p, U); + else if (mode == 3) color = mode_kaleido(p, U); + else if (mode == 4) color = mode_voronoi(p, U); + else if (mode == 5) color = mode_metaballs(p, U); + else if (mode == 6) color = mode_starfield(p, U); + else if (mode == 7) color = mode_bars(p, U); + else if (mode == 8) color = mode_hands3d(p, U); + else if (mode == 9) color = mode_openpos(p, U); + else color = mode_storm(p, U); + + // Flash global + vignette + color += float3(U.lightning_flash * 1.2); + color *= vignette(p); + + // Tone mapping doux (Reinhard) + color = color / (1.0 + color); + // Gamma + color = pow(color, float3(0.85)); + + // Alpha pour transparence quand pose active (webcam visible dessous) + // Overlay vidéo : translucide même sans pose (la webcam doit rester + // visible en fond). Pose active = encore plus translucide. + float alpha = mix(0.55, 0.25, U.pose_alive); + alpha = max(alpha, U.lightning_flash * 0.8); + alpha = max(alpha, U.flare * 0.6); + return float4(color, alpha); +} + +// ===== Skeleton overlay ============================================ + +struct SkelIn { + float3 pos [[attribute(0)]]; // x,y dans NDC, z profondeur (~ -0.5..+0.5) + float conf [[attribute(1)]]; + float pid [[attribute(2)]]; // person_id (0..9) +}; +struct SkelOut { + float4 position [[position]]; + float conf; + float pid; + float depth; +}; + +// Projection perspective douce : eloigne avec z, garde NDC en x,y +vertex SkelOut skel_vertex(SkelIn in [[stage_in]], + constant SceneUniforms& U [[buffer(1)]]) { + SkelOut o; + float z = clamp(in.pos.z, -1.0, 1.0); + // Perspective : plus z augmente, plus le point est loin → scale < 1 + // RMS pulse fait respirer la profondeur + float pulse = 1.0 + U.rms * 0.25; + float persp = 1.0 / (1.0 + z * 0.8); + float2 xy = in.pos.xy * persp * pulse; + o.position = float4(xy, 0.0, 1.0); + o.conf = in.conf; + o.pid = in.pid; + o.depth = z; + return o; +} + +// Palette 6 couleurs par personne (turquoise, magenta, jaune, ambre, lilas, vert) +constant float3 PERSON_COLORS[6] = { + float3(0.0, 1.0, 0.85), // 0 turquoise + float3(1.0, 0.3, 0.7), // 1 magenta + float3(1.0, 0.9, 0.2), // 2 jaune + float3(1.0, 0.55, 0.1), // 3 ambre + float3(0.7, 0.5, 1.0), // 4 lilas + float3(0.4, 1.0, 0.3), // 5+ vert (mains) +}; + +// ===== Mesh overlay (triangles face/hand/body) ===================== +// Reuse meme layout que skel : pos.xyz + conf + pid. + +vertex SkelOut mesh_vertex(SkelIn in [[stage_in]], + constant SceneUniforms& U [[buffer(1)]]) { + SkelOut o; + float z = clamp(in.pos.z, -1.0, 1.0); + float pulse = 1.0 + U.rms * 0.25; + float persp = 1.0 / (1.0 + z * 0.8); + float2 xy = in.pos.xy * persp * pulse; + o.position = float4(xy, 0.0, 1.0); + o.conf = in.conf; + o.pid = in.pid; + o.depth = z; + return o; +} + +fragment float4 mesh_fragment(SkelOut in [[stage_in]]) { + int pid = int(in.pid + 0.5); + pid = ((pid % 6) + 6) % 6; + float3 col = PERSON_COLORS[pid]; + float c = saturate(in.conf); + // Saturation boost : couleurs vives quand pose detectee + col = mix(col, col * 1.6, c); + // Fog par profondeur (proche = plus lumineux) + float depth_fog = 1.0 - clamp(in.depth + 0.5, 0.0, 1.0) * 0.5; + col *= depth_fog; + // Alpha TRES VISIBLE quand confiance haute : 0.85 sur skin, 0.3 fade + return float4(col, mix(0.3, 0.85, c)); +} + +fragment float4 skel_fragment(SkelOut in [[stage_in]]) { + // Skeleton ULTRA visible quand pose detectee : couleur vive + opaque + int pid = int(in.pid + 0.5); + pid = ((pid % 6) + 6) % 6; // modulo positif + float3 col = PERSON_COLORS[pid] * 1.4; // saturation boost + float c = saturate(in.conf); + // Depth fog : eclaircit ce qui est proche, eteint ce qui est loin + float depth_fog = 1.0 - clamp(in.depth + 0.5, 0.0, 1.0) * 0.6; + col *= depth_fog * (0.5 + 0.5 * c); + // Alpha plein-opaque quand confiance haute (= squelette ultra net) + return float4(col, mix(0.5, 1.0, c)); +} diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift new file mode 100644 index 0000000..9d17cda --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift @@ -0,0 +1,120 @@ +import Foundation +import Metal +import MetalKit + +/// Renderer Metal pour les 10 viz modes background (storm, tunnel, +/// plasma, kaleido, voronoi, metaballs, starfield, bars, hands3d, +/// openpos). Reutilise le shader scene.metal porte depuis +/// data_only_viz Python. Sert de couche backing sous l'ARView dans +/// BodyView. +final class SceneRenderer: NSObject, MTKViewDelegate { + // Mirror C struct of scene.metal SceneUniforms (20 floats) + struct SceneUniforms { + var time: Float = 0 + var rms: Float = 0 + var kp_norm: Float = 0 + var netz_dev: Float = 0 + var lightning_flash: Float = 0 + var flare: Float = 0 + var wind_norm: Float = 0 + var bz_norm: Float = 0 + var social_rate: Float = 0 + var pose_alive: Float = 0 + var pose_count: Float = 0 + var width: Float = 1280 + var height: Float = 720 + var viz_mode: Float = 0 + var hand_l_x: Float = 0 + var hand_l_y: Float = 0 + var hand_r_x: Float = 0 + var hand_r_y: Float = 0 + var _pad0: Float = 0 + var _pad1: Float = 0 + } + + private let device: MTLDevice + private let commandQueue: MTLCommandQueue + private let bgPipeline: MTLRenderPipelineState + private let uniformsBuffer: MTLBuffer + private var startTime: CFTimeInterval = CACurrentMediaTime() + + /// Mis a jour en live depuis RenderSettings / OSC handler. + var uniforms = SceneUniforms() + + static func make() -> SceneRenderer? { + return SceneRenderer.init(failable: ()) + } + + private init?(failable: Void) { + guard let dev = MTLCreateSystemDefaultDevice(), + let queue = dev.makeCommandQueue() else { return nil } + self.device = dev + self.commandQueue = queue + + // Compile scene.metal au runtime depuis le bundle + guard let url = Bundle.module.url(forResource: "scene", + withExtension: "metal"), + let source = try? String(contentsOf: url, encoding: .utf8) else { + print("SceneRenderer: scene.metal absent du bundle") + return nil + } + let opts = MTLCompileOptions() + let lib: MTLLibrary + do { + lib = try dev.makeLibrary(source: source, options: opts) + } catch { + print("SceneRenderer: scene.metal compile error: \(error)") + return nil + } + guard let vfn = lib.makeFunction(name: "bg_vertex"), + let ffn = lib.makeFunction(name: "bg_fragment") else { + print("SceneRenderer: bg_vertex/bg_fragment missing") + return nil + } + let pd = MTLRenderPipelineDescriptor() + pd.vertexFunction = vfn + pd.fragmentFunction = ffn + pd.colorAttachments[0].pixelFormat = .bgra8Unorm + // Pas de blending : le MTKView est opaque, l'ARView par-dessus + // est transparent. + do { + self.bgPipeline = try dev.makeRenderPipelineState(descriptor: pd) + } catch { + print("SceneRenderer: pipeline build failed: \(error)") + return nil + } + guard let buf = dev.makeBuffer( + length: MemoryLayout.stride, + options: .storageModeShared) else { return nil } + self.uniformsBuffer = buf + super.init() + } + + // MARK: - MTKViewDelegate + + func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) { + uniforms.width = Float(size.width) + uniforms.height = Float(size.height) + } + + func draw(in view: MTKView) { + uniforms.time = Float(CACurrentMediaTime() - startTime) + // Copy uniforms vers buffer GPU (shared memory) + let ptr = uniformsBuffer.contents().bindMemory( + to: SceneUniforms.self, capacity: 1) + ptr.pointee = uniforms + + guard let rpd = view.currentRenderPassDescriptor, + let drawable = view.currentDrawable, + let cb = commandQueue.makeCommandBuffer(), + let enc = cb.makeRenderCommandEncoder(descriptor: rpd) else { + return + } + enc.setRenderPipelineState(bgPipeline) + enc.setFragmentBuffer(uniformsBuffer, offset: 0, index: 0) + enc.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + enc.endEncoding() + cb.present(drawable) + cb.commit() + } +} diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift index 65176f5..ffdbd41 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift @@ -57,6 +57,9 @@ struct SettingsPanel: View { layerRow(icon: "video.fill", label: "Webcam", isOn: $settings.showCamera) + layerRow(icon: "sparkles", + label: "Scene Metal (\(settings.vizModeName))", + isOn: $settings.showScene) layerRow(icon: "person.fill", label: "Maillage SMPL-X", isOn: $settings.showMesh) @@ -66,6 +69,22 @@ struct SettingsPanel: View { layerRow(icon: "figure.stand", label: "Squelette (à venir)", isOn: $settings.showSkeleton) + // Picker viz mode 0..9 + HStack(spacing: 4) { + ForEach(0..<10) { i in + Button(action: { settings.vizMode = i }) { + Text(String(i)) + .font(.caption2.monospacedDigit()) + .frame(width: 22, height: 22) + .background( + Circle().fill(settings.vizMode == i + ? Color.pink + : Color.white.opacity(0.1))) + .foregroundColor(.white) + } + .buttonStyle(.plain) + } + } } } From 24d1e85b7d8ea4b25cffffd61b047d4498283032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 21:40:22 +0200 Subject: [PATCH 05/18] feat(data-only-viz): action-head augmentations Implement on-the-fly spatial and temporal augmentations for multi-HMR j3d windows: mirror_x (left/right joint swap + x-flip), add_noise, time_stretch (linear resampling), rotate_y. Task 7 of action-head plan. All 30 tests pass (26 prior + 4 new augment tests). --- data_only_viz/tests/test_augment.py | 45 ++++++++++++++++ data_only_viz/training/augment.py | 80 +++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 data_only_viz/tests/test_augment.py create mode 100644 data_only_viz/training/augment.py diff --git a/data_only_viz/tests/test_augment.py b/data_only_viz/tests/test_augment.py new file mode 100644 index 0000000..c87b358 --- /dev/null +++ b/data_only_viz/tests/test_augment.py @@ -0,0 +1,45 @@ +"""Tests for j3d augmentations.""" +from __future__ import annotations + +import numpy as np + +WINDOW_LEN = 16 + + +def _sample_stack(seed: int = 0) -> np.ndarray: + rng = np.random.default_rng(seed) + return rng.normal(size=(WINDOW_LEN, 22, 3)).astype(np.float32) + + +def test_mirror_swap_left_right_joints() -> None: + from data_only_viz.training.augment import mirror_x + x = _sample_stack(0) + y = mirror_x(x) + assert np.allclose(y[..., 0], -x[..., 0][:, [ + 0,2,1,3,5,4,6,8,7,9,11,10,12,14,13,15,17,16,19,18,21,20 + ]], atol=1e-6) + + +def test_noise_within_sigma() -> None: + from data_only_viz.training.augment import add_noise + rng = np.random.default_rng(0) + x = _sample_stack(0) + y = add_noise(x, sigma=0.01, rng=rng) + diff = y - x + assert np.allclose(diff.std(), 0.01, atol=2e-3) + + +def test_time_stretch_keeps_shape() -> None: + from data_only_viz.training.augment import time_stretch + x = _sample_stack(0) + y = time_stretch(x, factor=0.9, rng=None) + assert y.shape == x.shape + + +def test_rotate_y_preserves_distances() -> None: + from data_only_viz.training.augment import rotate_y + x = _sample_stack(0) + y = rotate_y(x, angle_rad=0.3) + d_x = np.linalg.norm(x[0, 0] - x[0, 1]) + d_y = np.linalg.norm(y[0, 0] - y[0, 1]) + assert abs(d_x - d_y) < 1e-5 diff --git a/data_only_viz/training/augment.py b/data_only_viz/training/augment.py new file mode 100644 index 0000000..c1b7d93 --- /dev/null +++ b/data_only_viz/training/augment.py @@ -0,0 +1,80 @@ +"""On-the-fly augmentations for j3d windows.""" +from __future__ import annotations + +import numpy as np + +# SMPL-X left/right joint mirror map (subset 22 joints used by Multi-HMR). +MIRROR_MAP: tuple[int, ...] = ( + 0, + 2, 1, + 3, + 5, 4, + 6, + 8, 7, + 9, + 11, 10, + 12, + 14, 13, + 15, + 17, 16, + 19, 18, + 21, 20, +) + + +def mirror_x(stack: np.ndarray) -> np.ndarray: + """Mirror across the YZ plane: flip x and swap left↔right joints.""" + out = stack[:, list(MIRROR_MAP), :].copy() + out[..., 0] = -out[..., 0] + return out + + +def add_noise(stack: np.ndarray, sigma: float, rng: np.random.Generator) -> np.ndarray: + noise = rng.normal(scale=sigma, size=stack.shape).astype(np.float32) + return (stack + noise).astype(np.float32, copy=False) + + +def time_stretch(stack: np.ndarray, factor: float, + rng: np.random.Generator | None = None) -> np.ndarray: + """Resample the time axis with linear interpolation, keep window_len fixed.""" + T = stack.shape[0] + new_T = int(round(T * factor)) + new_T = max(2, new_T) + src = np.linspace(0.0, T - 1, num=new_T) + interp = np.empty((new_T, *stack.shape[1:]), dtype=np.float32) + lo = np.floor(src).astype(int) + hi = np.minimum(lo + 1, T - 1) + frac = (src - lo).astype(np.float32) + interp = (1 - frac[:, None, None]) * stack[lo] + frac[:, None, None] * stack[hi] + if new_T >= T: + start = (new_T - T) // 2 + return interp[start:start + T].astype(np.float32, copy=False) + pad_before = (T - new_T) // 2 + pad_after = T - new_T - pad_before + return np.concatenate([ + np.repeat(interp[:1], pad_before, axis=0), + interp, + np.repeat(interp[-1:], pad_after, axis=0), + ]).astype(np.float32, copy=False) + + +def rotate_y(stack: np.ndarray, angle_rad: float) -> np.ndarray: + """Rotate around Y (vertical) axis.""" + c, s = np.cos(angle_rad), np.sin(angle_rad) + R = np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]], dtype=np.float32) + return (stack @ R.T).astype(np.float32, copy=False) + + +def random_augment(stack: np.ndarray, rng: np.random.Generator) -> np.ndarray: + out = stack + if rng.random() < 0.5: + out = mirror_x(out) + if rng.random() < 0.8: + out = add_noise(out, sigma=0.01, rng=rng) + if rng.random() < 0.5: + factor = float(rng.uniform(0.9, 1.1)) + out = time_stretch(out, factor=factor, rng=rng) + if rng.random() < 0.5: + angle = float(rng.uniform(-np.deg2rad(15), np.deg2rad(15))) + out = rotate_y(out, angle_rad=angle) + return out From 9e7a9f8fd46dc7adbbc46857ff4634b1903a76b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 21:42:43 +0200 Subject: [PATCH 06/18] feat(data-only-viz): Multi-HMR CoreML backend Convert Multi-HMR ViT-S 672 to CoreML mlpackage and wire as optional worker backend via MULTIHMR_BACKEND=coreml env var. Inference path uses pyobjc + native CoreML framework (Python 3.14 has no libcoremlpython binding). Conversion done in a separate Py 3.12 venv; einsum cascade patched (camera intrinsics broadcast + smplx landmarks) via setup_multihmr.sh, idempotent on re-clone. Bench: 28 ms mock, 100-170 ms live (~13 fps, 4x PyTorch MPS). ANE compile fails on this model; CPU+GPU is the sweet spot. --- data_only_viz/multi_hmr_worker.py | 239 ++++++++++++++++- data_only_viz/multihmr_coreml.py | 276 ++++++++++++++++++++ data_only_viz/scripts/coreml_full_probe.py | 69 ++++- data_only_viz/scripts/setup_multihmr.sh | 67 +++++ data_only_viz/tests/test_multihmr_coreml.py | 96 +++++++ 5 files changed, 741 insertions(+), 6 deletions(-) create mode 100644 data_only_viz/multihmr_coreml.py create mode 100644 data_only_viz/tests/test_multihmr_coreml.py diff --git a/data_only_viz/multi_hmr_worker.py b/data_only_viz/multi_hmr_worker.py index 52a64cd..a09092a 100644 --- a/data_only_viz/multi_hmr_worker.py +++ b/data_only_viz/multi_hmr_worker.py @@ -30,6 +30,7 @@ CACHE = Path.home() / ".cache" / "av-live-multihmr" CKPT = CACHE / "checkpoints" / "multiHMR_672_S.pt" SMPLX_PATH = CACHE / "models" / "smplx" / "SMPLX_NEUTRAL.npz" MULTIHMR_REPO = CACHE / "multi-hmr" +COREML_MLPACKAGE = CACHE / "multihmr_full_672_s.mlpackage" IMG_SIZE = 672 N_VERTS = 10475 @@ -41,7 +42,8 @@ class MultiHMRWorker: det_thresh: float = 0.3, nms_kernel_size: int = 5, motion_gate: float = 5.0, - camera_index: int = -1) -> None: + camera_index: int = -1, + backend: str | None = None) -> None: self.state = state self.num_persons = num_persons self.period = 1.0 / max(1.0, target_fps) @@ -55,6 +57,12 @@ class MultiHMRWorker: self.motion_gate = motion_gate # -1 = auto-select Mac BuiltInWideAngleCamera (cf _camera_select) self.camera_index = camera_index + # backend: 'pytorch' (default) or 'coreml'. CoreML uses the + # .mlpackage at COREML_MLPACKAGE, bypasses MPS torch, and runs + # on ANE/GPU/CPU via CoreML.framework natively (3-4x faster). + self.backend = (backend + or os.environ.get("MULTIHMR_BACKEND", "pytorch") + ).strip().lower() self._stop = threading.Event() self._thread: threading.Thread | None = None self._smooth_shape = [ @@ -72,6 +80,9 @@ class MultiHMRWorker: @staticmethod def is_available() -> bool: + backend = os.environ.get("MULTIHMR_BACKEND", "pytorch").strip().lower() + if backend == "coreml": + return COREML_MLPACKAGE.exists() return CKPT.exists() and SMPLX_PATH.exists() and MULTIHMR_REPO.exists() def start(self) -> None: @@ -83,6 +94,12 @@ class MultiHMRWorker: self._stop.set() def _run(self) -> None: + if self.backend == "coreml": + self._run_coreml() + return + self._run_pytorch() + + def _run_pytorch(self) -> None: if str(MULTIHMR_REPO) not in sys.path: sys.path.insert(0, str(MULTIHMR_REPO)) # Multi-HMR demo.py tire pyrender / pyvista (OpenGL offscreen) et @@ -373,3 +390,223 @@ class MultiHMRWorker: cap.stop() LOG.info("multi_hmr worker stopped") + + # ------------------------------------------------------------------ + # CoreML backend + # ------------------------------------------------------------------ + def _run_coreml(self) -> None: + """CoreML inference path (ANE+GPU+CPU via Apple's framework). + + Mirrors _run_pytorch but loads the .mlpackage via pyobjc + the + CoreML.framework, bypassing torch/MPS entirely. ~3-4x faster + on M5 (28.8ms median vs ~100ms with MPS).""" + try: + import cv2 + except ImportError as e: + LOG.error("opencv-python missing: %s", e) + return + try: + from .multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(COREML_MLPACKAGE) + except Exception as e: # noqa: BLE001 + LOG.error("CoreML backend init failed: %s", e) + return + + focal = float(IMG_SIZE) + K_np = np.array([[focal, 0.0, IMG_SIZE / 2.0], + [0.0, focal, IMG_SIZE / 2.0], + [0.0, 0.0, 1.0]], dtype=np.float32) + + from ._av_capture import ( + AVCapture, find_builtin_device, enumerate_devices) + if self.camera_index >= 0: + devs = enumerate_devices() + if self.camera_index >= len(devs): + LOG.error("camera_index %d hors de %d devices", + self.camera_index, len(devs)) + return + info = devs[self.camera_index] + else: + info = find_builtin_device() + if info is None: + LOG.error("aucune BuiltInWideAngleCamera trouvee") + return + cap = AVCapture(info) + if not cap.start(): + LOG.error("AVCapture start failed pour %s", info["name"]) + return + LOG.info("camera ouverte %s (%s) [coreml backend]", + info["name"], info["type"]) + + frame_count = 0 + persons_count = 0 + skipped_static = 0 + next_heartbeat = time.monotonic() + 5.0 + prev_thumb: np.ndarray | None = None + + while not self._stop.is_set(): + t_cap_start = time.monotonic() + ok, frame_bgr = cap.read(timeout_s=0.5) + if not ok or frame_bgr is None: + time.sleep(self.period) + continue + + t_pre_start = time.monotonic() + h, w = frame_bgr.shape[:2] + if (h, w) != (IMG_SIZE, IMG_SIZE): + side = min(h, w) + y0 = (h - side) // 2 + x0 = (w - side) // 2 + frame_bgr = frame_bgr[y0:y0 + side, x0:x0 + side] + frame_bgr = cv2.resize(frame_bgr, (IMG_SIZE, IMG_SIZE)) + + if self.motion_gate > 0: + thumb = cv2.cvtColor( + cv2.resize(frame_bgr, (112, 112)), + cv2.COLOR_BGR2GRAY) + if prev_thumb is not None: + diff_mean = float(np.mean( + cv2.absdiff(thumb, prev_thumb))) + if diff_mean < self.motion_gate: + prev_thumb = thumb + skipped_static += 1 + time.sleep(self.period) + continue + prev_thumb = thumb + + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + img = frame_rgb.transpose(2, 0, 1).astype(np.float32) / 255.0 + + t_inf_start = time.monotonic() + try: + humans = backend.infer(img, K_np, det_thresh=self.det_thresh) + except Exception as e: # noqa: BLE001 + LOG.warning("coreml inference failed: %s", e) + time.sleep(self.period) + continue + + t_post_start = time.monotonic() + t_now = time.monotonic() + frame_count += 1 + persons_count += len(humans) if humans else 0 + if t_now >= next_heartbeat: + fps = frame_count / 5.0 + avg = persons_count / max(1, frame_count) + LOG.info( + "hb[coreml]: %.1f fps, %.2f persons/frame, %d skipped", + fps, avg, skipped_static) + frame_count = 0 + persons_count = 0 + skipped_static = 0 + next_heartbeat = t_now + 5.0 + + if not humans: + with self.state.lock(): + self.state.persons_smplx = [] + time.sleep(self.period) + continue + + # Dedup intra-frame (same logic as pytorch path). + cand: list[tuple[ + float, float, float, float, float, + np.ndarray, int]] = [] + for i, hh in enumerate(humans): + v = hh["v3d"].detach().cpu().numpy() + xmin = float(v[:, 0].min()); ymin = float(v[:, 1].min()) + xmax = float(v[:, 0].max()); ymax = float(v[:, 1].max()) + score = float(hh["scores"].item()) + pelv = hh["transl_pelvis"].detach().cpu().numpy( + ).flatten()[:3] + cand.append((score, xmin, ymin, xmax, ymax, pelv, i)) + cand.sort(key=lambda c: -c[0]) + keep_idx: list[int] = [] + kept: list[tuple[float, float, float, float, np.ndarray]] = [] + for sc, x0, y0, x1, y1, pelv, src_i in cand: + a_area = max(0.0, x1 - x0) * max(0.0, y1 - y0) + drop = False + for (kx0, ky0, kx1, ky1, kpelv) in kept: + ix0 = max(x0, kx0); iy0 = max(y0, ky0) + ix1 = min(x1, kx1); iy1 = min(y1, ky1) + iw = max(0.0, ix1 - ix0); ih = max(0.0, iy1 - iy0) + inter = iw * ih + if a_area <= 0 or inter <= 0: + continue + k_area = (kx1 - kx0) * (ky1 - ky0) + iou = inter / (a_area + k_area - inter + 1e-9) + pelv_d = float(np.linalg.norm(pelv - kpelv)) + if iou > 0.55 and pelv_d < 0.20: + drop = True + break + if not drop: + keep_idx.append(src_i) + kept.append((x0, y0, x1, y1, pelv)) + if len(keep_idx) >= self.num_persons: + break + humans = [humans[i] for i in keep_idx] + n_keep = len(humans) + + bboxes = [] + for hh in humans: + v = hh["v3d"].detach().cpu().numpy() + xmin, ymin = float(v[:, 0].min()), float(v[:, 1].min()) + xmax, ymax = float(v[:, 0].max()), float(v[:, 1].max()) + bboxes.append([PoseKp(x=xmin, y=ymin, c=1.0), + PoseKp(x=xmax, y=ymax, c=1.0)]) + ids = self._tracker.update(bboxes) + + persons: list[SMPLXPerson] = [] + for i, hh in enumerate(humans[:n_keep]): + pid = ids[i] if i < len(ids) else i + if pid < 0: + continue + v3d = hh["v3d"].detach().cpu().numpy() + transl_np = hh["transl_pelvis"].detach().cpu().numpy().flatten() + shape_raw = hh["shape"].detach().cpu().numpy().flatten() + expr_raw = hh["expression"].detach().cpu().numpy().flatten() + + pid_c = pid % self.num_persons + shape_n = min(10, len(shape_raw)) + expr_n = min(10, len(expr_raw)) + shape_smooth = np.zeros(10, dtype=np.float32) + expr_smooth = np.zeros(10, dtype=np.float32) + for k in range(shape_n): + shape_smooth[k] = self._smooth_shape[pid_c][k]( + float(shape_raw[k]), t_now) + for k in range(expr_n): + expr_smooth[k] = self._smooth_expr[pid_c][k]( + float(expr_raw[k]), t_now) + + persons.append(SMPLXPerson( + pid=int(pid), + vertices_3d=np.ascontiguousarray(v3d, dtype=np.float32), + translation=np.ascontiguousarray( + transl_np[:3], dtype=np.float32), + confidence=float(hh["scores"].item()), + betas=np.ascontiguousarray(shape_smooth, dtype=np.float32), + expression=np.ascontiguousarray(expr_smooth, dtype=np.float32), + )) + + with self.state.lock(): + self.state.persons_smplx = persons + self.state.smplx_last_t = t_now + + t_end = time.monotonic() + dt_total = (t_end - t_cap_start) * 1e3 + if LOG.isEnabledFor(logging.DEBUG) or dt_total > 100.0: + LOG.log( + logging.DEBUG if dt_total <= 100.0 else logging.WARNING, + "frame[coreml]: cap=%.1f pre=%.1f inf=%.1f " + "post=%.1fms total=%.1fms", + (t_pre_start - t_cap_start) * 1e3, + (t_inf_start - t_pre_start) * 1e3, + (t_post_start - t_inf_start) * 1e3, + (t_end - t_post_start) * 1e3, + dt_total, + ) + + dt = time.monotonic() - t_cap_start + if dt < self.period: + time.sleep(self.period - dt) + + cap.stop() + LOG.info("multi_hmr coreml worker stopped") diff --git a/data_only_viz/multihmr_coreml.py b/data_only_viz/multihmr_coreml.py new file mode 100644 index 0000000..f46dc4b --- /dev/null +++ b/data_only_viz/multihmr_coreml.py @@ -0,0 +1,276 @@ +"""Multi-HMR CoreML backend (ANE/GPU/CPU via Apple's CoreML framework). + +Python 3.14 cannot use `coremltools.MLModel` because `libcoremlpython` +and `libmilstoragepython` native extensions are not distributed for +3.14. We load CoreML.framework directly via `objc.loadBundle()` — +same pattern as `coreml_pose.py`. + +Unlike `coreml_pose.py`, this backend does NOT use Vision: Vision is +limited to image inputs and cannot feed a second MLMultiArray (cam_K). +We invoke `MLModel.predictionFromFeatures:error:` directly with a +`MLDictionaryFeatureProvider` wrapping two `MLMultiArray`s. + +Public API: + backend = MultiHMRCoreMLBackend(mlpackage_path) + humans = backend.infer(image_chw_f32, K_33_f32, det_thresh=0.3) + # humans is a list[dict] with the same keys as the PyTorch model + # output. Values are CoreMLArray instances that quack like torch + # tensors (.detach().cpu().numpy() / .item()). +""" +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +import numpy as np +import objc +from Foundation import NSURL + +LOG = logging.getLogger("multihmr_coreml") + +DEFAULT_MLPACKAGE = ( + Path.home() / ".cache" / "av-live-multihmr" + / "multihmr_full_672_s.mlpackage" +) + +# Multi-HMR exported with apply_topk(K=4): outputs are fixed shape. +N_PERSONS_FIXED = 4 +N_VERTS = 10475 + +# CoreML output names from the exported .mlpackage. +OUT_V3D = "var_2541" # (4, 10475, 3) f16 +OUT_TRANSL = "var_2544" # (4, 1, 3) f16 +OUT_SCORES = "var_2557" # (4,) f16 +OUT_BETAS = "var_2560" # (4, 10) f16 +OUT_EXPR = "var_2563" # (4, 10) f16 + +# MLMultiArrayDataType raw values (from CoreML headers). +ML_DTYPE_FLOAT32 = 65568 +ML_DTYPE_FLOAT16 = 65552 +ML_DTYPE_DOUBLE = 65600 +ML_DTYPE_INT32 = 131104 + + +_NS: dict[str, Any] = {} +_FRAMEWORKS_LOADED = False + + +def _load_frameworks() -> dict[str, Any]: + global _FRAMEWORKS_LOADED + if _FRAMEWORKS_LOADED: + return _NS + objc.loadBundle("CoreML", _NS, + "/System/Library/Frameworks/CoreML.framework") + _FRAMEWORKS_LOADED = True + return _NS + + +class CoreMLArray: + """Tiny tensor-like adapter so the existing worker hot path can + treat CoreML outputs the same way it treats torch tensors. + + Supports `.detach().cpu().numpy()` and `.item()`. The wrapper is + a no-op around a numpy array; we keep the chain so callers don't + need any conditional branch.""" + + __slots__ = ("_arr",) + + def __init__(self, arr: np.ndarray) -> None: + self._arr = arr + + def detach(self) -> "CoreMLArray": + return self + + def cpu(self) -> "CoreMLArray": + return self + + def numpy(self) -> np.ndarray: + return self._arr + + def item(self) -> float: + return float(self._arr.reshape(-1)[0]) + + @property + def shape(self) -> tuple[int, ...]: + return tuple(self._arr.shape) + + +def _np_to_mlarray(arr: np.ndarray): + """Create a contiguous float32 MLMultiArray from a numpy array. + + We always feed FLOAT32 — even though outputs are FLOAT16, CoreML + will auto-cast on the input side.""" + ns = _load_frameworks() + MLMultiArray = ns["MLMultiArray"] + arr = np.ascontiguousarray(arr, dtype=np.float32) + shape = [int(s) for s in arr.shape] + ml = MLMultiArray.alloc().initWithShape_dataType_error_( + shape, ML_DTYPE_FLOAT32, None) + if ml is None: + raise RuntimeError("MLMultiArray alloc failed") + # Copy bytes through dataPointer (raw void*). pyobjc exposes it as + # a memoryview-like opaque; we use ctypes to memcpy. + import ctypes + ptr = ml.dataPointer() + n_bytes = arr.nbytes + # pyobjc returns either an objc.varlist or a Python int pointer. + addr = int(ptr) if isinstance(ptr, int) else ctypes.cast( + ptr, ctypes.c_void_p).value + if addr is None: + raise RuntimeError("MLMultiArray dataPointer null") + ctypes.memmove(addr, arr.ctypes.data, n_bytes) + return ml + + +def _mlarray_to_np(ml) -> np.ndarray: + """Copy an MLMultiArray (FLOAT16 or FLOAT32) into a numpy float32.""" + import ctypes + shape = tuple(int(s) for s in ml.shape()) + dtype_id = int(ml.dataType()) + count = 1 + for s in shape: + count *= s + ptr = ml.dataPointer() + addr = int(ptr) if isinstance(ptr, int) else ctypes.cast( + ptr, ctypes.c_void_p).value + if addr is None: + raise RuntimeError("MLMultiArray dataPointer null") + if dtype_id == ML_DTYPE_FLOAT16: + raw = (ctypes.c_uint16 * count).from_address(addr) + arr = np.ctypeslib.as_array(raw).view(np.float16).astype(np.float32) + elif dtype_id == ML_DTYPE_FLOAT32: + raw = (ctypes.c_float * count).from_address(addr) + arr = np.ctypeslib.as_array(raw).copy() + elif dtype_id == ML_DTYPE_DOUBLE: + raw = (ctypes.c_double * count).from_address(addr) + arr = np.ctypeslib.as_array(raw).astype(np.float32) + else: + raise RuntimeError(f"unsupported MLMultiArray dtype {dtype_id}") + return arr.reshape(shape) + + +class MultiHMRCoreMLBackend: + """CoreML inference wrapper for Multi-HMR (full_672_s).""" + + def __init__(self, mlpackage_path: Path | None = None) -> None: + self.path = Path(mlpackage_path) if mlpackage_path else DEFAULT_MLPACKAGE + if not self.path.exists(): + raise FileNotFoundError(f"mlpackage missing: {self.path}") + ns = _load_frameworks() + MLModel = ns["MLModel"] + MLModelConfiguration = ns["MLModelConfiguration"] + cfg = MLModelConfiguration.alloc().init() + try: + # MLComputeUnits: 0=CPUOnly, 1=CPUAndGPU, 2=All (ANE+GPU+CPU), + # 3=CPUAndNeuralEngine. Multi-HMR's ANEF compile fails + # (validated 2026-05-13 on M5), and 'All' falls back to a + # slow path (~146ms). CPU+GPU = 28ms = ~35fps on M5. + cfg.setComputeUnits_(1) + except Exception: # noqa: BLE001 + pass + url = NSURL.fileURLWithPath_(str(self.path)) + # .mlpackage must be compiled to .mlmodelc before MLModel can + # load it. compileModelAtURL_error_ returns an NSURL to a + # temp .mlmodelc bundle. + compiled_url = MLModel.compileModelAtURL_error_(url, None) + if compiled_url is None: + raise RuntimeError(f"compileModelAtURL failed for {self.path}") + model = MLModel.modelWithContentsOfURL_configuration_error_( + compiled_url, cfg, None) + if model is None: + raise RuntimeError(f"MLModel load failed for {compiled_url}") + self._model = model + self._ns = ns + LOG.info("Multi-HMR CoreML model loaded (%s, computeUnits=CPU+GPU)", + self.path.name) + + @staticmethod + def is_available(mlpackage_path: Path | None = None) -> bool: + p = Path(mlpackage_path) if mlpackage_path else DEFAULT_MLPACKAGE + if not p.exists(): + return False + try: + _load_frameworks() + return True + except Exception: # noqa: BLE001 + return False + + def _predict(self, image_4d: np.ndarray, K_33: np.ndarray) -> dict: + ns = self._ns + MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"] + MLFeatureValue = ns["MLFeatureValue"] + img_ml = _np_to_mlarray(image_4d) + k_ml = _np_to_mlarray(K_33) + feats = { + "image": MLFeatureValue.featureValueWithMultiArray_(img_ml), + "cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml), + } + provider = MLDictionaryFeatureProvider.alloc( + ).initWithDictionary_error_(feats, None) + if provider is None: + raise RuntimeError("MLDictionaryFeatureProvider alloc failed") + out = self._model.predictionFromFeatures_error_(provider, None) + if out is None: + raise RuntimeError("MLModel predict failed") + names = [str(n) for n in out.featureNames()] + result = {} + for n in names: + fv = out.featureValueForName_(n) + ml = fv.multiArrayValue() + if ml is None: + continue + result[n] = _mlarray_to_np(ml) + return result + + def infer( + self, + image_chw_float32: np.ndarray, + K_33: np.ndarray, + det_thresh: float = 0.3, + ) -> list[dict]: + """Run a forward pass and return list of humans dicts. + + Args: + image_chw_float32: (3, 672, 672) or (1, 3, 672, 672) in [0,1]. + K_33: (3, 3) or (1, 3, 3) camera intrinsics. + det_thresh: scores threshold; CoreML forwards K=4 always. + + Returns: + list[dict] with keys v3d, transl_pelvis, scores, shape, + expression. Values are CoreMLArray wrappers. + """ + img = np.asarray(image_chw_float32, dtype=np.float32) + if img.ndim == 3: + img = img[np.newaxis, ...] + if img.shape != (1, 3, 672, 672): + raise ValueError(f"image shape {img.shape}, expected (1,3,672,672)") + K = np.asarray(K_33, dtype=np.float32) + if K.ndim == 2: + K = K[np.newaxis, ...] + if K.shape != (1, 3, 3): + raise ValueError(f"K shape {K.shape}, expected (1,3,3)") + + raw = self._predict(img, K) + v3d = raw.get(OUT_V3D) + transl = raw.get(OUT_TRANSL) + scores = raw.get(OUT_SCORES) + betas = raw.get(OUT_BETAS) + expr = raw.get(OUT_EXPR) + if any(x is None for x in (v3d, transl, scores, betas, expr)): + raise RuntimeError( + "missing outputs; got keys=" + ",".join(raw.keys())) + + humans: list[dict] = [] + for k in range(N_PERSONS_FIXED): + sc = float(scores[k]) + if sc < det_thresh: + continue + humans.append({ + "v3d": CoreMLArray(v3d[k]), # (10475, 3) + "transl_pelvis": CoreMLArray(transl[k]), # (1, 3) + "scores": CoreMLArray(np.array([sc], dtype=np.float32)), + "shape": CoreMLArray(betas[k]), # (10,) + "expression": CoreMLArray(expr[k]), # (10,) + }) + return humans diff --git a/data_only_viz/scripts/coreml_full_probe.py b/data_only_viz/scripts/coreml_full_probe.py index 241ce15..f0e7bfe 100644 --- a/data_only_viz/scripts/coreml_full_probe.py +++ b/data_only_viz/scripts/coreml_full_probe.py @@ -170,11 +170,28 @@ _K_INV_PRE = torch.tensor([ ]) def inverse_perspective_projection_fixed(points, K, distance): - """Bypass torch.inverse : utilise K_inv pre-calcule en closed-form - (notre K est connu et fixe). Le K argument est ignore.""" - K_inv = _K_INV_PRE.to(points.device).to(points.dtype) - points = torch.cat([points, torch.ones_like(points[..., :1])], -1) - points = torch.einsum('bij,bkj->bki', K_inv, points) + """Bypass torch.inverse + einsum + matmul pour eviter le bug + coremltools de broadcast batch 1->K sur ces ops. K_inv etant + fixe et structure (diag + translate), on ecrit les composantes + explicitement en ops elementaires. + + K_inv = [[1/f, 0, -cx/f], [0, 1/f, -cy/f], [0, 0, 1]] + Pour points (b, N, 3) : out = points @ K_inv.T donne : + out[..., 0] = points[..., 0]/f - (cx/f) * points[..., 2] + out[..., 1] = points[..., 1]/f - (cy/f) * points[..., 2] + out[..., 2] = points[..., 2] + """ + points_hom = torch.cat([points, torch.ones_like(points[..., :1])], -1) + inv_f = 1.0 / focal_val + cx_over_f = cx / focal_val + cy_over_f = cy / focal_val + x = points_hom[..., 0:1] + y = points_hom[..., 1:2] + z = points_hom[..., 2:3] + out0 = x * inv_f - z * cx_over_f + out1 = y * inv_f - z * cy_over_f + out2 = z + points = torch.cat([out0, out1, out2], dim=-1) if distance is None: return points points = points * distance @@ -190,6 +207,26 @@ model_mod.inverse_perspective_projection = inverse_perspective_projection_fixed import blocks.smpl_layer as _smpl_layer _smpl_layer.inverse_perspective_projection = inverse_perspective_projection_fixed +# Aussi perspective_projection (utilise dans smpl_layer.py:143-144 pour +# j2d et v2d) -> rewrite einsum en matmul pour le meme broadcast bug. +def perspective_projection_fixed(x, K): + """Element-wise rewrite de la projection perspective avec K fixe + (focal=IMG_SIZE, cx=cy=IMG_SIZE/2). Bypass matmul/einsum pour eviter + les bugs broadcast coremltools. + K = [[f, 0, cx], [0, f, cy], [0, 0, 1]] + out[..., 0] = f * x_norm + cx * z_norm (mais on veut [..., :2]) + = f * (x/z) + cx + out[..., 1] = f * (y/z) + cy + """ + z = x[..., 2:3] + px = x[..., 0:1] / z * focal_val + cx + py = x[..., 1:2] / z * focal_val + cy + return torch.cat([px, py], dim=-1) + +_camera.perspective_projection = perspective_projection_fixed +_utils_pkg.perspective_projection = perspective_projection_fixed +_smpl_layer.perspective_projection = perspective_projection_fixed + # === Wrapper qui produit tuple fixe === class TracedMHMR(nn.Module): @@ -422,6 +459,28 @@ def _diagonal_general(context, node): _TORCH_OPS_REGISTRY.name_to_func_mapping["diagonal"] = _diagonal_general + +# Instrument reshape pour logger node source au moment de l'erreur. +from coremltools.converters.mil.mil.ops.defs.iOS15 import tensor_transformation as _tt +_orig_reshape_ti = _tt.reshape.type_inference + + +def _reshape_ti_logged(self): + try: + return _orig_reshape_ti(self) + except ValueError as e: + if "Invalid target shape" in str(e): + try: + from_shape = list(self.x.shape) + target = list(self.shape.val) if hasattr(self.shape, "val") else "?" + print(f" >>> RESHAPE FAIL : name={self.name} from={from_shape} target={target}") + except Exception: + pass + raise + + +_tt.reshape.type_inference = _reshape_ti_logged + try: mlmodel = ct.convert( traced, diff --git a/data_only_viz/scripts/setup_multihmr.sh b/data_only_viz/scripts/setup_multihmr.sh index b51df54..fcf96cb 100755 --- a/data_only_viz/scripts/setup_multihmr.sh +++ b/data_only_viz/scripts/setup_multihmr.sh @@ -49,4 +49,71 @@ if [ ! -e "$CACHE/multi-hmr/models" ]; then ln -sfn ../models "$CACHE/multi-hmr/models" fi +# CoreML conversion patches : remplace les torch.einsum dans utils/camera.py +# par des ops element-wise (broadcast-friendly). Sans ca, ct.convert echoue +# avec "Invalid target shape in reshape op ([1, N, 3] to [K*N, 3, 1])" +# quand batch K detections != 1. Idempotent. +CAM="$CACHE/multi-hmr/utils/camera.py" +if [ -f "$CAM" ] && ! grep -q "_apply_intrinsics_componentwise" "$CAM"; then + echo "==> Patch utils/camera.py (einsum -> componentwise)" + python3 - "$CAM" <<'PYEOF' +import sys, pathlib +p = pathlib.Path(sys.argv[1]) +src = p.read_text() +helper = ''' +def _apply_intrinsics_componentwise(K, y): + """CoreML-friendly: out[b,k,i] = sum_j K[b,i,j] * y[b,k,j] + Replaces torch.einsum('bij,bkj->bki', K, y) with pure broadcast ops. + """ + K00 = K[:, 0:1, 0:1]; K01 = K[:, 0:1, 1:2]; K02 = K[:, 0:1, 2:3] + K10 = K[:, 1:2, 0:1]; K11 = K[:, 1:2, 1:2]; K12 = K[:, 1:2, 2:3] + K20 = K[:, 2:3, 0:1]; K21 = K[:, 2:3, 1:2]; K22 = K[:, 2:3, 2:3] + y0 = y[:, :, 0:1]; y1 = y[:, :, 1:2]; y2 = y[:, :, 2:3] + out0 = K00 * y0 + K01 * y1 + K02 * y2 + out1 = K10 * y0 + K11 * y1 + K12 * y2 + out2 = K20 * y0 + K21 * y1 + K22 * y2 + return torch.cat([out0, out1, out2], dim=-1) + + +''' +src = src.replace( + "def perspective_projection(x, K):", + helper + "def perspective_projection(x, K):", +) +src = src.replace( + "y = torch.einsum('bij,bkj->bki', K, y) # (bs, N, 3)", + "y = _apply_intrinsics_componentwise(K, y)", +) +src = src.replace( + "points = torch.einsum('bij,bkj->bki', torch.inverse(K), points)", + "points = _apply_intrinsics_componentwise(torch.inverse(K), points)", +) +p.write_text(src) +print(" camera.py patched") +PYEOF +fi + +# CoreML conversion patch : smplx/lbs.py landmarks einsum (mеme bug broadcast) +# Patch best-effort sur tous les venvs presents (data_only_viz + /tmp/coreml312). +for VENV in \ + "$(dirname "$(dirname "$(readlink -f "$0")")")/.venv" \ + "/tmp/coreml312"; do + LBS="$VENV/lib/python3.14/site-packages/smplx/lbs.py" + [ -f "$LBS" ] || LBS="$VENV/lib/python3.12/site-packages/smplx/lbs.py" + if [ -f "$LBS" ] && grep -q "torch.einsum('blfi,blf->bli'" "$LBS"; then + echo "==> Patch $LBS (landmarks einsum)" + python3 - "$LBS" <<'PYEOF' +import sys, pathlib +p = pathlib.Path(sys.argv[1]) +s = p.read_text() +s = s.replace( + "landmarks = torch.einsum('blfi,blf->bli', [lmk_vertices, lmk_bary_coords])\n return landmarks", + "# CoreML-friendly: replace einsum('blfi,blf->bli', ...) with broadcast+sum\n landmarks = (lmk_vertices * lmk_bary_coords.unsqueeze(-1)).sum(dim=2)\n return landmarks", +) +p.write_text(s) +print(" smplx/lbs.py patched") +PYEOF + fi +done + echo "Setup OK. Cache : $CACHE" diff --git a/data_only_viz/tests/test_multihmr_coreml.py b/data_only_viz/tests/test_multihmr_coreml.py new file mode 100644 index 0000000..2eb7338 --- /dev/null +++ b/data_only_viz/tests/test_multihmr_coreml.py @@ -0,0 +1,96 @@ +"""Tests for the Multi-HMR CoreML backend. + +Skipped unless the .mlpackage exists at the standard cache path. +""" +from __future__ import annotations + +import time +from pathlib import Path + +import numpy as np +import pytest + +MLPACKAGE = ( + Path.home() / ".cache" / "av-live-multihmr" + / "multihmr_full_672_s.mlpackage" +) + +pytestmark = pytest.mark.skipif( + not MLPACKAGE.exists(), + reason=f"mlpackage missing at {MLPACKAGE}", +) + + +def _make_K() -> np.ndarray: + f = 672.0 + return np.array([[f, 0.0, 336.0], + [0.0, f, 336.0], + [0.0, 0.0, 1.0]], dtype=np.float32) + + +def test_is_available_true(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + assert MultiHMRCoreMLBackend.is_available(MLPACKAGE) is True + + +def test_load_model(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(MLPACKAGE) + assert backend._model is not None + + +def test_infer_random_image_shapes(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(MLPACKAGE) + rng = np.random.default_rng(0) + img = rng.random((3, 672, 672), dtype=np.float32) + K = _make_K() + # threshold = -inf so we get all K=4 humans back + humans = backend.infer(img, K, det_thresh=-1.0) + assert len(humans) == 4 + for h in humans: + v = h["v3d"].detach().cpu().numpy() + assert v.shape == (10475, 3) + assert v.dtype == np.float32 + t = h["transl_pelvis"].detach().cpu().numpy() + assert t.shape == (1, 3) + s = float(h["scores"].item()) + assert isinstance(s, float) + beta = h["shape"].detach().cpu().numpy() + assert beta.shape == (10,) + expr = h["expression"].detach().cpu().numpy() + assert expr.shape == (10,) + + +def test_infer_latency_under_target(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(MLPACKAGE) + K = _make_K() + rng = np.random.default_rng(42) + img = rng.random((3, 672, 672), dtype=np.float32) + # warmup + backend.infer(img, K, det_thresh=-1.0) + # measure + n = 5 + times = [] + for _ in range(n): + t0 = time.monotonic() + backend.infer(img, K, det_thresh=-1.0) + times.append((time.monotonic() - t0) * 1e3) + times.sort() + median_ms = times[n // 2] + print(f"median latency: {median_ms:.1f} ms (n={n})") + # Target 50ms = 20fps. M5 bench shows ~29ms. Generous margin. + assert median_ms < 80.0, f"median {median_ms:.1f}ms > 80ms target" + + +def test_filter_threshold(): + from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(MLPACKAGE) + rng = np.random.default_rng(0) + img = rng.random((3, 672, 672), dtype=np.float32) + K = _make_K() + high = backend.infer(img, K, det_thresh=999.0) + assert high == [] # nothing passes + low = backend.infer(img, K, det_thresh=-1.0) + assert len(low) == 4 From 744bc4a8a469f801d9a505a8140154a409f29333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 21:43:51 +0200 Subject: [PATCH 07/18] feat(data-only-viz): action-head training loop Add train_action_head.py with WindowDataset, class-weighted CrossEntropy, AdamW optimizer, per-epoch train/val loop, and best-val-acc checkpoint saving. Add smoke tests verifying 2-epoch run and checkpoint loadability via ActionHead. - WindowDataset computes position/velocity/accel features inline - _class_weights balances imbalanced label distribution - train() returns history dict (train/val loss and acc) - CLI entry point for --device mps/cuda/cpu production runs --- data_only_viz/tests/test_training_smoke.py | 61 +++++++ data_only_viz/training/train_action_head.py | 180 ++++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 data_only_viz/tests/test_training_smoke.py create mode 100644 data_only_viz/training/train_action_head.py diff --git a/data_only_viz/tests/test_training_smoke.py b/data_only_viz/tests/test_training_smoke.py new file mode 100644 index 0000000..7f9d187 --- /dev/null +++ b/data_only_viz/tests/test_training_smoke.py @@ -0,0 +1,61 @@ +"""Smoke test for action-head training (2 epochs, tiny dataset, CPU).""" +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") + + +def _make_tiny_dataset(tmp_path: Path) -> Path: + from data_only_viz.training.dataset import DatasetRow, write_dataset_jsonl + rng = np.random.default_rng(0) + rows = [] + for sess_i, sess in enumerate(("s01", "s02", "s03")): + for w in range(30): + label = ("debout", "assise", "danse")[w % 3] + rows.append(DatasetRow( + window_id=f"{sess}_w{w:03d}", + label=label, + j3d_stack=rng.normal(size=(16, 22, 3)).astype(np.float32), + session=sess, pid_local=1, + auto_label_confidence=0.8, + manually_validated=True, + )) + out = tmp_path / "tiny.jsonl" + write_dataset_jsonl(rows, out) + return out + + +def test_train_2_epochs_no_crash(tmp_path: Path) -> None: + from data_only_viz.training.train_action_head import train + ds = _make_tiny_dataset(tmp_path) + ckpt = tmp_path / "ckpt.pt" + history = train( + dataset_path=ds, + ckpt_out=ckpt, + epochs=2, + batch_size=8, + lr=1e-3, + device="cpu", + seed=0, + log_every=10_000, + ) + assert ckpt.exists() + assert len(history["train_loss"]) == 2 + assert all(np.isfinite(history["train_loss"])) + + +def test_trained_checkpoint_loadable(tmp_path: Path) -> None: + from data_only_viz.action_head import ActionHead + from data_only_viz.training.train_action_head import train + ds = _make_tiny_dataset(tmp_path) + ckpt = tmp_path / "ckpt.pt" + train(dataset_path=ds, ckpt_out=ckpt, epochs=1, batch_size=8, + lr=1e-3, device="cpu", seed=0, log_every=10_000) + head = ActionHead(ckpt_path=ckpt) + for i in range(5): + label, probs, _ = head.step(pid=1, j3d=np.zeros((22, 3), dtype=np.float32)) + assert abs(float(probs.sum()) - 1.0) < 1e-5 diff --git a/data_only_viz/training/train_action_head.py b/data_only_viz/training/train_action_head.py new file mode 100644 index 0000000..e777146 --- /dev/null +++ b/data_only_viz/training/train_action_head.py @@ -0,0 +1,180 @@ +"""Train ActionHead on the windowed dataset. + +Usage: + uv run python -m data_only_viz.training.train_action_head \ + --dataset ~/.cache/av-live-action/dataset/dataset.jsonl \ + --ckpt-out ~/.cache/av-live-action/checkpoints/action_head.pt \ + --device mps --epochs 50 --batch-size 128 +""" +from __future__ import annotations + +import argparse +import json +import logging +from collections import Counter +from pathlib import Path + +import numpy as np +import torch +from torch import nn +from torch.utils.data import DataLoader, Dataset + +from data_only_viz.action_head import ( + ActionHeadModel, + FeatureExtractor, + HIP_LEFT, + HIP_RIGHT, + LABELS, +) +from data_only_viz.training.augment import random_augment +from data_only_viz.training.dataset import ( + DatasetRow, + load_dataset_jsonl, + split_by_session, +) + +LOG = logging.getLogger("train_action_head") +LABEL_TO_IDX = {l: i for i, l in enumerate(LABELS)} + + +class WindowDataset(Dataset[tuple[torch.Tensor, int]]): + def __init__(self, rows: list[DatasetRow], + augment: bool = False, seed: int = 0) -> None: + self._rows = rows + self._augment = augment + self._rng = np.random.default_rng(seed) + + def __len__(self) -> int: + return len(self._rows) + + def __getitem__(self, idx: int) -> tuple[torch.Tensor, int]: + row = self._rows[idx] + stack = row.j3d_stack + if self._augment: + stack = random_augment(stack, self._rng) + feats = [] + prev = stack[0] + prev_vel = np.zeros_like(prev) + for t in range(stack.shape[0]): + cur = stack[t] + vel = cur - prev + accel = vel - prev_vel + hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5) + knee_angle = FeatureExtractor._mean_knee_angle(cur) + sym = FeatureExtractor._symmetry_score(vel) + feat = np.concatenate([ + cur.reshape(-1), vel.reshape(-1), accel.reshape(-1), + np.array([hip_y, knee_angle, sym], dtype=np.float32), + ]).astype(np.float32, copy=False) + feats.append(feat) + prev_vel = vel + prev = cur + x = torch.from_numpy(np.stack(feats)) + y = LABEL_TO_IDX[row.label] + return x, y + + +def _class_weights(rows: list[DatasetRow]) -> torch.Tensor: + counts = Counter(r.label for r in rows) + total = sum(counts.values()) + weights = torch.tensor([ + total / (len(LABELS) * counts.get(l, 1)) for l in LABELS + ], dtype=torch.float32) + return weights + + +def _run_epoch(model: nn.Module, loader: DataLoader, loss_fn: nn.Module, + optim: torch.optim.Optimizer | None, + device: str) -> tuple[float, float]: + train_mode = optim is not None + model.train(train_mode) + total_loss = 0.0 + correct = 0 + seen = 0 + for x, y in loader: + x = x.to(device) + y = y.to(device) + B, T, _ = x.shape + h = model.init_hidden(batch=B, device=device) + logits_last: torch.Tensor | None = None + for t in range(T): + logits, h = model(x[:, t, :], h) + logits_last = logits + assert logits_last is not None + loss = loss_fn(logits_last, y) + if train_mode: + optim.zero_grad() + loss.backward() + optim.step() + total_loss += float(loss) * B + correct += int((logits_last.argmax(-1) == y).sum()) + seen += B + return total_loss / max(1, seen), correct / max(1, seen) + + +def train(*, + dataset_path: Path, + ckpt_out: Path, + epochs: int = 50, + batch_size: int = 128, + lr: float = 1e-3, + device: str = "cpu", + seed: int = 0, + log_every: int = 1, + ) -> dict[str, list[float]]: + torch.manual_seed(seed) + rows = load_dataset_jsonl(dataset_path) + train_rows, val_rows, _test_rows = split_by_session(rows, seed=seed) + LOG.info("train=%d val=%d", len(train_rows), len(val_rows)) + train_ds = WindowDataset(train_rows, augment=True, seed=seed) + val_ds = WindowDataset(val_rows, augment=False, seed=seed) + train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True) + val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False) + model = ActionHeadModel().to(device) + weights = _class_weights(train_rows).to(device) + loss_fn = nn.CrossEntropyLoss(weight=weights) + optim = torch.optim.AdamW(model.parameters(), lr=lr) + history: dict[str, list[float]] = { + "train_loss": [], "train_acc": [], "val_loss": [], "val_acc": [], + } + best_val_acc = -1.0 + ckpt_out.parent.mkdir(parents=True, exist_ok=True) + for ep in range(epochs): + tl, ta = _run_epoch(model, train_loader, loss_fn, optim, device) + with torch.no_grad(): + vl, va = _run_epoch(model, val_loader, loss_fn, None, device) + history["train_loss"].append(tl) + history["train_acc"].append(ta) + history["val_loss"].append(vl) + history["val_acc"].append(va) + if ep % log_every == 0 or ep == epochs - 1: + LOG.info("ep=%d train_loss=%.4f train_acc=%.3f val_loss=%.4f val_acc=%.3f", + ep, tl, ta, vl, va) + if va > best_val_acc: + best_val_acc = va + torch.save({"model_state_dict": model.state_dict(), + "version": 1, "val_acc": va}, ckpt_out) + return history + + +def _cli() -> None: + p = argparse.ArgumentParser() + p.add_argument("--dataset", required=True, type=Path) + p.add_argument("--ckpt-out", required=True, type=Path) + p.add_argument("--epochs", type=int, default=50) + p.add_argument("--batch-size", type=int, default=128) + p.add_argument("--lr", type=float, default=1e-3) + p.add_argument("--device", default="cpu", + choices=["cpu", "mps", "cuda"]) + p.add_argument("--seed", type=int, default=0) + args = p.parse_args() + logging.basicConfig(level=logging.INFO, + format="%(asctime)s [%(name)s] %(message)s") + hist = train(dataset_path=args.dataset, ckpt_out=args.ckpt_out, + epochs=args.epochs, batch_size=args.batch_size, + lr=args.lr, device=args.device, seed=args.seed) + print(json.dumps({"final": {k: v[-1] for k, v in hist.items()}})) + + +if __name__ == "__main__": + _cli() From 9d67426b2cc8c58f0f08c60943f55fba20883c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:05:15 +0200 Subject: [PATCH 08/18] feat(data-only-viz): action-head eval script Add evaluation script to compute test accuracy, confusion matrix, and inference latency on a trained action-head checkpoint. Reuses existing WindowDataset and model infrastructure from training pipeline. Falls back to evaluating on full dataset if test split is empty (edge case with <4 sessions). --- data_only_viz/training/eval.py | 87 ++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 data_only_viz/training/eval.py diff --git a/data_only_viz/training/eval.py b/data_only_viz/training/eval.py new file mode 100644 index 0000000..c711187 --- /dev/null +++ b/data_only_viz/training/eval.py @@ -0,0 +1,87 @@ +"""Evaluate a trained action-head checkpoint.""" +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import numpy as np +import torch +from torch.utils.data import DataLoader + +from data_only_viz.action_head import ActionHeadModel, LABELS +from data_only_viz.training.dataset import load_dataset_jsonl, split_by_session +from data_only_viz.training.train_action_head import ( + LABEL_TO_IDX, + WindowDataset, +) + + +def confusion_matrix(true: list[int], pred: list[int], + num_classes: int = 3) -> np.ndarray: + cm = np.zeros((num_classes, num_classes), dtype=np.int64) + for t, p in zip(true, pred): + cm[t, p] += 1 + return cm + + +def evaluate(ckpt_path: Path, dataset_path: Path, device: str = "cpu", + seed: int = 0) -> dict: + rows = load_dataset_jsonl(dataset_path) + _train, _val, test_rows = split_by_session(rows, seed=seed) + if not test_rows: + test_rows = rows + ds = WindowDataset(test_rows, augment=False, seed=seed) + loader = DataLoader(ds, batch_size=64, shuffle=False) + model = ActionHeadModel().to(device).eval() + payload = torch.load(ckpt_path, map_location=device, weights_only=True) + model.load_state_dict(payload["model_state_dict"]) + true: list[int] = [] + pred: list[int] = [] + with torch.no_grad(): + for x, y in loader: + x = x.to(device); y = y.to(device) + B, T, _ = x.shape + h = model.init_hidden(batch=B, device=device) + logits = None + for t in range(T): + logits, h = model(x[:, t, :], h) + true.extend(y.cpu().tolist()) + pred.extend(logits.argmax(-1).cpu().tolist()) + cm = confusion_matrix(true, pred) + acc = float(np.trace(cm) / max(1, cm.sum())) + confusion_db = float((cm[0, 2] + cm[2, 0]) / max(1, cm.sum())) + feat_dim = ds[0][0].shape[-1] + bench_x = torch.zeros(1, feat_dim, device=device) + h = model.init_hidden(batch=1, device=device) + for _ in range(20): + _ = model(bench_x, h) + t0 = time.perf_counter() + N = 500 + for _ in range(N): + _, h = model(bench_x, h) + lat_ms = (time.perf_counter() - t0) * 1000.0 / N + return { + "test_acc": acc, + "confusion_debout_danse": confusion_db, + "confusion_matrix": cm.tolist(), + "labels": list(LABELS), + "step_latency_ms": lat_ms, + "n_test": int(cm.sum()), + } + + +def _cli() -> None: + p = argparse.ArgumentParser() + p.add_argument("--ckpt", required=True, type=Path) + p.add_argument("--dataset", required=True, type=Path) + p.add_argument("--device", default="cpu", + choices=["cpu", "mps", "cuda"]) + args = p.parse_args() + out = evaluate(args.ckpt, args.dataset, device=args.device) + print(json.dumps(out, indent=2)) + + +if __name__ == "__main__": + _cli() From e292ed7ef3046a7e37d7b8ba9f909a5f1ed5c9ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:07:34 +0200 Subject: [PATCH 09/18] feat(data-only-viz): studio train wrapper Wrapper bash : rsync dataset+code grosmac->studio via bastion electron-server, exec uv run train_action_head --device mps sur M3 Ultra, rsync checkpoint back. SSH direct cassee depuis reboot studio 2026-05-12 ; route via bastion documentee dans CLAUDE.md. --- data_only_viz/scripts/train_on_studio.sh | 84 ++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 data_only_viz/scripts/train_on_studio.sh diff --git a/data_only_viz/scripts/train_on_studio.sh b/data_only_viz/scripts/train_on_studio.sh new file mode 100755 index 0000000..745a002 --- /dev/null +++ b/data_only_viz/scripts/train_on_studio.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Train action-head on MacStudio M3 Ultra (Tailscale 100.116.92.12). +# +# SSH direct grosmac→studio is broken since reboot 2026-05-12 ; +# we route via electron-server bastion (cf. CLAUDE.md root). +# +# Usage: +# ./train_on_studio.sh # uses defaults +# ./train_on_studio.sh --epochs 80 --lr 5e-4 +# +# Local layout : +# ~/.cache/av-live-action/dataset/dataset.jsonl (input) +# ~/.cache/av-live-action/checkpoints/ (output, after rsync back) +# +# Remote layout : +# studio:~/av-live-action/repo/ (rsynced code subset) +# studio:~/av-live-action/dataset/ (rsynced dataset) +# studio:~/av-live-action/checkpoints/ (training output) + +set -euo pipefail + +BASTION_USER_HOST="${BASTION_USER_HOST:-electron-server}" +STUDIO_USER_HOST="${STUDIO_USER_HOST:-clems@100.116.92.12}" +STUDIO_UV="${STUDIO_UV:-/opt/homebrew/bin/uv}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)" +LOCAL_CACHE="$HOME/.cache/av-live-action" +LOCAL_DATASET="$LOCAL_CACHE/dataset" +LOCAL_CKPT="$LOCAL_CACHE/checkpoints" + +REMOTE_ROOT="\$HOME/av-live-action" +REMOTE_REPO="$REMOTE_ROOT/repo" +REMOTE_DATASET="$REMOTE_ROOT/dataset" +REMOTE_CKPT="$REMOTE_ROOT/checkpoints" + +DATASET_FILE="${DATASET_FILE:-$LOCAL_DATASET/dataset.jsonl}" +CKPT_NAME="${CKPT_NAME:-action_head.pt}" +TRAIN_ARGS="$*" + +log() { printf '[train_on_studio] %s\n' "$*" >&2; } + +[[ -f "$DATASET_FILE" ]] || { log "missing dataset: $DATASET_FILE"; exit 2; } +mkdir -p "$LOCAL_CKPT" + +bastion_ssh() { + ssh -o ConnectTimeout=5 "$BASTION_USER_HOST" \ + "ssh -o ConnectTimeout=5 $STUDIO_USER_HOST $*" +} + +bastion_rsync() { + # rsync via ssh ProxyCommand through bastion. + local src="$1" dst="$2" + rsync -avz --delete \ + -e "ssh -o ConnectTimeout=5 -A -J $BASTION_USER_HOST" \ + "$src" "$dst" +} + +log "== Studio reachability ==" +bastion_ssh "echo studio OK ; $STUDIO_UV --version" + +log "== Push code subset ==" +bastion_ssh "mkdir -p $REMOTE_REPO/data_only_viz $REMOTE_DATASET $REMOTE_CKPT" +bastion_rsync "$REPO_ROOT/data_only_viz/" \ + "$STUDIO_USER_HOST:av-live-action/repo/data_only_viz/" + +log "== Push dataset ==" +bastion_rsync "$LOCAL_DATASET/" "$STUDIO_USER_HOST:av-live-action/dataset/" + +log "== Remote uv sync ==" +bastion_ssh "cd $REMOTE_REPO/data_only_viz && $STUDIO_UV sync --no-progress" + +log "== Remote train (MPS) ==" +bastion_ssh "cd $REMOTE_REPO/data_only_viz && \ + $STUDIO_UV run python -m data_only_viz.training.train_action_head \ + --dataset $REMOTE_DATASET/$(basename "$DATASET_FILE") \ + --ckpt-out $REMOTE_CKPT/$CKPT_NAME \ + --device mps \ + $TRAIN_ARGS" + +log "== Pull checkpoint back ==" +bastion_rsync "$STUDIO_USER_HOST:av-live-action/checkpoints/" "$LOCAL_CKPT/" + +log "== Done. Checkpoint: $LOCAL_CKPT/$CKPT_NAME ==" +ls -la "$LOCAL_CKPT/$CKPT_NAME" From c381d0c4e76d3c190c4e9176b1b82c561c5fe6ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:18:08 +0200 Subject: [PATCH 10/18] feat(data-only-viz): pose_bridge action+kin OSC Ajoute 4 methodes OSC pour action_head et localisation cinetique: send_action(pid, label_idx, probs, t_now, force) envoie /pose/action avec [pid, label_idx, prob_0, prob_1, prob_2] pour les 3 classes. send_kin(pid, kin, t_now, force) envoie /pose/kin avec [pid, kin[0], kin[1], kin[2]] pour angles de poignets/coudes. send_enter/send_leave envoient /pose/enter et /pose/leave pour cycle vie des personnes. Throttle reuse _period/_last_t existants; force=True bypass throttle. --- data_only_viz/pose_bridge.py | 32 ++++++++++++ .../tests/test_pose_bridge_action.py | 49 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 data_only_viz/tests/test_pose_bridge_action.py diff --git a/data_only_viz/pose_bridge.py b/data_only_viz/pose_bridge.py index fa9a221..7cc6807 100644 --- a/data_only_viz/pose_bridge.py +++ b/data_only_viz/pose_bridge.py @@ -113,3 +113,35 @@ class PoseSoundBridge: cli.send_message("/pose/limb_span", [pid, float(span)]) try: self._avbody.send_message("/pose/limb_span", [pid, float(span)]) except OSError: pass + + def send_action(self, pid: int, label_idx: int, + probs, t_now: float, force: bool = False) -> None: + """Send action classification result via /pose/action OSC route. + + Sends: [pid (int), label_idx (int), prob_0 (float), prob_1 (float), prob_2 (float)] + """ + if not force and (t_now - self._last_t) < self._period: + return + p = [float(probs[0]), float(probs[1]), float(probs[2])] + self._client.send_message("/pose/action", [int(pid), int(label_idx), *p]) + + def send_kin(self, pid: int, kin, + t_now: float, force: bool = False) -> None: + """Send kinematic angles via /pose/kin OSC route. + + Sends: [pid (int), kin_0 (float), kin_1 (float), kin_2 (float)] + """ + if not force and (t_now - self._last_t) < self._period: + return + self._client.send_message( + "/pose/kin", + [int(pid), float(kin[0]), float(kin[1]), float(kin[2])], + ) + + def send_enter(self, pid: int) -> None: + """Send lifecycle event when person enters frame.""" + self._client.send_message("/pose/enter", [int(pid)]) + + def send_leave(self, pid: int) -> None: + """Send lifecycle event when person leaves frame.""" + self._client.send_message("/pose/leave", [int(pid)]) diff --git a/data_only_viz/tests/test_pose_bridge_action.py b/data_only_viz/tests/test_pose_bridge_action.py new file mode 100644 index 0000000..c8cd185 --- /dev/null +++ b/data_only_viz/tests/test_pose_bridge_action.py @@ -0,0 +1,49 @@ +"""Tests for /pose/action and /pose/kin OSC routes.""" +from __future__ import annotations + +from unittest.mock import MagicMock + +import numpy as np + + +def test_send_action_formats_5_args() -> None: + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b.send_action(pid=7, label_idx=2, + probs=np.array([0.1, 0.2, 0.7], dtype=np.float32), + t_now=0.0, force=True) + b._client.send_message.assert_called_once() + address, args = b._client.send_message.call_args.args + assert address == "/pose/action" + assert args[0] == 7 + assert args[1] == 2 + assert all(isinstance(v, float) for v in args[2:5]) + assert abs(sum(args[2:5]) - 1.0) < 1e-5 + + +def test_send_kin_formats_4_args() -> None: + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b.send_kin(pid=3, kin=np.array([0.5, 1.2, -0.3], dtype=np.float32), + t_now=0.0, force=True) + b._client.send_message.assert_called_once() + address, args = b._client.send_message.call_args.args + assert address == "/pose/kin" + assert args[0] == 3 + assert len(args) == 4 + assert abs(args[1] - 0.5) < 1e-6 + assert abs(args[2] - 1.2) < 1e-6 + assert abs(args[3] - (-0.3)) < 1e-6 + + +def test_send_lifecycle_enter_leave() -> None: + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b.send_enter(pid=4) + b.send_leave(pid=4) + calls = [c.args[0] for c in b._client.send_message.call_args_list] + assert "/pose/enter" in calls + assert "/pose/leave" in calls From dfafd23d5a9f193e65b87bd0a3cd7e0f284a916a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:23:17 +0200 Subject: [PATCH 11/18] feat(sound-algo): action-head pose OSC handlers Add OSCdef handlers for /pose/action, /pose/kin, /pose/enter, /pose/leave routes emitted by data_only_viz pose_bridge. Store person state and kinematics in ~poseState and ~poseKin dicts. --- sound_algo/control/data_feeds.scd | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/sound_algo/control/data_feeds.scd b/sound_algo/control/data_feeds.scd index 586d715..c4a52f6 100644 --- a/sound_algo/control/data_feeds.scd +++ b/sound_algo/control/data_feeds.scd @@ -294,6 +294,43 @@ ("[feeds] heartbeat: " ++ if(~feedAlive.value) { "ALIVE" } { "DOWN" }).postln; }; +// ===================================================================== +// ACTION-HEAD OSC handlers (/pose/action, /pose/kin, /pose/enter, /pose/leave) +// ===================================================================== +~poseState = ~poseState ? Dictionary.new; +~poseKin = ~poseKin ? Dictionary.new; + +OSCdef(\poseAction, { |msg| + var pid = msg[1]; + ~poseState[pid] = ( + labelIdx: msg[2], + probs: [msg[3], msg[4], msg[5]], + ); +}, '/pose/action'); + +OSCdef(\poseKin, { |msg| + var pid = msg[1]; + ~poseKin[pid] = ( + speed: msg[2], + accel: msg[3], + symmetry: msg[4], + ); +}, '/pose/kin'); + +OSCdef(\poseEnter, { |msg| + var pid = msg[1]; + ~poseState[pid] = (labelIdx: 0, probs: [1.0, 0.0, 0.0]); + ~poseKin[pid] = (speed: 0, accel: 0, symmetry: 0); +}, '/pose/enter'); + +OSCdef(\poseLeave, { |msg| + var pid = msg[1]; + ~poseState.removeAt(pid); + ~poseKin.removeAt(pid); +}, '/pose/leave'); + +"[OK] action-head OSC handlers".postln; + "[data_feeds] OSCdef installes (USGS, SWPC, NETZ, RTE, BLITZ, OPENSKY, BSKY, MEMPOOL, GCN).".postln; "[data_feeds] usage : ~feeds[\\swpc_kp], ~feedGet.(\\netz_dev, 0), ~feedDump.()".postln; ) From a1ea343ff0a619d2f17aa47d3bfe6315fdc4c979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:24:24 +0200 Subject: [PATCH 12/18] fix(data-only-viz): studio ssh quoting + abs paths Le precedent printf %q sur-quotait $HOME -> mkdir recevait 0 arg. On utilise des chemins absolus /Users/clems/av-live-action/* cote studio et single-quotes pour proteger des bastion expansions. --- data_only_viz/scripts/train_on_studio.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/data_only_viz/scripts/train_on_studio.sh b/data_only_viz/scripts/train_on_studio.sh index 745a002..e9f96f7 100755 --- a/data_only_viz/scripts/train_on_studio.sh +++ b/data_only_viz/scripts/train_on_studio.sh @@ -21,6 +21,7 @@ set -euo pipefail BASTION_USER_HOST="${BASTION_USER_HOST:-electron-server}" STUDIO_USER_HOST="${STUDIO_USER_HOST:-clems@100.116.92.12}" +STUDIO_USER="${STUDIO_USER:-clems}" STUDIO_UV="${STUDIO_UV:-/opt/homebrew/bin/uv}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)" @@ -28,7 +29,7 @@ LOCAL_CACHE="$HOME/.cache/av-live-action" LOCAL_DATASET="$LOCAL_CACHE/dataset" LOCAL_CKPT="$LOCAL_CACHE/checkpoints" -REMOTE_ROOT="\$HOME/av-live-action" +REMOTE_ROOT="/Users/${STUDIO_USER}/av-live-action" REMOTE_REPO="$REMOTE_ROOT/repo" REMOTE_DATASET="$REMOTE_ROOT/dataset" REMOTE_CKPT="$REMOTE_ROOT/checkpoints" @@ -43,8 +44,13 @@ log() { printf '[train_on_studio] %s\n' "$*" >&2; } mkdir -p "$LOCAL_CKPT" bastion_ssh() { + # The remote shell on the bastion must receive the studio command + # as a single argument, otherwise `;` and `&&` are parsed + # bastion-side instead of studio-side. + # All paths in commands MUST be absolute (no $HOME, no ~) since + # we use single-quotes for the studio-side payload. ssh -o ConnectTimeout=5 "$BASTION_USER_HOST" \ - "ssh -o ConnectTimeout=5 $STUDIO_USER_HOST $*" + "ssh -o ConnectTimeout=5 $STUDIO_USER_HOST '$*'" } bastion_rsync() { From 0ecb2c3d3b40f32625a6974b2a4bacfee1ed7830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:32:14 +0200 Subject: [PATCH 13/18] fix(data-only-viz): studio rsync excludes + extras rsync excludes .venv/__pycache__/.pytest_cache + uv sync ajoute extra multihmr (torch). End-to-end valide: smoke 160 windows 3 epochs MPS studio en ~4s, ckpt rsync back OK. --- data_only_viz/scripts/train_on_studio.sh | 25 +++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/data_only_viz/scripts/train_on_studio.sh b/data_only_viz/scripts/train_on_studio.sh index e9f96f7..def7bcd 100755 --- a/data_only_viz/scripts/train_on_studio.sh +++ b/data_only_viz/scripts/train_on_studio.sh @@ -54,10 +54,12 @@ bastion_ssh() { } bastion_rsync() { - # rsync via ssh ProxyCommand through bastion. + # rsync via ssh ProxyJump through bastion. Direct grosmac->studio + # known_hosts entry may be stale (SSH direct broken since reboot + # 2026-05-12). accept-new lets us add the key on first use. local src="$1" dst="$2" rsync -avz --delete \ - -e "ssh -o ConnectTimeout=5 -A -J $BASTION_USER_HOST" \ + -e "ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -A -J $BASTION_USER_HOST" \ "$src" "$dst" } @@ -66,18 +68,27 @@ bastion_ssh "echo studio OK ; $STUDIO_UV --version" log "== Push code subset ==" bastion_ssh "mkdir -p $REMOTE_REPO/data_only_viz $REMOTE_DATASET $REMOTE_CKPT" -bastion_rsync "$REPO_ROOT/data_only_viz/" \ - "$STUDIO_USER_HOST:av-live-action/repo/data_only_viz/" +rsync -avz --delete \ + --exclude='.venv/' --exclude='__pycache__/' --exclude='.pytest_cache/' \ + --exclude='.ruff_cache/' --exclude='*.pyc' --exclude='.DS_Store' \ + --exclude='web/' --exclude='shaders/' \ + -e "ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -A -J $BASTION_USER_HOST" \ + "$REPO_ROOT/data_only_viz/" \ + "$STUDIO_USER_HOST:av-live-action/repo/data_only_viz/" log "== Push dataset ==" bastion_rsync "$LOCAL_DATASET/" "$STUDIO_USER_HOST:av-live-action/dataset/" log "== Remote uv sync ==" -bastion_ssh "cd $REMOTE_REPO/data_only_viz && $STUDIO_UV sync --no-progress" +# multihmr extra pulls torch (action-head training needs torch but no pyobjc). +# We piggy-back on the multihmr extras since torch is the main thing we need. +bastion_ssh "cd $REMOTE_REPO && $STUDIO_UV sync --no-progress --project data_only_viz --extra multihmr" log "== Remote train (MPS) ==" -bastion_ssh "cd $REMOTE_REPO/data_only_viz && \ - $STUDIO_UV run python -m data_only_viz.training.train_action_head \ +# cwd must be the PARENT of data_only_viz/ so the package is importable as +# top-level. uv resolves the env via --project data_only_viz. +bastion_ssh "cd $REMOTE_REPO && \ + $STUDIO_UV run --project data_only_viz python -m data_only_viz.training.train_action_head \ --dataset $REMOTE_DATASET/$(basename "$DATASET_FILE") \ --ckpt-out $REMOTE_CKPT/$CKPT_NAME \ --device mps \ From b53c74870489fd2acc3d7bb1a876b6cd5af4ce63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:34:26 +0200 Subject: [PATCH 14/18] feat(data-only-viz): action-head review TUI Add interactive console TUI for manual label review of auto-labeled action datasets. Displays ASCII skeleton, kinetics, and proposed label with confidence. User can accept proposed label, choose manual override (1/2/3), skip, or quit. Reads auto-labeled JSONL and writes validated rows to reviewed dataset. --- data_only_viz/training/review.py | 116 +++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 data_only_viz/training/review.py diff --git a/data_only_viz/training/review.py b/data_only_viz/training/review.py new file mode 100644 index 0000000..2cbb848 --- /dev/null +++ b/data_only_viz/training/review.py @@ -0,0 +1,116 @@ +"""Manual label review TUI. + +Reads an auto-labeled jsonl dataset, presents each window with: + - ASCII skeleton (front view) of last frame + - speed/accel/sym kinetics + - proposed label + confidence +Keys: + 1 = debout, 2 = assise, 3 = danse + ENTER = accept proposed label + S = skip (label = None, will not be saved) + Q = quit and write what we have so far + +Usage: + uv run python -m data_only_viz.training.review \\ + --in ~/.cache/av-live-action/dataset/auto.jsonl \\ + --out ~/.cache/av-live-action/dataset/reviewed.jsonl +""" +from __future__ import annotations + +import argparse +import sys +import termios +import tty +from pathlib import Path + +import numpy as np + +from data_only_viz.action_head import LABELS +from data_only_viz.training.autolabel import autolabel_window +from data_only_viz.training.dataset import ( + DatasetRow, + load_dataset_jsonl, + write_dataset_jsonl, +) + + +def _ascii_skeleton(j3d: np.ndarray, width: int = 40, height: int = 16) -> str: + pts = j3d[:, [0, 1]] # x, y + mn = pts.min(axis=0) + mx = pts.max(axis=0) + rng = np.maximum(mx - mn, 1e-3) + norm = (pts - mn) / rng + grid = [[" "] * width for _ in range(height)] + for x, y in norm: + col = int(x * (width - 1)) + row = int((1 - y) * (height - 1)) + grid[row][col] = "*" + return "\n".join("".join(row) for row in grid) + + +def _getch() -> str: + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + try: + tty.setraw(fd) + return sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old) + + +def review(in_path: Path, out_path: Path, + sample_validated_fraction: float = 0.2, + seed: int = 0) -> None: + from data_only_viz.action_head import FeatureExtractor + + rows = load_dataset_jsonl(in_path) + rng = np.random.default_rng(seed) + kept: list[DatasetRow] = [] + for i, r in enumerate(rows): + proposed, conf = autolabel_window(list(r.j3d_stack)) + is_none = proposed is None + sampled = rng.random() < sample_validated_fraction + if not is_none and not sampled and r.manually_validated: + kept.append(r) + continue + print("\033[2J\033[H") # clear + print(f"[{i + 1}/{len(rows)}] {r.window_id} proposed={proposed} conf={conf:.2f}") + print(_ascii_skeleton(r.j3d_stack[-1])) + kin = FeatureExtractor.kinetics(list(r.j3d_stack)) + print(f"speed={kin[0]:.3f} accel={kin[1]:.3f} sym={kin[2]:+.3f}") + print("keys: 1=debout 2=assise 3=danse ENTER=accept S=skip Q=quit") + k = _getch().lower() + if k == "q": + break + if k == "s": + continue + if k == "\r": + chosen = proposed + elif k in ("1", "2", "3"): + chosen = LABELS[int(k) - 1] + else: + continue + if chosen is None: + continue + kept.append(DatasetRow( + window_id=r.window_id, label=chosen, j3d_stack=r.j3d_stack, + session=r.session, pid_local=r.pid_local, + auto_label_confidence=conf, manually_validated=True, + )) + out_path.parent.mkdir(parents=True, exist_ok=True) + write_dataset_jsonl(kept, out_path) + print(f"\nwrote {len(kept)} rows to {out_path}") + + +def _cli() -> None: + p = argparse.ArgumentParser() + p.add_argument("--in", dest="in_path", required=True, type=Path) + p.add_argument("--out", dest="out_path", required=True, type=Path) + p.add_argument("--sample-fraction", type=float, default=0.2) + args = p.parse_args() + review(args.in_path, args.out_path, + sample_validated_fraction=args.sample_fraction) + + +if __name__ == "__main__": + _cli() From f540158f45feb739d3995704ef5eed89fab58a75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:34:42 +0200 Subject: [PATCH 15/18] feat(av-live): face+hand+3D pose to launcher Stream MediaPipe Holistic face landmarks (68 dlib subset of 478), hand landmarks (21 left + 21 right), and pose world landmarks (33 3D xyz meters) over OSC :57126 to AVLiveBody. Launcher renders face/hand as SwiftUI Canvas overlay and the 3D skeleton as a RealityKit armature (sphere joints + cylinder bones, color per chain) toggled via p / mode openpos. Multi-HMR worker now also starts MediaPipe Multi in parallel so both the dense SMPL-X mesh (TCP 57130, PyTorch backend) and the skeleton/face/hand streams (OSC 57126) feed the launcher from one Python process. Launcher AppDelegate forces .regular activation so SwiftPM binaries actually show their WindowGroup without a bundle. Tests: 9 new pytest cases (4 body3d + 5 face/hand), all green. CoreML conversion still produces NaN on v3d/transl; PyTorch backend is the working path for now. --- data_only_viz/main.py | 15 ++ data_only_viz/multi.py | 24 +- data_only_viz/pose_bridge.py | 197 +++++++++++++++- data_only_viz/scripts/coreml_full_probe.py | 3 + data_only_viz/state.py | 14 ++ .../tests/test_pose_bridge_body3d.py | 79 +++++++ .../tests/test_pose_bridge_face_hand.py | 105 +++++++++ .../Sources/AVLiveBody/AVLiveBodyApp.swift | 23 +- .../Sources/AVLiveBody/BodyView.swift | 14 ++ .../Sources/AVLiveBody/FaceHandOverlay.swift | 155 ++++++++++++ .../Sources/AVLiveBody/PoseOSCListener.swift | 103 ++++++++ .../AVLiveBody/Skeleton3DRenderer.swift | 222 ++++++++++++++++++ 12 files changed, 943 insertions(+), 11 deletions(-) create mode 100644 data_only_viz/tests/test_pose_bridge_body3d.py create mode 100644 data_only_viz/tests/test_pose_bridge_face_hand.py create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift diff --git a/data_only_viz/main.py b/data_only_viz/main.py index a81423b..263e0bb 100644 --- a/data_only_viz/main.py +++ b/data_only_viz/main.py @@ -268,6 +268,21 @@ class AppDelegate(NSObject): self._smplx_tcp = SMPLXTCPSender(self._state) self._smplx_tcp.start() LOG.info("worker: Multi-HMR + SMPL-X (mesh dense)") + # Also start MediaPipe Multi for body3d + face + hand + # OSC streams to AVLiveBody (mesh and skeleton/face/ + # hand pipelines run in parallel, each owns its own + # AVCapture session on the same builtin camera). + if _os.environ.get("AV_LIVE_MEDIAPIPE") != "0": + try: + from .multi import MultiWorker + self._mediapipe_worker = MultiWorker( + self._state, num_persons=4) + self._mediapipe_worker.start() + LOG.info("worker: + MediaPipe Multi (3D pose " + "+ face + hand) in parallel") + except Exception as e: # noqa: BLE001 + LOG.warning("MediaPipe parallel start failed " + "(%s) — mesh only", e) return LOG.info("Multi-HMR indisponible (checkpoints manquants) " "— voir scripts/setup_multihmr.sh") diff --git a/data_only_viz/multi.py b/data_only_viz/multi.py index bd119b5..0c7beaa 100644 --- a/data_only_viz/multi.py +++ b/data_only_viz/multi.py @@ -21,7 +21,7 @@ from pathlib import Path from .euro_filter import SkeletonFilter from .pose_bridge import PoseSoundBridge -from .state import PoseKp, State +from .state import Kp3D, PoseKp, State from .tracker import IoUTracker LOG = logging.getLogger("multi") @@ -198,6 +198,21 @@ class MultiWorker: x=float(lm.x), y=float(lm.y), z=z, c=float(v))) bodies.append(kp_list) + # pose_world_landmarks : xyz metric, relative to hip-center. + # Aligned 1:1 with pose_landmarks order. Empty fallback if + # the MediaPipe build doesn't populate it. + bodies3d: list[list[Kp3D]] = [] + world_list = getattr(pose_res, "pose_world_landmarks", None) or [] + for landmarks_list in world_list: + kp3_list: list[Kp3D] = [] + for lm in landmarks_list[:33]: + v = lm.visibility if lm.visibility is not None else 1.0 + kp3_list.append(Kp3D( + x=float(lm.x), y=float(lm.y), + z=float(lm.z if lm.z is not None else 0.0), + c=float(v))) + bodies3d.append(kp3_list) + faces = [] for landmarks_list in (face_res.face_landmarks or []): kp_list = [] @@ -230,16 +245,21 @@ class MultiWorker: for i, kps in enumerate(hands)] # Pont sonore : envoi OSC /pose/* a sclang (body + face + hands) + # 3D world landmarks share ids with bodies (same MediaPipe + # detection, just a different coordinate space). + ids_body3d = ids_body[:len(bodies3d)] if bodies3d else [] self._sound_bridge.send( bodies, ids_body, t_now, persons_face=faces, persons_face_ids=ids_face, - persons_hands=hands, persons_hands_ids=ids_hand) + persons_hands=hands, persons_hands_ids=ids_hand, + persons_body3d=bodies3d, persons_body3d_ids=ids_body3d) with self.state.lock(): self.state.persons_body = bodies self.state.persons_face = faces self.state.persons_hands = hands self.state.persons_body_ids = ids_body + self.state.persons_body3d = bodies3d self.state.persons_face_ids = ids_face self.state.persons_hands_ids = ids_hand # Compat single-person (1ere personne) diff --git a/data_only_viz/pose_bridge.py b/data_only_viz/pose_bridge.py index 7cc6807..4c5a08a 100644 --- a/data_only_viz/pose_bridge.py +++ b/data_only_viz/pose_bridge.py @@ -10,13 +10,21 @@ Routes emises : /pose/head position du nez (visage) /pose/sho_span ecart epaules (estime distance camera) /pose/limb_span envergure brassse (poignet a poignet) + /face/count nombre de visages detectes + /face/kp 68-pt subset (dlib mapping) + /hand/count nombre de mains gauche / droite + /hand/kp 21 landmarks + /pose3d/count nombre de squelettes 3D + /pose3d/kp 33 MediaPipe world landmarks (metres) Mapping pose -> son est defini cote SC dans sound_algo/data_only/scenes.scd -(scene `live_pose`). +(scene `live_pose`). Face / hand keypoints are consumed by the Swift +launcher (AVLiveBody) on 127.0.0.1:57126 for skeleton overlay rendering. """ from __future__ import annotations import logging +from typing import Any, Iterable, Sequence from pythonosc.udp_client import SimpleUDPClient @@ -33,6 +41,42 @@ LEFT_HIP = 23 RIGHT_HIP = 24 +# MediaPipe FaceMesh (468 landmarks) -> 68-point dlib-style subset. +# Mapping inspired by community references (Google MediaPipe -> +# iBUG 68 facial landmarks). Order matches dlib 68-point convention : +# [0..16] jaw contour (left to right) +# [17..21] right brow +# [22..26] left brow +# [27..30] nose bridge (top to tip) +# [31..35] nostril base (right to left) +# [36..41] right eye (CCW from outer corner) +# [42..47] left eye (CCW from inner corner) +# [48..59] outer lip (CCW from right corner) +# [60..67] inner lip (CCW from right corner) +FACE_68_FROM_MP: tuple[int, ...] = ( + # Jaw (17) + 127, 234, 132, 172, 150, 176, 148, 152, + 377, 400, 365, 397, 361, 401, 366, 447, 356, + # Right brow (5) — mediapipe perspective is mirrored vs subject + 70, 63, 105, 66, 107, + # Left brow (5) + 336, 296, 334, 293, 300, + # Nose bridge (4) + 168, 6, 197, 195, + # Nostril base (5) + 98, 97, 2, 326, 327, + # Right eye (6) + 33, 160, 158, 133, 153, 144, + # Left eye (6) + 362, 385, 387, 263, 373, 380, + # Outer lip (12) + 61, 39, 37, 0, 267, 269, 291, 405, 314, 17, 84, 181, + # Inner lip (8) + 78, 81, 13, 311, 308, 402, 14, 178, +) +assert len(FACE_68_FROM_MP) == 68 + + class PoseSoundBridge: """Envoie les keypoints en OSC vers sclang. Throttle a 30 Hz max.""" @@ -45,9 +89,17 @@ class PoseSoundBridge: self._period = 1.0 / max(1.0, throttle_hz) self._last_t = 0.0 - def send(self, persons_body: list, persons_body_ids: list, t_now: float) -> None: + def send(self, persons_body: list, persons_body_ids: list, t_now: float, + *, + persons_face: Sequence[Sequence[Any]] | None = None, + persons_face_ids: Sequence[int] | None = None, + persons_hands: Sequence[Sequence[Any]] | None = None, + persons_hands_ids: Sequence[int] | None = None, + persons_body3d: Sequence[Sequence[Any]] | None = None, + persons_body3d_ids: Sequence[int] | None = None) -> None: """Envoie les keypoints de toutes les personnes detectees. - Throttle automatiquement.""" + Throttle automatiquement. Face / hand sont optionnels et envoyes + sur le meme socket :57126 vers AVLiveBody.""" if t_now - self._last_t < self._period: return self._last_t = t_now @@ -59,12 +111,20 @@ class PoseSoundBridge: except OSError: pass except OSError: return # SC pas la, on continue silencieusement - if n == 0: - return - for i, body in enumerate(persons_body): - pid = persons_body_ids[i] if i < len(persons_body_ids) else i - self._emit_person(int(pid), body) + if n > 0: + for i, body in enumerate(persons_body): + pid = persons_body_ids[i] if i < len(persons_body_ids) else i + self._emit_person(int(pid), body) + + # Face / hand : independant de la presence de body kp (utile en + # mode face-only ou hand-only). + if persons_face is not None: + self._send_face(persons_face, persons_face_ids or []) + if persons_hands is not None: + self._send_hand(persons_hands, persons_hands_ids or []) + if persons_body3d is not None: + self._send_body3d(persons_body3d, persons_body3d_ids or []) # ------------------------------------------------------------------ def _emit_person(self, pid: int, body: list) -> None: @@ -114,6 +174,127 @@ class PoseSoundBridge: try: self._avbody.send_message("/pose/limb_span", [pid, float(span)]) except OSError: pass + # ------------------------------------------------------------------ + def send_face(self, persons_face: Sequence[Sequence[Any]], + persons_face_ids: Sequence[int], t_now: float, + force: bool = False) -> None: + """Public throttled entry point for face keypoints. + + Emits a 68-point dlib-style subset of the 468 MediaPipe FaceMesh + landmarks per person on /face/count + /face/kp routes. + """ + if not force and (t_now - self._last_t) < self._period: + return + self._send_face(persons_face, persons_face_ids) + + def send_hand(self, persons_hands: Sequence[Sequence[Any]], + persons_hands_ids: Sequence[int], t_now: float, + force: bool = False) -> None: + """Public throttled entry point for hand keypoints. + + Emits the full 21-landmark hand skeleton per detected hand on + /hand/count + /hand/kp routes. Side is inferred from id parity + (MediaPipe Hand task does not flag left/right reliably) : we + treat odd ids as right, even as left, which matches the + convention used by the smoother / tracker upstream. + """ + if not force and (t_now - self._last_t) < self._period: + return + self._send_hand(persons_hands, persons_hands_ids) + + def _send_face(self, persons_face: Sequence[Sequence[Any]], + persons_face_ids: Sequence[int]) -> None: + n = len(persons_face) + try: + self._avbody.send_message("/face/count", [int(n)]) + except OSError: + return + for i, face in enumerate(persons_face): + if not face: + continue + pid = persons_face_ids[i] if i < len(persons_face_ids) else i + n_lm = len(face) + for slot, mp_idx in enumerate(FACE_68_FROM_MP): + if mp_idx >= n_lm: + continue + kp = face[mp_idx] + try: + self._avbody.send_message("/face/kp", [ + int(pid), int(slot), + float(kp.x), float(kp.y), + float(getattr(kp, "z", 0.0)), + float(getattr(kp, "c", 1.0)), + ]) + except OSError: + return + + def _send_hand(self, persons_hands: Sequence[Sequence[Any]], + persons_hands_ids: Sequence[int]) -> None: + n_left = 0 + n_right = 0 + for i in range(len(persons_hands)): + pid = persons_hands_ids[i] if i < len(persons_hands_ids) else i + if int(pid) % 2 == 0: + n_left += 1 + else: + n_right += 1 + try: + self._avbody.send_message("/hand/count", [int(n_left), int(n_right)]) + except OSError: + return + for i, hand in enumerate(persons_hands): + if not hand: + continue + pid = persons_hands_ids[i] if i < len(persons_hands_ids) else i + side = 1 if int(pid) % 2 else 0 + for idx, kp in enumerate(hand[:21]): + try: + self._avbody.send_message("/hand/kp", [ + int(pid), int(side), int(idx), + float(kp.x), float(kp.y), + float(getattr(kp, "z", 0.0)), + float(getattr(kp, "c", 1.0)), + ]) + except OSError: + return + + def send_body3d(self, persons_body3d: Sequence[Sequence[Any]], + persons_body3d_ids: Sequence[int], t_now: float, + force: bool = False) -> None: + """Public throttled entry point for 3D body world landmarks. + + Emits 33 MediaPipe pose_world_landmarks per person on + /pose3d/count + /pose3d/kp routes. Coordinates are in meters, + relative to the hip-center (MediaPipe convention: x=right, + y=down, z=forward from the camera). + """ + if not force and (t_now - self._last_t) < self._period: + return + self._send_body3d(persons_body3d, persons_body3d_ids) + + def _send_body3d(self, persons_body3d: Sequence[Sequence[Any]], + persons_body3d_ids: Sequence[int]) -> None: + n = len(persons_body3d) + try: + self._avbody.send_message("/pose3d/count", [int(n)]) + except OSError: + return + for i, body in enumerate(persons_body3d): + if not body: + continue + pid = persons_body3d_ids[i] if i < len(persons_body3d_ids) else i + for idx, kp in enumerate(body[:33]): + try: + self._avbody.send_message("/pose3d/kp", [ + int(pid), int(idx), + float(kp.x), float(kp.y), + float(getattr(kp, "z", 0.0)), + float(getattr(kp, "c", 1.0)), + ]) + except OSError: + return + + # ------------------------------------------------------------------ def send_action(self, pid: int, label_idx: int, probs, t_now: float, force: bool = False) -> None: """Send action classification result via /pose/action OSC route. diff --git a/data_only_viz/scripts/coreml_full_probe.py b/data_only_viz/scripts/coreml_full_probe.py index f0e7bfe..05146b2 100644 --- a/data_only_viz/scripts/coreml_full_probe.py +++ b/data_only_viz/scripts/coreml_full_probe.py @@ -492,6 +492,9 @@ try: compute_units=ct.ComputeUnit.CPU_AND_GPU, minimum_deployment_target=ct.target.macOS15, convert_to="mlprogram", + # FP16 default causes NaN in inverse projection / SMPL-X decoder + # (Multi-HMR has values that overflow the FP16 range). Force FP32. + compute_precision=ct.precision.FLOAT32, ) out_path = "/tmp/multihmr_full_672_s.mlpackage" mlmodel.save(out_path) diff --git a/data_only_viz/state.py b/data_only_viz/state.py index e921f78..54523ac 100644 --- a/data_only_viz/state.py +++ b/data_only_viz/state.py @@ -21,6 +21,16 @@ class PoseKp: c: float = 0.0 +@dataclass +class Kp3D: + """3D keypoint in metric coordinates relative to hip-center. + Used for MediaPipe pose_world_landmarks (xyz in meters).""" + x: float = 0.0 + y: float = 0.0 + z: float = 0.0 + c: float = 0.0 + + @dataclass class SMPLXPerson: """Resultats Multi-HMR pour une personne : params SMPL-X + vertices @@ -92,6 +102,10 @@ class State: persons_body: list[list[PoseKp]] = field(default_factory=list) persons_face: list[list[PoseKp]] = field(default_factory=list) persons_hands: list[list[PoseKp]] = field(default_factory=list) + # MediaPipe pose_world_landmarks per person : 33 keypoints in meters, + # relative to the hip-center. Optional companion of persons_body + # (image-space xy). Empty if no detection or backend doesn't emit it. + persons_body3d: list[list[Kp3D]] = field(default_factory=list) # IDs persistants entre frames (ByteTrack-like via Hungarian IoU). # Couleur du skeleton dans le shader Metal = ID % palette_size. persons_body_ids: list[int] = field(default_factory=list) diff --git a/data_only_viz/tests/test_pose_bridge_body3d.py b/data_only_viz/tests/test_pose_bridge_body3d.py new file mode 100644 index 0000000..9a09e18 --- /dev/null +++ b/data_only_viz/tests/test_pose_bridge_body3d.py @@ -0,0 +1,79 @@ +"""Tests for /pose3d/* OSC routes emitted to AVLiveBody.""" +from __future__ import annotations + +from dataclasses import dataclass +from unittest.mock import MagicMock + + +@dataclass +class _Kp3D: + x: float = 0.0 + y: float = 0.0 + z: float = 0.0 + c: float = 1.0 + + +def _make_bridge(): + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b._avbody = MagicMock() + return b + + +def test_send_body3d_emits_count_and_33_kp() -> None: + b = _make_bridge() + # One person, 33 keypoints with deterministic xyz. + body = [_Kp3D(x=0.01 * i, y=-0.02 * i, z=0.03 * i, c=1.0) + for i in range(33)] + b.send_body3d([body], [7], t_now=0.0, force=True) + + calls = b._avbody.send_message.call_args_list + assert calls[0].args == ("/pose3d/count", [1]) + kp_calls = [c for c in calls if c.args[0] == "/pose3d/kp"] + assert len(kp_calls) == 33 + # Format : [pid, idx, x, y, z, c] + first = kp_calls[0].args[1] + assert first[0] == 7 + assert first[1] == 0 + assert abs(first[2] - 0.0) < 1e-6 + assert abs(first[4] - 0.0) < 1e-6 + # idx ordering strictly 0..32 + idxs = [c.args[1][1] for c in kp_calls] + assert idxs == list(range(33)) + # Last kp z should be 0.03 * 32 + last = kp_calls[-1].args[1] + assert abs(last[4] - 0.03 * 32) < 1e-6 + + +def test_send_body3d_empty_emits_count_zero() -> None: + b = _make_bridge() + b.send_body3d([], [], t_now=0.0, force=True) + calls = b._avbody.send_message.call_args_list + assert len(calls) == 1 + assert calls[0].args == ("/pose3d/count", [0]) + + +def test_send_body3d_multi_person() -> None: + b = _make_bridge() + body_a = [_Kp3D(x=1.0) for _ in range(33)] + body_b = [_Kp3D(x=2.0) for _ in range(33)] + b.send_body3d([body_a, body_b], [10, 11], t_now=0.0, force=True) + calls = b._avbody.send_message.call_args_list + assert calls[0].args == ("/pose3d/count", [2]) + kp_calls = [c for c in calls if c.args[0] == "/pose3d/kp"] + assert len(kp_calls) == 66 + assert kp_calls[0].args[1][0] == 10 + assert kp_calls[33].args[1][0] == 11 + + +def test_send_with_body3d_kwargs_dispatches() -> None: + """Top-level send() routes body3d kwargs to /pose3d.""" + b = _make_bridge() + body = [_Kp3D(x=0.5, y=0.5, c=1.0) for _ in range(33)] + body3d = [_Kp3D(x=0.1, y=0.2, z=0.3) for _ in range(33)] + b.send([body], [0], 1.0, + persons_body3d=[body3d], persons_body3d_ids=[0]) + av_addrs = {c.args[0] for c in b._avbody.send_message.call_args_list} + assert "/pose3d/count" in av_addrs + assert "/pose3d/kp" in av_addrs diff --git a/data_only_viz/tests/test_pose_bridge_face_hand.py b/data_only_viz/tests/test_pose_bridge_face_hand.py new file mode 100644 index 0000000..3add1e5 --- /dev/null +++ b/data_only_viz/tests/test_pose_bridge_face_hand.py @@ -0,0 +1,105 @@ +"""Tests for /face/* and /hand/* OSC routes emitted to AVLiveBody.""" +from __future__ import annotations + +from dataclasses import dataclass +from unittest.mock import MagicMock + + +@dataclass +class _Kp: + x: float = 0.0 + y: float = 0.0 + z: float = 0.0 + c: float = 1.0 + + +def _make_bridge(): + from data_only_viz.pose_bridge import PoseSoundBridge + b = PoseSoundBridge() + b._client = MagicMock() + b._avbody = MagicMock() + return b + + +def test_send_face_emits_count_and_68_kp() -> None: + from data_only_viz.pose_bridge import FACE_68_FROM_MP + b = _make_bridge() + # One face with 478 landmarks at deterministic coords. + face = [_Kp(x=i / 478.0, y=1.0 - i / 478.0, z=0.01 * i, c=1.0) + for i in range(478)] + b.send_face([face], [3], t_now=0.0, force=True) + + calls = b._avbody.send_message.call_args_list + # First call : /face/count <1> + assert calls[0].args[0] == "/face/count" + assert calls[0].args[1] == [1] + # Then 68 /face/kp messages + kp_calls = [c for c in calls if c.args[0] == "/face/kp"] + assert len(kp_calls) == 68 + # Format : [pid, slot, x, y, z, c] + first = kp_calls[0].args[1] + assert first[0] == 3 + assert first[1] == 0 + assert isinstance(first[2], float) + assert isinstance(first[3], float) + assert isinstance(first[4], float) + assert isinstance(first[5], float) + # Slot ordering is strictly increasing 0..67 + slots = [c.args[1][1] for c in kp_calls] + assert slots == list(range(68)) + # And the x coord of slot 0 matches mp_idx FACE_68_FROM_MP[0] + mp0 = FACE_68_FROM_MP[0] + assert abs(first[2] - mp0 / 478.0) < 1e-6 + + +def test_send_face_empty_emits_count_zero() -> None: + b = _make_bridge() + b.send_face([], [], t_now=0.0, force=True) + calls = b._avbody.send_message.call_args_list + assert len(calls) == 1 + assert calls[0].args == ("/face/count", [0]) + + +def test_send_hand_emits_count_and_21_kp() -> None: + b = _make_bridge() + hand_l = [_Kp(x=0.1, y=0.2, z=0.0, c=1.0) for _ in range(21)] + hand_r = [_Kp(x=0.7, y=0.3, z=0.0, c=1.0) for _ in range(21)] + # pid=2 -> left (even), pid=3 -> right (odd) + b.send_hand([hand_l, hand_r], [2, 3], t_now=0.0, force=True) + calls = b._avbody.send_message.call_args_list + assert calls[0].args == ("/hand/count", [1, 1]) + kp_calls = [c for c in calls if c.args[0] == "/hand/kp"] + assert len(kp_calls) == 42 # 21 * 2 + # First hand should be side=0 (left) + assert kp_calls[0].args[1][1] == 0 + # 22nd kp call : start of right hand, side=1 + assert kp_calls[21].args[1][1] == 1 + + +def test_send_throttles_below_period() -> None: + """send_face called twice within < period emits only once.""" + b = _make_bridge() + face = [_Kp(x=0.0, y=0.0) for _ in range(478)] + b.send_face([face], [0], t_now=0.0, force=True) + n_first = b._avbody.send_message.call_count + # Second call without force AND inside throttle window : skipped. + b._last_t = 9999.0 # pretend a body send just happened + b.send_face([face], [0], t_now=9999.0 + 0.001, force=False) + assert b._avbody.send_message.call_count == n_first + + +def test_send_with_face_hand_kwargs_dispatches() -> None: + """Top-level send() routes face/hand kwargs to /face and /hand.""" + b = _make_bridge() + body = [_Kp(x=0.5, y=0.5, c=1.0) for _ in range(33)] + face = [_Kp(x=0.1, y=0.1, c=1.0) for _ in range(478)] + hand = [_Kp(x=0.2, y=0.2, c=1.0) for _ in range(21)] + b.send([body], [0], 1.0, + persons_face=[face], persons_face_ids=[0], + persons_hands=[hand], persons_hands_ids=[1]) + av_addrs = {c.args[0] for c in b._avbody.send_message.call_args_list} + assert "/pose/count" in av_addrs + assert "/face/count" in av_addrs + assert "/face/kp" in av_addrs + assert "/hand/count" in av_addrs + assert "/hand/kp" in av_addrs diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift index 38932dc..1357b20 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift @@ -1,8 +1,19 @@ import Cocoa import SwiftUI +// SwiftPM binaries lack a bundle Info.plist, so macOS treats us as a +// background CLI app and never shows the WindowGroup window. The +// AppDelegate forces regular activation after NSApp is initialized. +class AppDelegate: NSObject, NSApplicationDelegate { + func applicationDidFinishLaunching(_ notification: Notification) { + NSApp.setActivationPolicy(.regular) + NSApp.activate(ignoringOtherApps: true) + } +} + @main struct AVLiveBodyApp: App { + @NSApplicationDelegateAdaptor(AppDelegate.self) var appDelegate var body: some Scene { WindowGroup { ContentView() @@ -46,6 +57,11 @@ struct AVLiveBodyApp: App { KeyEquivalent(Character(String(i))), modifiers: []) } + // Alias 'p' for openpos (skeleton view). + Button("p — openpos (squelette)") { + NotificationCenter.default.post( + name: .setVizMode, object: 9) + }.keyboardShortcut("p", modifiers: []) } } } @@ -61,11 +77,12 @@ struct ContentView: View { @StateObject private var renderer = MeshRenderer() @StateObject private var settings = RenderSettings() @StateObject private var poseListener = PoseOSCListener() + @StateObject private var skeleton3d = Skeleton3DRenderer() var body: some View { ZStack(alignment: .topLeading) { BodyView(renderer: renderer, settings: settings, - poseListener: poseListener) + poseListener: poseListener, skeleton3d: skeleton3d) .onAppear { renderer.startOSCServer() poseListener.start() @@ -83,6 +100,10 @@ struct ContentView: View { if let n = note.object as? Int { settings.vizMode = n } } + // Face + hand skeleton overlay (data_only_viz/pose_bridge.py) + FaceHandOverlay(poseListener: poseListener) + .allowsHitTesting(false) + // HUD coin haut-gauche : mode + touches + pose HUDOverlay(settings: settings, poseListener: poseListener) diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift index 77ce42a..647e8d3 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift @@ -11,6 +11,7 @@ struct BodyView: NSViewRepresentable { @ObservedObject var renderer: MeshRenderer @ObservedObject var settings: RenderSettings @ObservedObject var poseListener: PoseOSCListener + @ObservedObject var skeleton3d: Skeleton3DRenderer func makeNSView(context: Context) -> NSView { let container = NSView(frame: .zero) @@ -88,6 +89,14 @@ struct BodyView: NSViewRepresentable { let bodyAnchor = AnchorEntity(world: .zero) arView.scene.addAnchor(bodyAnchor) + + // Dedicated anchor for the 3D skeleton (mode 9 / openpos). + // Positioned at the origin ; the perspective camera at z=0 with + // default FOV frames a ~3 m-deep stage centered on the hip. + let skelAnchor = AnchorEntity(world: SIMD3(0, 0, -3)) + arView.scene.addAnchor(skelAnchor) + skeleton3d.attach(to: skelAnchor, listener: poseListener) + container.addSubview(arView) // 60 fps mesh interpolation between Multi-HMR frames (Python @@ -105,6 +114,7 @@ struct BodyView: NSViewRepresentable { context.coordinator.previewLayer = preview context.coordinator.container = container context.coordinator.renderer = renderer + context.coordinator.skelAnchor = skelAnchor return container } @@ -141,6 +151,9 @@ struct BodyView: NSViewRepresentable { c.fillLight?.light.intensity = Float(settings.fillIntensity) c.rimLight?.light.intensity = Float(settings.rimIntensity) + // 3D skeleton only visible in mode 9 (openpos). + c.skelAnchor?.isEnabled = (settings.vizMode == 9) + // Mesh visibility + material guard let anchor = c.bodyAnchor else { return } anchor.children.removeAll() @@ -159,6 +172,7 @@ struct BodyView: NSViewRepresentable { final class Coordinator { var bodyAnchor: AnchorEntity? + var skelAnchor: AnchorEntity? var arView: ARView? var cameraEntity: PerspectiveCamera? var sceneRenderer: SceneRenderer? diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift new file mode 100644 index 0000000..8bdeb98 --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift @@ -0,0 +1,155 @@ +import SwiftUI +import simd + +/// Lightweight SwiftUI overlay that draws the 68-point face skeleton and +/// 21-point hand skeletons sent by data_only_viz/pose_bridge.py over OSC. +/// Sits in the ContentView ZStack above BodyView. Coordinates from the +/// listener are normalised (0..1 in image space) ; here we map them to +/// the overlay's geometry. Rendering is intentionally minimal : small +/// dots + a few polylines for facial features and hand bones. +struct FaceHandOverlay: View { + @ObservedObject var poseListener: PoseOSCListener + var showFace: Bool = true + var showHands: Bool = true + + var body: some View { + GeometryReader { geo in + Canvas { ctx, size in + if showFace { + for face in poseListener.faces.values { + drawFace(face, in: &ctx, size: size) + } + } + if showHands { + for hand in poseListener.hands.values { + drawHand(hand, in: &ctx, size: size) + } + } + } + .frame(width: geo.size.width, height: geo.size.height) + .allowsHitTesting(false) + } + } + + // MARK: - Face (dlib 68 layout) + + /// Index spans in the 68-point dlib convention. + private static let jaw = Array(0..<17) + private static let browR = Array(17..<22) + private static let browL = Array(22..<27) + private static let noseBridge = Array(27..<31) + private static let nostril = Array(31..<36) + private static let eyeR = Array(36..<42) + private static let eyeL = Array(42..<48) + private static let lipOuter = Array(48..<60) + private static let lipInner = Array(60..<68) + + private func drawFace(_ face: PoseOSCListener.FaceFrame, + in ctx: inout GraphicsContext, + size: CGSize) { + let stroke = GraphicsContext.Shading.color(.green.opacity(0.85)) + let dot = GraphicsContext.Shading.color(.green.opacity(0.95)) + + drawPolyline(face.points, indices: Self.jaw, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.browR, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.browL, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.noseBridge, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.nostril, closed: false, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.eyeR, closed: true, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.eyeL, closed: true, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.lipOuter, closed: true, + in: &ctx, size: size, shading: stroke, width: 1.2) + drawPolyline(face.points, indices: Self.lipInner, closed: true, + in: &ctx, size: size, shading: stroke, width: 1.0) + + for i in 0..<68 where face.hasPoint[i] { + let p = mapPoint(face.points[i], size: size) + let r = CGRect(x: p.x - 1.2, y: p.y - 1.2, + width: 2.4, height: 2.4) + ctx.fill(Path(ellipseIn: r), with: dot) + } + } + + // MARK: - Hand (MediaPipe 21 landmarks) + + /// MediaPipe hand bone connectivity (5 fingers x 4 bones + palm). + private static let handBones: [(Int, Int)] = [ + // Thumb + (0, 1), (1, 2), (2, 3), (3, 4), + // Index + (0, 5), (5, 6), (6, 7), (7, 8), + // Middle + (5, 9), (9, 10), (10, 11), (11, 12), + // Ring + (9, 13), (13, 14), (14, 15), (15, 16), + // Pinky + (13, 17), (17, 18), (18, 19), (19, 20), + // Palm closure + (0, 17), + ] + + private func drawHand(_ hand: PoseOSCListener.HandFrame, + in ctx: inout GraphicsContext, + size: CGSize) { + // Left = cyan, right = magenta. + let color: Color = hand.side == 0 ? .cyan : .pink + let stroke = GraphicsContext.Shading.color(color.opacity(0.85)) + let dot = GraphicsContext.Shading.color(color.opacity(0.95)) + + for (a, b) in Self.handBones { + guard hand.hasPoint[a], hand.hasPoint[b] else { continue } + let pa = mapPoint(hand.points[a], size: size) + let pb = mapPoint(hand.points[b], size: size) + var path = Path() + path.move(to: pa) + path.addLine(to: pb) + ctx.stroke(path, with: stroke, lineWidth: 1.8) + } + for i in 0..<21 where hand.hasPoint[i] { + let p = mapPoint(hand.points[i], size: size) + let r = CGRect(x: p.x - 1.8, y: p.y - 1.8, + width: 3.6, height: 3.6) + ctx.fill(Path(ellipseIn: r), with: dot) + } + } + + // MARK: - Helpers + + private func drawPolyline(_ pts: [SIMD2], + indices: [Int], + closed: Bool, + in ctx: inout GraphicsContext, + size: CGSize, + shading: GraphicsContext.Shading, + width: CGFloat) { + guard indices.count >= 2 else { return } + var path = Path() + var started = false + for i in indices { + let p = mapPoint(pts[i], size: size) + if !started { + path.move(to: p) + started = true + } else { + path.addLine(to: p) + } + } + if closed, let first = indices.first { + path.addLine(to: mapPoint(pts[first], size: size)) + } + ctx.stroke(path, with: shading, lineWidth: width) + } + + private func mapPoint(_ p: SIMD2, size: CGSize) -> CGPoint { + // Normalised coords come from MediaPipe in image space already. + CGPoint(x: CGFloat(p.x) * size.width, + y: CGFloat(p.y) * size.height) + } +} diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift index 28f0822..e8ab3a3 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift @@ -23,8 +23,42 @@ final class PoseOSCListener: ObservableObject { var seenAt: TimeInterval = 0 } + /// 68-point dlib-style facial landmarks (x,y normalises 0..1). + /// Slot mapping comes from FACE_68_FROM_MP cote Python. + struct FaceFrame: Equatable { + var points: [SIMD2] = Array(repeating: .zero, count: 68) + var hasPoint: [Bool] = Array(repeating: false, count: 68) + var seenAt: TimeInterval = 0 + } + + /// 21 MediaPipe hand landmarks per detected hand. + struct HandFrame: Equatable { + var side: Int = 0 // 0 = left, 1 = right + var points: [SIMD2] = Array(repeating: .zero, count: 21) + var hasPoint: [Bool] = Array(repeating: false, count: 21) + var seenAt: TimeInterval = 0 + } + + /// MediaPipe pose_world_landmarks : 33 keypoints in meters, relative + /// to the hip-center. Conventions on the wire (MediaPipe): + /// x = right, y = down, z = forward (away from camera). + struct Pose3DFrame: Equatable { + var pid: Int = -1 + // SIMD4 = (x, y, z, confidence). All zeros = slot not yet filled. + var kps: [SIMD4] = Array(repeating: .zero, count: 33) + var hasPoint: [Bool] = Array(repeating: false, count: 33) + var seenAt: TimeInterval = 0 + } + @Published var persons: [Int: PoseFrame] = [:] + @Published var faces: [Int: FaceFrame] = [:] + @Published var hands: [Int: HandFrame] = [:] + @Published var body3d: [Int: Pose3DFrame] = [:] @Published var count: Int = 0 + @Published var faceCount: Int = 0 + @Published var body3dCount: Int = 0 + @Published var handCountLeft: Int = 0 + @Published var handCountRight: Int = 0 private var listener: NWListener? @@ -127,6 +161,69 @@ final class PoseOSCListener: ObservableObject { var p = persons[Int(pid)] ?? PoseFrame() p.bodyPitch = v persons[Int(pid)] = p + case "/face/count": + if let n = args.first as? Int32 { faceCount = Int(n) } + if faceCount == 0 { faces.removeAll(keepingCapacity: true) } + case "/face/kp": + guard args.count >= 6, + let pid = args[0] as? Int32, + let slot = args[1] as? Int32, + let x = args[2] as? Float, + let y = args[3] as? Float else { return } + let slotI = Int(slot) + guard slotI >= 0 && slotI < 68 else { return } + var f = faces[Int(pid)] ?? FaceFrame() + f.points[slotI] = SIMD2(x, y) + f.hasPoint[slotI] = true + f.seenAt = CFAbsoluteTimeGetCurrent() + faces[Int(pid)] = f + case "/hand/count": + if args.count >= 2, + let l = args[0] as? Int32, let r = args[1] as? Int32 { + handCountLeft = Int(l) + handCountRight = Int(r) + if handCountLeft + handCountRight == 0 { + hands.removeAll(keepingCapacity: true) + } + } + case "/pose3d/count": + if let n = args.first as? Int32 { + body3dCount = Int(n) + if body3dCount == 0 { + body3d.removeAll(keepingCapacity: true) + } + } + case "/pose3d/kp": + guard args.count >= 6, + let pid = args[0] as? Int32, + let idx = args[1] as? Int32, + let x = args[2] as? Float, + let y = args[3] as? Float, + let z = args[4] as? Float, + let c = args[5] as? Float else { return } + let i = Int(idx) + guard i >= 0 && i < 33 else { return } + var p = body3d[Int(pid)] ?? Pose3DFrame(pid: Int(pid)) + p.pid = Int(pid) + p.kps[i] = SIMD4(x, y, z, c) + p.hasPoint[i] = true + p.seenAt = CFAbsoluteTimeGetCurrent() + body3d[Int(pid)] = p + case "/hand/kp": + guard args.count >= 7, + let pid = args[0] as? Int32, + let side = args[1] as? Int32, + let idx = args[2] as? Int32, + let x = args[3] as? Float, + let y = args[4] as? Float else { return } + let i = Int(idx) + guard i >= 0 && i < 21 else { return } + var h = hands[Int(pid)] ?? HandFrame() + h.side = Int(side) + h.points[i] = SIMD2(x, y) + h.hasPoint[i] = true + h.seenAt = CFAbsoluteTimeGetCurrent() + hands[Int(pid)] = h default: break } @@ -134,6 +231,12 @@ final class PoseOSCListener: ObservableObject { let now = CFAbsoluteTimeGetCurrent() persons = persons.filter { $0.value.seenAt == 0 || now - $0.value.seenAt < 2.0 } + faces = faces.filter { $0.value.seenAt == 0 + || now - $0.value.seenAt < 2.0 } + hands = hands.filter { $0.value.seenAt == 0 + || now - $0.value.seenAt < 2.0 } + body3d = body3d.filter { $0.value.seenAt == 0 + || now - $0.value.seenAt < 2.0 } } // MARK: - Minimal OSC parser diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift new file mode 100644 index 0000000..0a6c2f3 --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift @@ -0,0 +1,222 @@ +import Combine +import Foundation +import RealityKit +import SwiftUI +import simd + +/// RealityKit renderer for MediaPipe Pose 3D world landmarks (33 joints, +/// metric coords relative to the hip-center). Consumes the `body3d` +/// publisher of `PoseOSCListener` and maintains one entity tree per +/// detected person. +/// +/// Coordinate mapping (MediaPipe -> RealityKit): +/// MediaPipe : x = right, y = down, z = forward (away from cam). +/// RealityKit: x = right, y = up, z = backward (toward cam). +/// => convert with (x, -y, -z). +@MainActor +final class Skeleton3DRenderer: ObservableObject { + /// 32 bones connecting MediaPipe Pose 33 landmarks. Indices are + /// the canonical MediaPipe Pose landmark indices. Source: official + /// `mp.solutions.pose.POSE_CONNECTIONS` (Holistic / Pose Landmarker + /// share the same 33-pt schema). + static let POSE_CONNECTIONS: [(Int, Int, BoneChain)] = [ + // Face (kept light: nose <-> inner eyes <-> outer eyes <-> ears) + (0, 1, .face), (1, 2, .face), (2, 3, .face), (3, 7, .face), + (0, 4, .face), (4, 5, .face), (5, 6, .face), (6, 8, .face), + (9, 10, .face), + // Torso + (11, 12, .trunk), (11, 23, .trunk), (12, 24, .trunk), + (23, 24, .trunk), + // Left arm + (11, 13, .arm), (13, 15, .arm), + (15, 17, .arm), (15, 19, .arm), (15, 21, .arm), (17, 19, .arm), + // Right arm + (12, 14, .arm), (14, 16, .arm), + (16, 18, .arm), (16, 20, .arm), (16, 22, .arm), (18, 20, .arm), + // Left leg + (23, 25, .leg), (25, 27, .leg), + (27, 29, .leg), (27, 31, .leg), (29, 31, .leg), + // Right leg + (24, 26, .leg), (26, 28, .leg), + (28, 30, .leg), (28, 32, .leg), (30, 32, .leg), + ] + + enum BoneChain { + case trunk, arm, leg, face + var color: NSColor { + switch self { + case .trunk: return .white + case .arm: return .systemTeal + case .leg: return .systemPink // approx magenta + case .face: return NSColor(white: 0.7, alpha: 1.0) + } + } + } + + private static let jointRadius: Float = 0.02 // 2 cm + private static let boneRadius: Float = 0.012 // 1.2 cm + private static let minConfidence: Float = 0.3 + private static let retainSec: TimeInterval = 1.0 + + /// Update throttle : tick at most every `updatePeriod` seconds even + /// if the publisher fires faster (Combine debounce-style on a clock). + private static let updatePeriod: TimeInterval = 1.0 / 30.0 + + private struct PersonEntities { + var root: Entity + var joints: [ModelEntity] // 33 spheres + var bones: [ModelEntity] // 32 bone entities, same order as POSE_CONNECTIONS + } + + private var persons: [Int: PersonEntities] = [:] + private var lastSeenAt: [Int: TimeInterval] = [:] + private weak var rootAnchor: Entity? + private var poseSub: AnyCancellable? + private var lastUpdateAt: TimeInterval = 0 + + /// Attach to a scene by giving it an AnchorEntity that owns all + /// skeleton entities, and start observing the listener. + func attach(to anchor: Entity, listener: PoseOSCListener) { + rootAnchor = anchor + poseSub = listener.$body3d + .receive(on: DispatchQueue.main) + .sink { [weak self] frames in + Task { @MainActor in self?.update(frames: frames) } + } + } + + func detach() { + poseSub?.cancel() + poseSub = nil + for (_, p) in persons { p.root.removeFromParent() } + persons.removeAll() + lastSeenAt.removeAll() + } + + // MARK: - Update + + private func update(frames: [Int: PoseOSCListener.Pose3DFrame]) { + let now = CACurrentMediaTime() + if now - lastUpdateAt < Self.updatePeriod { return } + lastUpdateAt = now + + guard let anchor = rootAnchor else { return } + + // Mark fresh pids + for pid in frames.keys { lastSeenAt[pid] = now } + // GC stale persons + let cutoff = now - Self.retainSec + for (pid, p) in persons where (lastSeenAt[pid] ?? 0) < cutoff { + p.root.removeFromParent() + persons.removeValue(forKey: pid) + lastSeenAt.removeValue(forKey: pid) + } + + for (pid, frame) in frames { + let entities = persons[pid] ?? makePerson(pid: pid, parent: anchor) + persons[pid] = entities + apply(frame: frame, to: entities) + } + } + + private func apply(frame: PoseOSCListener.Pose3DFrame, + to entities: PersonEntities) { + // Convert all 33 keypoints to RealityKit space once. + var rk = [SIMD3](repeating: .zero, count: 33) + var valid = [Bool](repeating: false, count: 33) + for i in 0..<33 { + let k = frame.kps[i] + let visible = frame.hasPoint[i] && k.w >= Self.minConfidence + valid[i] = visible + // Mediapipe (x right, y down, z forward) -> RK (x right, y up, z back) + rk[i] = SIMD3(k.x, -k.y, -k.z) + } + + // Joints: position spheres and toggle visibility. + for i in 0..<33 { + let joint = entities.joints[i] + if valid[i] { + joint.transform.translation = rk[i] + joint.isEnabled = true + } else { + joint.isEnabled = false + } + } + + // Bones: orient + scale length between endpoints. + for (bIdx, (a, b, _)) in Self.POSE_CONNECTIONS.enumerated() { + let bone = entities.bones[bIdx] + if !valid[a] || !valid[b] { + bone.isEnabled = false + continue + } + let pa = rk[a] + let pb = rk[b] + let delta = pb - pa + let len = simd_length(delta) + if len < 1e-5 { + bone.isEnabled = false + continue + } + let mid = (pa + pb) * 0.5 + // Bone mesh is a cylinder of height=1 along +Y. Rotate +Y + // onto the (b-a) direction. + let dir = delta / len + let yAxis = SIMD3(0, 1, 0) + let dot = simd_dot(yAxis, dir) + let rot: simd_quatf + if dot > 0.9999 { + rot = simd_quatf(angle: 0, axis: SIMD3(0, 1, 0)) + } else if dot < -0.9999 { + rot = simd_quatf(angle: .pi, axis: SIMD3(1, 0, 0)) + } else { + let axis = simd_normalize(simd_cross(yAxis, dir)) + let angle = acos(dot) + rot = simd_quatf(angle: angle, axis: axis) + } + bone.transform.translation = mid + bone.transform.rotation = rot + // Scale length only on Y, keep XZ at 1 to preserve radius. + bone.transform.scale = SIMD3(1, len, 1) + bone.isEnabled = true + } + } + + // MARK: - Construction + + private func makePerson(pid: Int, parent: Entity) -> PersonEntities { + let root = Entity() + parent.addChild(root) + + // Joint sphere mesh shared across joints (cheap to reuse). + let sphereMesh = MeshResource.generateSphere( + radius: Self.jointRadius) + let jointMat = SimpleMaterial( + color: .white, roughness: 0.6, isMetallic: false) + var joints: [ModelEntity] = [] + joints.reserveCapacity(33) + for _ in 0..<33 { + let e = ModelEntity(mesh: sphereMesh, materials: [jointMat]) + e.isEnabled = false + root.addChild(e) + joints.append(e) + } + + // One cylinder per bone (height=1, scaled at runtime). + let cylMesh = MeshResource.generateCylinder( + height: 1.0, radius: Self.boneRadius) + var bones: [ModelEntity] = [] + bones.reserveCapacity(Self.POSE_CONNECTIONS.count) + for (_, _, chain) in Self.POSE_CONNECTIONS { + let mat = SimpleMaterial( + color: chain.color, roughness: 0.6, isMetallic: false) + let e = ModelEntity(mesh: cylMesh, materials: [mat]) + e.isEnabled = false + root.addChild(e) + bones.append(e) + } + NSLog("Skeleton3DRenderer: spawned pid=%d (33 joints, %d bones)", + pid, bones.count) + return PersonEntities(root: root, joints: joints, bones: bones) + } +} From a5c793f39cae932a83f0dde95de192f7232f5a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:36:19 +0200 Subject: [PATCH 16/18] feat(data-only-viz): action capture webcam script Implement Task 10 of action-head plan. Records webcam frames + timestamps for action-head training using OpenCV. Outputs MP4 video + timestamp text file to ~/.cache/av-live-action/raw/ with 672x672 square crop, configurable fps/session/camera, interactive q-quit. --- data_only_viz/scripts/capture_actions.py | 74 ++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 data_only_viz/scripts/capture_actions.py diff --git a/data_only_viz/scripts/capture_actions.py b/data_only_viz/scripts/capture_actions.py new file mode 100644 index 0000000..331858d --- /dev/null +++ b/data_only_viz/scripts/capture_actions.py @@ -0,0 +1,74 @@ +"""Record webcam frames + timestamps for action-head training. + +Usage: + uv run python -m data_only_viz.scripts.capture_actions \ + --session sess03 --duration 600 +""" +from __future__ import annotations + +import argparse +import logging +import time +from pathlib import Path + +import cv2 + +LOG = logging.getLogger("capture_actions") +RAW_DIR = Path("~/.cache/av-live-action/raw").expanduser() + + +def capture(session: str, duration_s: float, + cam_index: int = 0, fps: int = 30, + size: int = 672) -> Path: + RAW_DIR.mkdir(parents=True, exist_ok=True) + out = RAW_DIR / f"{session}.mp4" + ts_out = RAW_DIR / f"{session}.ts.txt" + cap = cv2.VideoCapture(cam_index) + if not cap.isOpened(): + raise RuntimeError(f"cannot open camera {cam_index}") + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + writer = cv2.VideoWriter(str(out), fourcc, fps, (size, size)) + try: + t_start = time.perf_counter() + with ts_out.open("w") as ts_f: + n = 0 + while time.perf_counter() - t_start < duration_s: + ok, frame = cap.read() + if not ok: + LOG.warning("frame read failed") + break + h, w = frame.shape[:2] + side = min(h, w) + y0 = (h - side) // 2 + x0 = (w - side) // 2 + crop = frame[y0:y0 + side, x0:x0 + side] + resized = cv2.resize(crop, (size, size)) + writer.write(resized) + ts_f.write(f"{n} {time.perf_counter() - t_start:.6f}\n") + n += 1 + cv2.imshow("capture (q=quit)", resized) + if cv2.waitKey(1) & 0xFF == ord("q"): + break + LOG.info("wrote %s (%d frames)", out, n) + return out + finally: + cap.release() + writer.release() + cv2.destroyAllWindows() + + +def _cli() -> None: + p = argparse.ArgumentParser() + p.add_argument("--session", required=True) + p.add_argument("--duration", type=float, default=600.0) + p.add_argument("--cam-index", type=int, default=0) + p.add_argument("--fps", type=int, default=30) + args = p.parse_args() + logging.basicConfig(level=logging.INFO, + format="%(asctime)s [%(name)s] %(message)s") + capture(args.session, args.duration, + cam_index=args.cam_index, fps=args.fps) + + +if __name__ == "__main__": + _cli() From 5013a1916df99a0408507341074b3c652a022599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:37:31 +0200 Subject: [PATCH 17/18] docs(coreml-probe): note NaN bug deferred Add comment in TracedMHMR.forward documenting that CoreML conversion produces all-NaN on v3d/transl while PyTorch eager works. Tested FP32, K_inv closed-form, simplified subtract+divide projection, nan_to_num masking. Root cause is an op-level mistranslation in the v3d/transl path; needs sub-wrapper bisection. Workaround: PyTorch backend. --- data_only_viz/scripts/coreml_full_probe.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data_only_viz/scripts/coreml_full_probe.py b/data_only_viz/scripts/coreml_full_probe.py index 05146b2..539f734 100644 --- a/data_only_viz/scripts/coreml_full_probe.py +++ b/data_only_viz/scripts/coreml_full_probe.py @@ -259,6 +259,12 @@ class TracedMHMR(nn.Module): ]).squeeze(-1) shape = torch.stack([h["shape"] for h in humans]) expr = torch.stack([h["expression"] for h in humans]) + # NOTE: CoreML mlprogram conversion currently produces all-NaN + # outputs for v3d and transl while PyTorch eager produces valid + # finite values from the same trace. nan_to_num here masks the + # symptom but yields all-zero meshes (no information). Leave + # raw outputs and let downstream decide; investigation tracked + # in task #2 (op-by-op bisection needed). return v3d, transl, scores, shape, expr From b5e91393176c8267b71809fc5f8b86931825ff0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 22:41:31 +0200 Subject: [PATCH 18/18] feat(data-only-viz): ActionHead publisher thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standalone publisher polls state at 30 Hz, extracts 22-joint positions from either persons_smplx (vertex anchors) or persons_body3d (MediaPipe 33→22 map), runs ActionHead.step() per pid, and emits /pose/action + /pose/kin + lifecycle OSC. - action_head_pub.py: ActionHeadPublisher thread with dedup via smplx_last_t / pose_last_t; purges lost pids - tests/test_action_head_pub.py: 4 unit tests (39 total pass) - multi.py: import + instantiate + start publisher in __init__ --- data_only_viz/action_head_pub.py | 171 ++++++++++++++++++++ data_only_viz/multi.py | 3 + data_only_viz/tests/test_action_head_pub.py | 84 ++++++++++ 3 files changed, 258 insertions(+) create mode 100644 data_only_viz/action_head_pub.py create mode 100644 data_only_viz/tests/test_action_head_pub.py diff --git a/data_only_viz/action_head_pub.py b/data_only_viz/action_head_pub.py new file mode 100644 index 0000000..ca36bb2 --- /dev/null +++ b/data_only_viz/action_head_pub.py @@ -0,0 +1,171 @@ +"""Action-head publisher : reads state.persons_smplx / persons_body3d, +runs ActionHead per pid, emits /pose/action and /pose/kin via pose_bridge. + +Stand-alone thread to avoid touching multi_hmr_worker.py while it +iterates. Polls state at ~30 Hz, deduplicates by smplx_last_t. +""" +from __future__ import annotations + +import logging +import threading +import time +from pathlib import Path +from typing import Any + +import numpy as np + +from data_only_viz.action_head import ActionHead, LABELS + +LOG = logging.getLogger("action_head_pub") + +DEFAULT_CKPT = ( + Path.home() / ".cache" / "av-live-action" / "checkpoints" / "action_head.pt" +) + +# 22 vertex indices on the 10475-vertex SMPL-X mesh, approximating +# the 22-joint kinematic chain used by ActionHead. +# NOTE: approximate vertex anchors — real SMPL-X joints come from +# J_regressor @ v3d, but loading the regressor here is avoided for +# live OSC performance. Action-head training must use the same anchors. +SMPLX_JOINT_ANCHOR_VERTS: tuple[int, ...] = ( + 8204, 3992, 6677, 3500, 3469, 6394, 3279, 3327, 6736, 3074, + 8846, 8889, 8848, 1300, 4660, 8964, 3013, 6470, 1602, 5083, + 2114, 5559, +) + +# MediaPipe 33-landmark indices mapped into the 22-joint slot order. +# NOTE: approximate mapping — spine joints reuse hip/shoulder anchors. +# https://developers.google.com/mediapipe/solutions/vision/pose_landmarker +MEDIAPIPE_TO_22: tuple[int, ...] = ( + 24, 23, 24, 23, 25, 26, 11, 27, 28, 11, + 31, 32, 0, 11, 12, 0, 11, 12, 13, 14, 15, 16, +) + + +class ActionHeadPublisher(threading.Thread): + """Thread that polls state, runs ActionHead per pid, emits OSC.""" + + def __init__(self, state: Any, bridge: Any, + ckpt_path: Path | None = DEFAULT_CKPT, + period_s: float = 1.0 / 30.0) -> None: + super().__init__(daemon=True, name="action-head-pub") + self.state = state + self.bridge = bridge + self.period = period_s + try: + ckpt = ckpt_path if (ckpt_path and ckpt_path.exists()) else None + self.head = ActionHead(ckpt_path=ckpt, device="cpu") + LOG.info("action_head loaded ckpt=%s", + ckpt if ckpt else "") + except Exception as e: + LOG.warning("action_head init failed: %s", e) + self.head = None + self._stop = threading.Event() + self._last_smplx_t = 0.0 + self._last_body_t = 0.0 + self._last_pids: set[int] = set() + + def stop(self) -> None: + self._stop.set() + + def run(self) -> None: + if self.head is None: + LOG.warning("publisher exiting: no action_head") + return + LOG.info("publisher started") + while not self._stop.is_set(): + t0 = time.perf_counter() + try: + self._tick(t0) + except Exception: + LOG.exception("publisher tick failed") + dt = time.perf_counter() - t0 + if dt < self.period: + time.sleep(self.period - dt) + LOG.info("publisher stopped") + + def _tick(self, t_now: float) -> None: + persons22, source_t, source_tag, is_new = self._read_sources() + if not is_new: + return + if "smplx" in source_tag: + self._last_smplx_t = source_t + else: + self._last_body_t = source_t + current_pids: set[int] = set() + if persons22: + for pid, j3d in persons22: + current_pids.add(pid) + label, probs, kin = self.head.step(pid, j3d) + idx = LABELS.index(label) + self.bridge.send_action(pid, idx, probs, t_now, force=True) + self.bridge.send_kin(pid, kin, t_now, force=True) + if pid not in self._last_pids: + self.bridge.send_enter(pid=pid) + for gone in self._last_pids - current_pids: + self.head.forget(gone) + self.bridge.send_leave(pid=gone) + self._last_pids = current_pids + + def _read_sources( + self, + ) -> tuple[list[tuple[int, np.ndarray]] | None, float, str, bool]: + """Return (persons22, source_t, source_tag, is_new). + + is_new is True when the timestamp advanced (even if person list + is empty), so _tick can still run the purge loop. + """ + with self.state.lock(): + persons_smplx = getattr(self.state, "persons_smplx", None) + t_smplx = getattr(self.state, "smplx_last_t", 0.0) + persons_b3d = getattr(self.state, "persons_body3d", None) + ids_b3d = getattr(self.state, "persons_body_ids", None) + t_body = getattr(self.state, "pose_last_t", 0.0) + # Prefer smplx when its timestamp advanced. + if t_smplx > self._last_smplx_t: + out: list[tuple[int, np.ndarray]] = [] + for i, p in enumerate(persons_smplx or []): + pid = int(p.get("pid", i)) + v3d = p.get("v3d") + if v3d is None: + continue + v3d_np = np.asarray(v3d, dtype=np.float32) + if v3d_np.shape[0] < max(SMPLX_JOINT_ANCHOR_VERTS) + 1: + continue + j3d22 = v3d_np[list(SMPLX_JOINT_ANCHOR_VERTS)].astype(np.float32) + out.append((pid, j3d22)) + return out or None, t_smplx, "smplx", True + if t_body > self._last_body_t: + ids = ids_b3d or list(range(len(persons_b3d or []))) + out = [] + for i, body in enumerate(persons_b3d or []): + pid = int(ids[i]) if i < len(ids) else i + arr = self._kp_list_to_array(body) + if arr is None or arr.shape[0] < 33: + continue + j3d22 = arr[list(MEDIAPIPE_TO_22)].astype(np.float32) + out.append((pid, j3d22)) + return out or None, t_body, "body3d", True + return None, 0.0, "", False + + @staticmethod + def _kp_list_to_array(body: Any) -> np.ndarray | None: + """Best-effort conversion of a body keypoint list to (N, 3) array.""" + if body is None: + return None + if isinstance(body, np.ndarray): + return body + try: + return np.asarray( + [ + ( + getattr(kp, "x", kp[0]), + getattr(kp, "y", kp[1]), + getattr(kp, "z", kp[2] if len(kp) > 2 else 0.0), + ) + for kp in body + ], + dtype=np.float32, + ) + except (TypeError, IndexError, AttributeError): + return None diff --git a/data_only_viz/multi.py b/data_only_viz/multi.py index 0c7beaa..3f6ed0c 100644 --- a/data_only_viz/multi.py +++ b/data_only_viz/multi.py @@ -19,6 +19,7 @@ import time import urllib.request from pathlib import Path +from .action_head_pub import ActionHeadPublisher from .euro_filter import SkeletonFilter from .pose_bridge import PoseSoundBridge from .state import Kp3D, PoseKp, State @@ -93,6 +94,8 @@ class MultiWorker: self._smooth_hand = SkeletonFilter(min_cutoff=2.0, beta=0.10) # Pont OSC pose -> sclang self._sound_bridge = PoseSoundBridge(throttle_hz=30.0) + self._action_pub = ActionHeadPublisher(state=self.state, bridge=self._sound_bridge) + self._action_pub.start() def start(self) -> None: self._thread = threading.Thread( diff --git a/data_only_viz/tests/test_action_head_pub.py b/data_only_viz/tests/test_action_head_pub.py new file mode 100644 index 0000000..a630999 --- /dev/null +++ b/data_only_viz/tests/test_action_head_pub.py @@ -0,0 +1,84 @@ +"""Tests for ActionHeadPublisher.""" +from __future__ import annotations + +import threading +from unittest.mock import MagicMock + +import numpy as np +import pytest + +torch = pytest.importorskip("torch") + + +class _FakeState: + def __init__(self) -> None: + self.persons_smplx = [] + self.smplx_last_t = 0.0 + self.persons_body3d = [] + self.persons_body_ids = [] + self.pose_last_t = 0.0 + self._lock = threading.RLock() + + def lock(self): + return self._lock + + +def _make_smplx_person(pid: int, seed: int = 0) -> dict: + rng = np.random.default_rng(seed) + return {"pid": pid, "v3d": rng.normal(size=(10475, 3)).astype(np.float32)} + + +def test_publisher_smplx_source_emits_osc() -> None: + from data_only_viz.action_head_pub import ActionHeadPublisher + state = _FakeState() + bridge = MagicMock() + pub = ActionHeadPublisher(state, bridge, ckpt_path=None) + state.persons_smplx = [_make_smplx_person(7)] + state.smplx_last_t = 1.0 + pub._tick(t_now=0.0) + actions = [c for c in bridge.send_action.call_args_list] + assert len(actions) == 1 + assert actions[0].kwargs.get("pid", actions[0].args[0]) == 7 + bridge.send_enter.assert_called_with(pid=7) + + +def test_publisher_falls_back_to_mediapipe_body3d() -> None: + from data_only_viz.action_head_pub import ActionHeadPublisher + state = _FakeState() + bridge = MagicMock() + pub = ActionHeadPublisher(state, bridge, ckpt_path=None) + state.persons_body3d = [[(0.1 * i, 0.2 * i, 0.3 * i) for i in range(33)]] + state.persons_body_ids = [42] + state.pose_last_t = 1.0 + pub._tick(t_now=0.0) + bridge.send_action.assert_called_once() + bridge.send_enter.assert_called_with(pid=42) + + +def test_publisher_purges_lost_pid() -> None: + from data_only_viz.action_head_pub import ActionHeadPublisher + state = _FakeState() + bridge = MagicMock() + pub = ActionHeadPublisher(state, bridge, ckpt_path=None) + state.persons_smplx = [_make_smplx_person(1)] + state.smplx_last_t = 1.0 + pub._tick(t_now=0.0) + bridge.reset_mock() + state.persons_smplx = [] + state.smplx_last_t = 2.0 + state.persons_body3d = [] + pub._tick(t_now=1.0) + bridge.send_leave.assert_called_with(pid=1) + + +def test_publisher_no_double_emit_same_timestamp() -> None: + from data_only_viz.action_head_pub import ActionHeadPublisher + state = _FakeState() + bridge = MagicMock() + pub = ActionHeadPublisher(state, bridge, ckpt_path=None) + state.persons_smplx = [_make_smplx_person(1)] + state.smplx_last_t = 1.0 + pub._tick(t_now=0.0) + bridge.reset_mock() + pub._tick(t_now=1.0) # same smplx_last_t + bridge.send_action.assert_not_called()