From 8e05bc575131597bb523bb21deace7439a3aedc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Wed, 13 May 2026 14:27:16 +0200 Subject: [PATCH] perf(data-only-viz): swap Multi-HMR to ViT-S 672 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-HMR ViT-S is the only S variant published (672x672 only). Camera bench on M5 MPS: 3.8 fps median (228 ms/frame), no speedup vs ViT-L — bottleneck is SMPL-X head, not DINOv2 backbone. Heartbeat moved before no-detect early-return so FPS metric stays meaningful with empty camera view. --- data_only_viz/MULTIHMR_README.md | 21 +++++++---- data_only_viz/multi_hmr_worker.py | 36 ++++++++++--------- data_only_viz/scripts/setup_multihmr.sh | 8 ++--- ...026-05-13-multihmr-realitykit-body-mesh.md | 2 +- 4 files changed, 38 insertions(+), 29 deletions(-) diff --git a/data_only_viz/MULTIHMR_README.md b/data_only_viz/MULTIHMR_README.md index 23bd1dd..2695ded 100644 --- a/data_only_viz/MULTIHMR_README.md +++ b/data_only_viz/MULTIHMR_README.md @@ -40,10 +40,10 @@ data_only_viz/.venv/bin/python -m data_only_viz.main \ ## Architecture ``` -webcam Mac (cv2 idx 0, 896x896) +webcam Mac (cv2 idx 0, 672x672) | v -Multi-HMR ViT-L (PyTorch MPS, ~317M params) +Multi-HMR ViT-S (PyTorch MPS, ~31.5M params) | v humans = [{v3d (10475,3), j3d, transl, shape, expression, ...}, ...] @@ -81,9 +81,16 @@ AVLiveBody Swift ## FPS attendu -- Multi-HMR ViT-L MPS : ~150-200 ms / frame = 5-7 fps -- ViT-B (`multiHMR_672_B.pt`) : ~80 ms = 12 fps si on accepte moins - de precision +- Multi-HMR ViT-S MPS mesure 2026-05-13 : + - bench headless (dummy 672x672, 20 iter) : 199 ms median = 5.0 fps + - bench camera live (30 s real capture) : 228 ms median = 3.8 fps + (overhead capture/pre/tensor ~30 ms) +- Pas de speedup significatif vs ViT-L : le bottleneck n'est PAS le + backbone DINOv2 sur MPS — probablement la tete SMPL-X (identique + entre variantes S/B/L) et l'absence de SDPA fused / xFormers sur MPS. + La piste "ViT-S pour gagner du FPS" est invalidee par mesure. +- Fallback ViT-L (`multiHMR_896_L.pt`) : ~150-200 ms = 5-7 fps si + precision insuffisante avec ViT-S - TCP sender throttle a 12 fps cible - RealityKit render : 60 fps Cocoa, interpole entre frames Multi-HMR @@ -92,7 +99,7 @@ AVLiveBody Swift ``` ~/.cache/av-live-multihmr/ ├── checkpoints/ -│ └── multiHMR_896_L.pt (1.28 GB) +│ └── multiHMR_672_S.pt (124 MB) ├── models/ │ ├── smplx/ │ │ ├── SMPLX_NEUTRAL.npz (108 MB) @@ -109,7 +116,7 @@ AVLiveBody Swift | Symptome | Verification | |----------|--------------| -| Pas de mesh visible | `ls ~/.cache/av-live-multihmr/checkpoints/multiHMR_896_L.pt` | +| Pas de mesh visible | `ls ~/.cache/av-live-multihmr/checkpoints/multiHMR_672_S.pt` | | `Multi-HMR load failed` | `~/.cache/av-live-multihmr/models/smplx/SMPLX_NEUTRAL.npz` present ? | | `TCP refused on :57130` | Lancer l'app Swift AVANT le worker Python | | FPS trop bas | Switcher vers ViT-B en editant `CKPT` dans `multi_hmr_worker.py` | diff --git a/data_only_viz/multi_hmr_worker.py b/data_only_viz/multi_hmr_worker.py index 1262924..8b6eed0 100644 --- a/data_only_viz/multi_hmr_worker.py +++ b/data_only_viz/multi_hmr_worker.py @@ -6,7 +6,7 @@ sys.path au runtime. Chaque humain renvoye contient deja les vertices SMPL-X decodes (cle `v3d`, shape (10475, 3)) ; pas besoin du decoder SMPL-X separe en hot path (il reste utile pour les tests). -Cadence cible : 8-12 fps sur M5 (ViT-L). Lissage One Euro sur les +Cadence cible : 8-12 fps sur M5 (ViT-S). Lissage One Euro sur les shapes/expression pour limiter le jitter trame-a-trame. """ from __future__ import annotations @@ -27,11 +27,11 @@ from .tracker import IoUTracker LOG = logging.getLogger("multi_hmr") CACHE = Path.home() / ".cache" / "av-live-multihmr" -CKPT = CACHE / "checkpoints" / "multiHMR_896_L.pt" +CKPT = CACHE / "checkpoints" / "multiHMR_672_S.pt" SMPLX_PATH = CACHE / "models" / "smplx" / "SMPLX_NEUTRAL.npz" MULTIHMR_REPO = CACHE / "multi-hmr" -IMG_SIZE = 896 +IMG_SIZE = 672 N_VERTS = 10475 @@ -191,14 +191,28 @@ class MultiHMRWorker: continue t_post_start = time.monotonic() + t_now = time.monotonic() + # Count frame + heartbeat regardless of detection — keeps the + # FPS metric meaningful when nobody is in the camera view. + frame_count += 1 + persons_count += len(humans) if humans else 0 + if t_now >= next_heartbeat: + fps = frame_count / 5.0 + avg = persons_count / max(1, frame_count) + LOG.info( + "hb: %.1f fps, %.2f persons/frame (%d frames)", + fps, avg, frame_count) + frame_count = 0 + persons_count = 0 + next_heartbeat = t_now + 5.0 if not humans: with self.state.lock(): self.state.persons_smplx = [] + inf_ms = (t_post_start - t_inf_start) * 1e3 + LOG.debug("frame (no detect): inf=%.1fms", inf_ms) time.sleep(self.period) continue - t_now = time.monotonic() - # Tracking via bbox approximee depuis verts projetes (xy) bboxes = [] n_keep = min(len(humans), self.num_persons) @@ -263,18 +277,6 @@ class MultiHMRWorker: dt_total, ) - frame_count += 1 - persons_count += len(persons) - if t_now >= next_heartbeat: - fps = frame_count / 5.0 - avg = persons_count / max(1, frame_count) - LOG.info( - "hb: %.1f fps, %.2f persons/frame (%d frames)", - fps, avg, frame_count) - frame_count = 0 - persons_count = 0 - next_heartbeat = t_now + 5.0 - dt = time.monotonic() - t_cap_start if dt < self.period: time.sleep(self.period - dt) diff --git a/data_only_viz/scripts/setup_multihmr.sh b/data_only_viz/scripts/setup_multihmr.sh index 5a79438..b51df54 100755 --- a/data_only_viz/scripts/setup_multihmr.sh +++ b/data_only_viz/scripts/setup_multihmr.sh @@ -11,16 +11,16 @@ if [ ! -d "$CACHE/multi-hmr" ]; then git clone --depth=1 https://github.com/naver/multi-hmr.git "$CACHE/multi-hmr" fi -CKPT="$CACHE/checkpoints/multiHMR_896_L.pt" +CKPT="$CACHE/checkpoints/multiHMR_672_S.pt" if [ ! -f "$CKPT" ]; then - echo "==> Telechargement checkpoint multiHMR_896_L (ViT-L)" + echo "==> Telechargement checkpoint multiHMR_672_S (ViT-S)" # Source primaire : Naver Labs Europe ; fallback : HuggingFace mirror if ! curl -fL --progress-bar \ - "https://download.europe.naverlabs.com/ComputerVision/MultiHMR/multiHMR_896_L.pt" \ + "https://download.europe.naverlabs.com/ComputerVision/MultiHMR/multiHMR_672_S.pt" \ -o "$CKPT"; then echo "==> Fallback HuggingFace" curl -fL --progress-bar \ - "https://huggingface.co/naver/multiHMR_896_L/resolve/main/multiHMR_896_L.pt" \ + "https://huggingface.co/naver/multiHMR_672_S/resolve/main/multiHMR_672_S.pt" \ -o "$CKPT" fi fi diff --git a/docs/superpowers/plans/2026-05-13-multihmr-realitykit-body-mesh.md b/docs/superpowers/plans/2026-05-13-multihmr-realitykit-body-mesh.md index dce1b68..3f33957 100644 --- a/docs/superpowers/plans/2026-05-13-multihmr-realitykit-body-mesh.md +++ b/docs/superpowers/plans/2026-05-13-multihmr-realitykit-body-mesh.md @@ -1578,7 +1578,7 @@ RealityKit ARView : mesh skinné en couleur par pid - Pas de mesh affiché : vérifier `~/.cache/av-live-multihmr/checkpoints/multiHMR_*.pt` existe - Pas de SMPL-X : vérifier `~/.cache/av-live-multihmr/models/smplx/SMPLX_NEUTRAL.npz` - TCP refused : l'app Swift doit tourner AVANT le worker Python -- FPS bas : passer à ViT-S (`multiHMR_896_S_*.pt`) qui est plus rapide +- Precision insuffisante : passer à ViT-L (`multiHMR_896_L.pt`) qui est plus précis (default est ViT-S depuis 2026-05-13) ``` - [ ] **Step 3: Commit final**