Compare commits
154 Commits
v0.1.0
...
v0.1.0-rc1
| Author | SHA1 | Date | |
|---|---|---|---|
| b6cd8cf4c7 | |||
| bd46f6e052 | |||
| 1828d7ca3e | |||
| 67302e7ca2 | |||
| 9838da3137 | |||
| 91f4a46ceb | |||
| c4bc5c2e5a | |||
| 7ed2e2764a | |||
| aad56e5bf9 | |||
| 06f2a55f08 | |||
| 4e7101c54e | |||
| 87b76a42d0 | |||
| 52588b9910 | |||
| 4717da385c | |||
| 6af220dd59 | |||
| 1f623fe331 | |||
| beb94d2a4c | |||
| 623c47983d | |||
| 2c8094c06c | |||
| aedcb0f01b | |||
| 28d562b11c | |||
| 31ba587a63 | |||
| 2a732faffc | |||
| b5e9139317 | |||
| 5013a1916d | |||
| a5c793f39c | |||
| f540158f45 | |||
| b53c748704 | |||
| 0ecb2c3d3b | |||
| a1ea343ff0 | |||
| dfafd23d5a | |||
| c381d0c4e7 | |||
| e292ed7ef3 | |||
| 9d67426b2c | |||
| 744bc4a8a4 | |||
| 9e7a9f8fd4 | |||
| 24d1e85b7d | |||
| f0f79b3478 | |||
| 214b1545c4 | |||
| 96a326d3c6 | |||
| 667f63c385 | |||
| 7f0ac97a21 | |||
| 9ab82e47aa | |||
| a199c50297 | |||
| b410023d03 | |||
| 2d92cceef9 | |||
| bd3dd89511 | |||
| 50343be6ec | |||
| 17d93fd351 | |||
| ee86990195 | |||
| bcb259d045 | |||
| 46d619199b | |||
| 2a26e2dfd4 | |||
| 9d15c74c01 | |||
| 55a0390c77 | |||
| a4706d68d8 | |||
| 4fd37b4faf | |||
| b361c1c09b | |||
| c650915f8a | |||
| 651187f097 | |||
| 7162f76d6f | |||
| e0cf814e72 | |||
| af0504f713 | |||
| 1fed0ad485 | |||
| 788f1fe0aa | |||
| b21651dbe6 | |||
| 60112b0790 | |||
| ae5d814128 | |||
| 1e2d9314a7 | |||
| f581cc3f55 | |||
| dc7de90b18 | |||
| 278ed55f02 | |||
| b88dd6ceef | |||
| 0293cdea17 | |||
| 0eebcb3aec | |||
| 39d8739f4c | |||
| 8e05bc5751 | |||
| 29ea203c67 | |||
| c631f62a27 | |||
| 1ed006966c | |||
| 300d501f67 | |||
| a276d00bc6 | |||
| 9cbcbd2025 | |||
| c924847dfc | |||
| d4ac1da686 | |||
| 1ec57ae3f9 | |||
| daba927b1e | |||
| 44bc0f0d28 | |||
| e3ad38eafd | |||
| fcd7857c44 | |||
| fca86269e2 | |||
| 0b8a21e6aa | |||
| 72006d92b7 | |||
| 041c6b42be | |||
| dce27dc5a2 | |||
| 085d5806e4 | |||
| be6d0b39b0 | |||
| c2777fc6f9 | |||
| 62fd30e936 | |||
| d266a98ef2 | |||
| 8294c56486 | |||
| b0d3ec15aa | |||
| 722208e0eb | |||
| 67dc83b34e | |||
| 352eab18cc | |||
| c49f171334 | |||
| b6badea216 | |||
| 934bf99ba7 | |||
| e5e133a26b | |||
| 18368b2ded | |||
| a1b3ba1f91 | |||
| ae72d963a4 | |||
| 498ca2be70 | |||
| d033282aa3 | |||
| 052d1d0927 | |||
| a69d096d15 | |||
| 20acbeb704 | |||
| 5eefe7baa3 | |||
| 146fc335cf | |||
| f58f1d40e8 | |||
| 0d1e97865f | |||
| cfab3c908a | |||
| 511881f56d | |||
| b6b381bc29 | |||
| a682985065 | |||
| 500abb937f | |||
| 7378b879fa | |||
| c61d25a8b8 | |||
| d0c29d9a7f | |||
| ed2e0eb9fe | |||
| 8cd4c1bf8e | |||
| 358f00d040 | |||
| 403bb20d53 | |||
| 2fdda086c2 | |||
| b639ce7823 | |||
| 0825b35463 | |||
| ef288a7664 | |||
| 6932de5e74 | |||
| e1880053c8 | |||
| 304b58b553 | |||
| 2c858bfc52 | |||
| eb8afd5bac | |||
| cba128a079 | |||
| a46b408fee | |||
| 1c6bf1039c | |||
| 8237be9b15 | |||
| 0497a8951a | |||
| ee434b4b96 | |||
| f0a38b3cca | |||
| fac0f7e055 | |||
| 8913393e4c | |||
| 4d32e61bae | |||
| 02de5302c9 | |||
| 0d189a2139 |
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(awk 'BEGIN{p=0;b=0;tlb=0;depth=0} /^\\\\\\(/{if\\(depth==0\\)tlb++; depth++} {for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"\\(\"\\)p++; else if\\(c==\"\\)\"\\)p--; else if\\(c==\"[\"\\)b++; else if\\(c==\"]\"\\)b--}} END{print \"P:\" p \" B:\" b \" TLB:\" tlb}' sound_algo/control/data_feeds.scd)",
|
||||
"Bash(awk '{for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"\\(\"\\)p++; else if\\(c==\"\\)\"\\)p--; else if\\(c==\"[\"\\)b++; else if\\(c==\"]\"\\)b--}} END{print \"P:\" p \" B:\" b}' sound_algo/examples/16_data_feeds.scd sound_algo/control/data_feeds.scd)",
|
||||
"Bash(awk '{for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"\\(\"\\)p++; else if\\(c==\"\\)\"\\)p--; else if\\(c==\"[\"\\)b++; else if\\(c==\"]\"\\)b--}} END{print \"P:\" p \" B:\" b}' sound_algo/control/data_feeds.scd)",
|
||||
"Bash(find . -name \"*.yml\" -path \"*workflows*\" 2>/dev/null *)",
|
||||
"Read(//Applications/**)",
|
||||
"Bash(swift build *)",
|
||||
"Bash(./build.sh)",
|
||||
"Bash(open build/AVLiveLauncher.app)",
|
||||
"Bash(defaults delete *)",
|
||||
"Bash(tccutil reset *)",
|
||||
"Bash(awk '{for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"\\(\"\\)p++; else if\\(c==\"\\)\"\\)p--; else if\\(c==\"[\"\\)b++; else if\\(c==\"]\"\\)b--}} END{print FILENAME\": P:\" p \" B:\" b}' sound_algo/boot.data-only.scd sound_algo/live/data_only_program.scd)",
|
||||
"Bash(awk '{for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"\\(\"\\)p++; else if\\(c==\"\\)\"\\)p--; else if\\(c==\"[\"\\)b++; else if\\(c==\"]\"\\)b--}} END{print \"P:\" p \" B:\" b}' sound_algo/boot.data-only.scd)",
|
||||
"Bash(awk '{for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"\\(\"\\)p++; else if\\(c==\"\\)\"\\)p--}; if\\(p<0||b<0\\) print NR\": \"$0\" [p=\"p\"]\"}' sound_algo/live/data_only_program.scd)",
|
||||
"Bash(awk '{ *)",
|
||||
"Bash(awk '{l=$0; i=index\\(l,\"//\"\\); if\\(i>0\\)l=substr\\(l,1,i-1\\); *)",
|
||||
"Bash(open launcher/build/AVLiveLauncher.app)",
|
||||
"Bash(brew search *)",
|
||||
"Bash(brew info *)",
|
||||
"Bash(ls /Applications/of_v* 2>/dev/null | head -3 *)",
|
||||
"Read(//opt/homebrew/Caskroom/**)",
|
||||
"Bash(awk '{for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"{\"\\)o++; else if\\(c==\"}\"\\)cl++}} END{print \"metal braces O:\" o \" C:\" cl}' data_only_viz/shaders/scene.metal)",
|
||||
"Bash(data_only_viz/.venv/bin/python -c ' *)",
|
||||
"Bash(data_only_viz/.venv/bin/python -m data_only_viz.main -v)",
|
||||
"Bash(echo \"PID=$!\")",
|
||||
"Bash(kill 73822)",
|
||||
"Bash(timeout 3 netstat -an)",
|
||||
"Bash(netstat -an -p udp)",
|
||||
"Bash(host mainsfrequenz.de)",
|
||||
"Bash(host www.mainsfrequenz.de)",
|
||||
"Bash(echo \"feeds PID=$!\")",
|
||||
"Bash(/Applications/SuperCollider.app/Contents/MacOS/sclang sound_algo/data_only/boot.scd)",
|
||||
"Bash(awk '{print $2, $11, $12, $13}')",
|
||||
"Bash(xargs -r kill -9)",
|
||||
"Bash(kill -9 95574 95565 95564)",
|
||||
"Bash(awk '{for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"{\"\\)o++; else if\\(c==\"}\"\\)cl++}} END{print \"{:\" o \" }:\" cl}' data_only_viz/shaders/scene.metal)",
|
||||
"Bash(data_only_viz/.venv/bin/python)",
|
||||
"Bash(open /Users/electron/Documents/Projets/AV-Live/launcher/build/AVLiveLauncher.app)",
|
||||
"Bash(awk '{printf \"%-7s %s\\\\n\", $2, substr\\($0, index\\($0,$11\\)\\)}')",
|
||||
"Bash(ps -p 23891 -o pid,etime,command)",
|
||||
"Bash(awk '{print $2,$11,$12,$13,$14}')",
|
||||
"Bash(data_only_viz/.venv/bin/python -c \"import data_only_viz.main; print\\('main OK'\\)\")",
|
||||
"Bash(awk '{printf \"%-6s %s\\\\n\", $2, substr\\($0, index\\($0,$11\\)\\)}')",
|
||||
"Bash(awk '{print $2, $11, $12}')",
|
||||
"Bash(awk '{print $2, $11, $12, $13, $14}')",
|
||||
"Bash(xargs -r -n1 ps -o pid,etime,pcpu= -p)",
|
||||
"Bash(data_only_viz/.venv/bin/python -c \"from data_only_viz import main; print\\('main OK'\\)\")",
|
||||
"Bash(data_only_viz/.venv/bin/python -m data_only_viz.main -v --pose)",
|
||||
"Bash(defaults write *)",
|
||||
"Bash(awk '{for\\(i=1;i<=length\\($0\\);i++\\){c=substr\\($0,i,1\\); if\\(c==\"{\"\\)o++; else if\\(c==\"}\"\\)cl++}} END{print \"{:\"o\" }:\"cl}' data_only_viz/shaders/scene.metal)",
|
||||
"Bash(ps -p 76056 -o pcpu,pmem)",
|
||||
"Bash(gh release *)",
|
||||
"Bash(data_only_viz/.venv/bin/python -c \"from data_only_viz import detrpose; print\\('import OK'\\); print\\('is_available:', detrpose.is_available\\(\\)\\); print\\('REPO_DIR:', detrpose.REPO_DIR\\); print\\('CKPT:', detrpose.DEFAULT_CKPT\\)\")",
|
||||
"Bash(/Users/electron/Documents/Projets/AV-Live/data_only_viz/.venv/bin/python -c \"import Vision; print\\(dir\\(Vision\\)\\)\")",
|
||||
"Bash(/Users/electron/Documents/Projets/AV-Live/data_only_viz/.venv/bin/python -c ' *)",
|
||||
"Bash(data_only_viz/.venv/bin/python -m data_only_viz.scripts.convert_coreml)",
|
||||
"Bash(ps -p 28282 -o pcpu,pmem)",
|
||||
"Bash(/tmp/av-coreml-export/bin/python -c \"import coremltools; print\\(coremltools.__version__\\)\")",
|
||||
"Bash(/tmp/av-coreml-export/bin/python -c ' *)",
|
||||
"Bash(xcrun -sdk macosx metal -c /Users/electron/Documents/Projets/AV-Live/data_only_viz/shaders/scene.metal -o /tmp/scene.air)",
|
||||
"Bash(sed -i '' 's/A\\\\.c < 0\\\\.3 or B\\\\.c < 0\\\\.3 or C\\\\.c < 0\\\\.3/A.c < 0.15 or B.c < 0.15 or C.c < 0.15/' data_only_viz/renderer.py)",
|
||||
"Bash(sed -i '' 's/A\\\\.c < 0\\\\.3 or B\\\\.c < 0\\\\.3: continue/A.c < 0.15 or B.c < 0.15: continue/g' data_only_viz/renderer.py)"
|
||||
]
|
||||
}
|
||||
}
|
||||
+11
@@ -28,6 +28,17 @@ oscope-of/Project.xcconfig
|
||||
sound_algo/synthdefs/*.scsyndef
|
||||
sound_algo/*.log
|
||||
|
||||
# ML model weights downloaded at runtime
|
||||
*.pt
|
||||
*.ckpt
|
||||
*.safetensors
|
||||
*.mlpackage
|
||||
*.onnx
|
||||
*.gguf
|
||||
|
||||
# Claude MCP artifacts
|
||||
.playwright-mcp/
|
||||
|
||||
# Editors
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
[submodule "third_party/SMPLer-X"]
|
||||
path = third_party/SMPLer-X
|
||||
url = https://github.com/electron-rare/SMPLer-X.git
|
||||
@@ -0,0 +1,64 @@
|
||||
# AV-Live
|
||||
|
||||
Live coding audio-visual performance system : moteur SuperCollider, visualiseur openFrameworks piloté par un oscilloscope Hantek 6022BL, app menubar macOS qui orchestre le tout. **RC0.1+ (tag `v0.1.0-rc1`)** ajoute le pipeline Multi-HMR distribué M5 ↔ macm1 sur LAN gigabit + 10 scènes Metal pose-réactives + unified 3D armature wireframe (body+face+hands).
|
||||
|
||||
## Communication
|
||||
|
||||
Toujours répondre en français à l'utilisateur. Code, commentaires de code, commits et docs en anglais.
|
||||
|
||||
## Stack par sous-projet
|
||||
|
||||
| Sous-projet | Stack |
|
||||
|-------------|-------|
|
||||
| `sound_algo/` | SuperCollider (sclang + scsynth), 1099 SynthDefs, 345 tracks |
|
||||
| `oscope-of/` | openFrameworks C++, libusb (Hantek bulk), GLSL 150 GL 3.2 core |
|
||||
| `launcher/` | SwiftUI menubar app, Swift Package Manager |
|
||||
| `data_only_viz/` | Python 3.11+ via `uv`, Metal natif (pyobjc), multi-backends pose |
|
||||
| `web_realart/` | Node.js, Express, OSC bridge |
|
||||
|
||||
## Where to Look
|
||||
|
||||
| Tâche | Emplacement |
|
||||
|-------|-------------|
|
||||
| Ajouter / modifier un SynthDef, track, palette | `sound_algo/` (déjà documenté en nested) |
|
||||
| Toucher au visualiseur, shaders, FFT, Hantek | `oscope-of/` |
|
||||
| Modifier l'app menubar macOS | `launcher/` |
|
||||
| Détection pose / mesh / body tracking | `data_only_viz/` |
|
||||
| Bridge web / UI de live coding | `web_realart/` |
|
||||
| Plans / specs en cours | `docs/superpowers/plans/` |
|
||||
| Multi-HMR remote server pyobjc | `data_only_viz/scripts/multihmr_server.py` (tourne sur macm1) |
|
||||
| Mesh dense rigger 27 fps perçu | `data_only_viz/mesh_rigger.py` |
|
||||
| 3D wireframe armature (body+face+hands) | `launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift` |
|
||||
| Pose → Metal scenes uniforms | `launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift` + `Resources/scene.metal` |
|
||||
| Filter chain (median + Kalman + lookahead + IK) | `data_only_viz/pose_filter.py` |
|
||||
| DINO re-id pid matching | `data_only_viz/dino_reid.py` |
|
||||
|
||||
## RC0.1+ environment variables
|
||||
|
||||
| Env | Default | Effect |
|
||||
|-----|---------|--------|
|
||||
| `MULTIHMR_BACKEND` | `pytorch` | `pytorch`, `coreml`, `remote` |
|
||||
| `MULTIHMR_REMOTE_HOST` | `127.0.0.1` | macm1 IP for remote inference |
|
||||
| `MULTIHMR_REMOTE_JPEG` | `1` | JPEG q=80 on the wire |
|
||||
| `MULTIHMR_REMOTE_ASYNC` | `1` | client double-buffer queue |
|
||||
| `MULTIHMR_SERVER_BACKEND` | `pyobjc` | server: `pyobjc` or `coremltools` |
|
||||
| `MULTIHMR_LOOP_FPS` | `30` | Python worker loop target_fps |
|
||||
| `AVBODY_HOST` | `127.0.0.1` | route TCP mesh + OSC to remote AVLiveBody |
|
||||
| `MEDIAPIPE_DELEGATE` | `cpu` | `gpu` Metal SRGBA (faster, flake on M5) |
|
||||
| `POSE_FILTER` | `median+kalman+lookahead+ik` | filter chain stages |
|
||||
| `MULTIHMR_REID` | `dino` | DINO cosine matching, `iou` fallback |
|
||||
|
||||
## Conventions globales
|
||||
|
||||
- Python : **uv** systématiquement (jamais pip/poetry/conda directs).
|
||||
- Pas d'emojis dans code/docs/commits sauf demande explicite.
|
||||
- Commits : sujet ≤ 50 char, body ≤ 72 char/ligne, pas d'attribution AI, pas de `--no-verify`, pas d'underscore dans le scope (hooks enforcent).
|
||||
- `*.pt`, `*.ckpt`, `*.safetensors`, `*.mlpackage` exclus par `.gitignore` racine.
|
||||
|
||||
## Agent Workflow
|
||||
|
||||
Explore localise → librarian lit (>500 lignes) → tu raisonnes → general-purpose implémente → validator vérifie. Lance les tâches indépendantes en parallèle.
|
||||
|
||||
## Guidance imbriquée
|
||||
|
||||
Chaque sous-projet majeur a son propre `CLAUDE.md`. Claude charge automatiquement le plus proche du fichier édité (« closest wins »). N'ajouter ici que ce qui s'applique à TOUS les sous-projets.
|
||||
@@ -0,0 +1,40 @@
|
||||
# AV-Live Third-Party Notices
|
||||
|
||||
AV-Live includes or depends on the following third-party works,
|
||||
which are licensed under their own terms.
|
||||
|
||||
## Body Mesh Recovery Models
|
||||
|
||||
### Multi-HMR (NAVER, 2025)
|
||||
|
||||
- **License**: NAVER Non-Commercial License
|
||||
(https://github.com/naver/multi-hmr/blob/main/LICENSE.txt)
|
||||
- **Use**: Non-commercial only. Body mesh recovery from camera input.
|
||||
- **Integration**: model checkpoint downloaded at runtime to
|
||||
`~/.cache/av-live-multihmr/`. No source code vendored.
|
||||
|
||||
### SMPLer-X (S-Lab, ECCV 2024)
|
||||
|
||||
- **License**: S-Lab License 1.0
|
||||
(`third_party/SMPLer-X/LICENSE` after submodule init)
|
||||
- **Use**: Non-commercial only. Whole-body mesh recovery (SMPL-X).
|
||||
- **Integration**: vendored as git submodule at
|
||||
`third_party/SMPLer-X` (fork `electron-rare/SMPLer-X`, kept in
|
||||
sync with `caizhongang/SMPLer-X` or its current upstream).
|
||||
- **Modifications in fork**: minimal patches for ARM macOS Python
|
||||
3.14 inference (replace mmdet detector with Ultralytics YOLO,
|
||||
shim `mmcv.Config` via `mmcv-lite`).
|
||||
|
||||
## Implication
|
||||
|
||||
**AV-Live integrates non-commercial-only model code from Multi-HMR
|
||||
and SMPLer-X.** Commercial deployment (paid performances, packaged
|
||||
installations sold to clients) requires either :
|
||||
|
||||
1. Obtaining commercial licenses from NAVER and S-Lab respectively, OR
|
||||
2. Replacing both integrations with MIT/Apache-licensed alternatives
|
||||
(see `docs/superpowers/plans/2026-05-13-modern-body-mesh-survey.md`
|
||||
for candidates like Fast-SAM-3D-Body which is MIT-licensed).
|
||||
|
||||
The AV-Live source code itself (sound_algo, oscope-of, launcher,
|
||||
data_only_viz integration glue) is under GPL-3.0 (see root LICENSE).
|
||||
@@ -1,49 +1,191 @@
|
||||
# AV-Live
|
||||
|
||||
> **Live coding audio-visual performance system** built around a SuperCollider sound engine, an openFrameworks oscilloscope visualizer driven by a real Hantek 6022BL USB scope, and a macOS menubar launcher that ties it all together.
|
||||
**Release : `v0.1.0-rc1` (2026-05-14)** — distributed Multi-HMR (M5 ↔ macm1 LAN), unified 3D armature openpos, hybrid mesh rigger 27 fps perceived, 10 pose-reactive Metal scenes. See [RC0.1+ architecture](#rc01-distributed-multi-hmr-architecture) below.
|
||||
|
||||
> **Live coding audio-visual performance system** built around a SuperCollider sound engine, an openFrameworks oscilloscope visualizer driven by a real Hantek 6022BL USB scope, a macOS menubar launcher, and a Metal-native pose / body-mesh visualizer that listens to the same audio bus.
|
||||
>
|
||||
> 15 scripted demoparties, 41 fullscreen visual backgrounds, 35 3D parametric meshes, 14 retro OS pixel-art tributes, 1099 SynthDefs across 345 tracks, all driven by FFT analysis of the actual audio physically passing through the scope probes.
|
||||
> 15 scripted demoparties · ~47 fullscreen visuals (26 3D parametric meshes + 18 procedural shaders + 3 dedicated C++ scenes) · 5 OS pixel-art shaders · 14 retro OS logos · 33 GLSL shader pairs (~65 files) · 1099 SynthDefs across 368 tracks (23 albums × 16) · **20 real-world data feeds** · 7 pose-estimation backends · **SMPL-X body mesh** (10 475 vertices via Multi-HMR + RealityKit) · **3 launch modes** (Full / Data-only / Body Mesh) — all reactive to the audio physically passing through the scope probes.
|
||||
|
||||
## Top-level architecture
|
||||
|
||||
ASCII (always renders):
|
||||
|
||||
```
|
||||
AV-LIVE
|
||||
┌──────────────────────────────────────────┐
|
||||
│ AVLiveLauncher.app (menubar) │
|
||||
└────┬───────────────┬──────────────┬──────┘
|
||||
│ │ │
|
||||
┌───────▼───────┐ ┌────▼─────┐ ┌────▼─────────────┐
|
||||
│ sclang + │ │ node │ │ oscope-of (oF) │
|
||||
│ scsynth │ │ web ui │ │ │
|
||||
│ sound_algo/ │ │ :3000 │ │ Hantek bulk USB │
|
||||
│ 1099 SynthDef │ │ │ │ FFT 2048 audio │
|
||||
│ 345 tracks │ │ │ │ 41 backgrounds │
|
||||
│ humanize+sig │ │ │ │ 15 demoparties │
|
||||
└───────┬───────┘ └────┬─────┘ └──────┬───────────┘
|
||||
:57121 OSC :3000 HTTP :57123 OSC in
|
||||
│ │ │
|
||||
└───────────────┴──────────────┘
|
||||
└────┬────────────┬─────────────┬──────┬───┘
|
||||
│ │ │ │
|
||||
┌───────▼──────┐ ┌───▼────┐ ┌──────▼───┐ ┌▼────────────┐
|
||||
│ sclang + │ │ node │ │ oscope-of│ │ data_only_viz│
|
||||
│ scsynth │ │ web ui │ │ (oF C++) │ │ (Metal py) │
|
||||
│ sound_algo/ │ │ :3000 │ │ Hantek │ │ webcam pose │
|
||||
│ 1099 SynthDef│ │ │ │ FFT 2048 │ │ 7 backends │
|
||||
│ 368 tracks │ │ │ │ ~47 bg │ │ NLF / SMPL │
|
||||
└───────┬──────┘ └───┬────┘ └──────┬───┘ └───┬─────────┘
|
||||
:57121 OSC :3000 HTTP :57123 OSC :57123 OSC out
|
||||
│ │ ▲ │
|
||||
└────────────┴─────────────┴──────────────┘
|
||||
▲
|
||||
data_feeds/ (11 sources OSC)
|
||||
```
|
||||
|
||||
Mermaid (rendered on GitHub):
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
L["AVLiveLauncher.app (menubar SwiftUI)"]
|
||||
SC["sound_algo/<br/>sclang + scsynth<br/>1099 SynthDefs · 368 tracks"]
|
||||
W["sound_algo/web/<br/>node :3000"]
|
||||
OF["oscope-of/<br/>openFrameworks C++<br/>Hantek 6022BL · FFT 2048<br/>~47 backgrounds · 33 shader pairs"]
|
||||
DOV["data_only_viz/<br/>Metal natif (pyobjc)<br/>7 pose backends · NLF/SMPL mesh"]
|
||||
DF["data_feeds/<br/>11 real-world sources"]
|
||||
WR["web_realart/<br/>WebGPU + Web Audio + Hydra"]
|
||||
|
||||
L -->|spawn| SC
|
||||
L -->|spawn| W
|
||||
L -->|spawn| OF
|
||||
L -->|spawn| DOV
|
||||
|
||||
SC -->|OSC :57123| OF
|
||||
SC -->|OSC :57123| DOV
|
||||
DOV -->|OSC pose →| SC
|
||||
DF -->|OSC :57121| SC
|
||||
DF -->|OSC :57123| OF
|
||||
DF -->|OSC :57124| WR
|
||||
|
||||
W <-->|WebSocket| SC
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
### 🎵 Sound engine — `sound_algo/`
|
||||
- **1099 SynthDefs** auto-loaded (acid_v1..v19, bass808_v1..v19, lead_v1..v19, kicks, drums, world, fx, master)
|
||||
- **23 albums × 15 tracks = 345 tracks** ready to play
|
||||
- **1099 SynthDefs** auto-loaded (acid_v1..v19, bass808_v1..v19, lead_v1..v19, kicks, drums, world, fx, master) — 1059 synth + 40 fx confirmed
|
||||
- **23 albums × 16 tracks = 368 tracks** ready to play (album letters A-W)
|
||||
- **Subtle humanize** : timing ±3.5 ms, velocity ±10 %, micro-detune ±3 cents per note
|
||||
- **Synth rotation** : cycles through 19 versions of each SynthDef every 8 notes
|
||||
- **Auto layering** : 3 voice layers parallel to melody/harmony/acid
|
||||
- **15 album signatures** with per-album palette, FX drives, layer config
|
||||
- **OSC bridge** to oscope-of on :57123 (kick/snare/melody/lead/bass/bpm)
|
||||
- **Web UI** on :3000 with live coding control surface
|
||||
- **OSC bridge** to oscope-of and data_only_viz on `:57123` (kick/snare/melody/lead/bass/bpm/album)
|
||||
- **Web UI** on `:3000` with live coding control surface (Express + WebSocket bridge)
|
||||
- **18 example scripts** in `sound_algo/examples/00..17_*.scd`
|
||||
|
||||
### 📺 Visualizer — `oscope-of/`
|
||||
- **Hantek 6022BL** real USB oscilloscope capture (1-48 MS/s, 2 channels, 8-bit)
|
||||
- **AudioAnalyzer** : downsamples Hantek to 48 kHz, FFT 2048 (~23 Hz/bin), bands bass/lowMid/mid/treble/kick/snare
|
||||
- **41 backgrounds** : tunnel 3D, starfield, metaballs, voronoi, twister bars, plasma FBM, rotozoom, truchet kaleidoscope, SDF tunnel raymarched, KIFS fractal, fire, perspective grid, tunnel cubes, caustics, vortex, octahedron chrome, vector cubes, plus Möbius/Klein/trefoil/Lucy/helix/catenoid/Boys/penrose/sphere/icosahedron/dodecahedron/torus/twisted torus/lemniscate/hopf/Lorenz attractor/Enneper/hopf link/supershape/gear/cone/pyramid/rose3D/geosphere/DNA helix/sphere wave 3D
|
||||
- **Hantek 6022BL** real USB oscilloscope capture (1–48 MS/s, 2 channels, 8-bit) via libusb bulk transfers
|
||||
- **AudioAnalyzer** : downsamples Hantek to 48 kHz, FFT 2048 (~23 Hz/bin), bands bass / lowMid / mid / treble / kick / snare
|
||||
- **~47 fullscreen backgrounds** (enum `BgKind` in `src/ofApp.h:98-113`, 53 enum values) :
|
||||
- **26 ModelVis 3D parametric meshes** : Möbius, Klein, trefoil, twisted torus, Lucy, helix, catenoid, Boys, lemniscate, Penrose, sphere, icosahedron, dodecahedron, torus, supershape, Lorenz attractor, Hopf, Enneper, Hopf link, gear, cone, pyramid, rose3D, geosphere, DNA helix, hyperboloid
|
||||
- **18 ShaderVis procedural** : metaballs, voronoi, twister bars, plasma FBM, rotozoom, truchet kaleido, SDF tunnel raymarched, KIFS fractal, fire, perspective grid, tunnel cubes, caustics, vortex, octahedron chrome, Boing ball, Mode7, plasma C64, dot tunnel
|
||||
- **3 dedicated C++ scenes** : TunnelVis, VectorCubesVis, SphereWaveVis
|
||||
- **+ starfield** (DemoFx) and **+ live webcam** (WebcamVis)
|
||||
- **5 OS pixel-art shaders** : Workbench (Amiga), Mac OS Classic, Atari Fuji, Ocean Loader, Win95
|
||||
- **14 OS logo PNGs** with demoscene FX (mirror reflection, ghost trail, RGB chroma, twister wobble, scanlines, glitch tear) : Win 1.0/3.11/95, Lotus 1-2-3, MS-DOS, Apple Rainbow, Amiga WB, NeXTSTEP, BeOS, OS/2 Warp, IRIX, ZX Spectrum, C64, Atari
|
||||
- **10 scroller styles** : Classic, Wavy3D, Rainbow, Mirror, Glitch, Neon, Cascade, Chrome, Bouncy, Squashy
|
||||
- **Demoscene FX** : sine scroller from `greetings.txt`, copper bars, logo bobs, starfield with motion blur
|
||||
- **Post-FX shader chain** : ACES tone-map, bloom, chromatic aberration, hue rotation, scanlines, vignette, grain, pixelate, kaleido, feedback FBO, glitch displacement, all GLSL 150 GL 3.2 core
|
||||
- **33 GLSL shader pairs (~65 files)** : 14 backgrounds + 5 OS pixel-art + 9 post-FX/transitions + 5 mesh/3D + shared utilities (`postfx.vert`, `transition.vert`), all GLSL 150 GL 3.2 core
|
||||
- **Post-FX shader chain** : ACES tone-map, bloom, chromatic aberration, hue rotation, scanlines, vignette, grain, pixelate, kaleido, feedback FBO, glitch displacement
|
||||
|
||||
### 🤸 Pose / body mesh — `data_only_viz/` *(new)*
|
||||
Metal-native fullscreen visualizer (pyobjc, MTKView, ~60 fps) driven by webcam pose detection. Listens to the same OSC bus as `oscope-of` (`:57123`) so audio sync and data feeds drive its shaders identically, **and** ships pose features back to `sound_algo` on `:57121` to close the loop (the dancer steers the synths).
|
||||
|
||||
- **7 pose backends** (drop-in via `pyproject.toml` extras) :
|
||||
|
||||
| Backend | Module | Model | Status |
|
||||
|---------|--------|-------|--------|
|
||||
| MediaPipe Holistic | `holistic.py` | 33 body + 478 face + 42 hand kp | stable |
|
||||
| YOLOv8-pose | `pose.py` | Ultralytics, 17 COCO kp | stable (MPS) |
|
||||
| Apple Vision | `apple_vision_pose.py` | VNDetectHumanBodyPoseRequest (ANE) | macOS only |
|
||||
| Core ML pose | `coreml_pose.py` | YOLO11n-pose CoreML, AVCapture zero-copy | stable |
|
||||
| DETRPose | `detrpose.py` | DETR transformer, COCO 17 kp, multi-person | manual clone + ckpt |
|
||||
| NLF | `nlf_worker.py` | SMPL body mesh, 6890 vertices, TorchScript | **CUDA-only** (CPU/MPS bricked) |
|
||||
| MediaPipe multi | `multi.py` | Pose/Face/Hand ×4 personnes | stable |
|
||||
|
||||
- **Renderer** : Metal pipelines compiled at runtime (`shaders/*.metal`), `bg_pipeline` (full-screen FBM) + `skel_pipeline` (skeleton lines). SMPL face topology shipped as binary (`mesh_topology.py`) for RealityKit-compatible mesh rendering.
|
||||
- **Tracker** : One Euro Filter on keypoints + IoU multi-person association (`scipy.linear_sum_assignment`, ByteTrack-like).
|
||||
- **OSC out → sclang** `:57121` : `/pose/count`, `/pose/center`, `/pose/wrist`, `/pose/head`, `/pose/sho_span`, `/pose/limb_span`.
|
||||
- **OSC out → AVLiveBody** `:57126` UDP (mode openpos, mode 9 / touche `p`) : `/pose/skel`, `/face/kp` (68 dlib landmarks), `/hand/kp` (21 × 2 hands), `/pose3d/kp` (33 MediaPipe pose_world_landmarks 3D meters).
|
||||
- **TCP out → AVLiveBody** `:57130` : SMPL-X dense mesh (10475 verts) frame-packed binary, 30 Hz rigged interpolation between Multi-HMR keyframes.
|
||||
- **Thread-safe state** : `state.py` exposes `State.lock()` ; dataclasses `PoseKp`, `Kp3D`, `SMPLXPerson`, multi-person container.
|
||||
|
||||
## RC0.1+ distributed Multi-HMR architecture
|
||||
|
||||
`feat/action-head` + tag `v0.1.0-rc1` (2026-05-14). The body-mesh pipeline is now **distributed across two Apple-silicon Macs over LAN gigabit** :
|
||||
|
||||
```
|
||||
M5 (capture host) macm1 (compute host)
|
||||
┌─────────────────────────────────┐ ┌──────────────────────────────────┐
|
||||
│ Caméra MacBook Pro │ │ Multi-HMR server :57140 (pyobjc) │
|
||||
│ data_only_viz/ │ JPEG q80 │ multihmr_full_672_s.mlpackage │
|
||||
│ ├─ multi_hmr_worker ├──TCP────▶│ ├─ Pyobjc direct CoreML.fwk │
|
||||
│ │ backend=remote (async) │ │ ├─ ~87 ms predict (M1 Max GPU) │
|
||||
│ ├─ MultiHMRRemoteBackend │◀──RSP───┤ └─ 6 outputs (v3d 10475, │
|
||||
│ │ queue maxsize 2/3 │ │ transl, scores, betas, │
|
||||
│ │ JPEG encode q80 │ │ expression, joints 127) │
|
||||
│ │ │ │ │
|
||||
│ ├─ MediaPipe Holistic │ │ AVLiveBody display (RealityKit) │
|
||||
│ │ Metal GPU delegate │ │ ├─ MeshRenderer (TCP :57130) │
|
||||
│ │ pose 33 + face 478 + │ /pose/* │ │ SMPL-X dense, low-level │
|
||||
│ │ hand 21×2 ├──UDP────▶│ │ mesh, 30 fps rigged interp │
|
||||
│ ├─ pose_bridge (UDP :57126) │ /face/* │ │ │
|
||||
│ ├─ smplx_tcp (TCP :57130, 30 Hz) │ /hand/* │ ├─ Skeleton3DRenderer (OSC) │
|
||||
│ ├─ MeshRigger (Hungarian + DINO │ /pose3d/│ │ 1 fused LowLevelMesh / │
|
||||
│ │ re-id, sticky pid 0.30/0.15) │ │ │ person : body 33 + face │
|
||||
│ └─ PoseFilterChain │ │ │ 68 + hand 21×2 = 143 vts │
|
||||
│ (median + Kalman CV + │ │ │ 288 line indices │
|
||||
│ lookahead + IK clamps) │ │ └─ 10 Metal scenes (storm, │
|
||||
│ │ │ tunnel, plasma, kaleido, │
|
||||
│ TCP loop fps : 25 │ │ voronoi, metaballs, │
|
||||
│ Multi-HMR fresh fps : 9 │ │ starfield, bars, hands3d, │
|
||||
│ MeshRigger perceived : 27 │ │ openpos), all consume │
|
||||
│ │ │ pose uniforms (mouth, │
|
||||
└─────────────────────────────────┘ │ velocity, head_tilt, │
|
||||
│ arm_spread, eye_open, │
|
||||
│ finger_pinch, body_xyz) │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Key technical wins (commits in `feat/action-head`)
|
||||
|
||||
| Commit | Win |
|
||||
|--------|-----|
|
||||
| `5800156` | Multi-HMR ViT-S/672 → CoreML mlpackage (FP32, 6 outputs) |
|
||||
| `52588b9` | roma.rotmat_to_rotvec → branchless atan2 (fixed all-NaN v3d/transl bug) |
|
||||
| `4e7101c` | NaN/Inf guard on v3d before TCP ship |
|
||||
| `2c8094c` | MeshRigger : 27 fps perceived via Hungarian pid match + Vision pelvis delta |
|
||||
| `1f623fe` | AVLiveBody mirror webcam preview (CATransform3D scale -1) |
|
||||
| `9838da3` | Remote inference protocol (TCP :57140) + multi-buffer async client |
|
||||
| `67302e7` | Pyobjc server (drops coremltools overhead) + MediaPipe Metal GPU SRGBA |
|
||||
| `1828d7c` | 12 new SceneUniforms (mouth, eye, head, finger, body) drive 5 scenes |
|
||||
| `bd46f6e` | Lift Python self-throttle 10 → 30 fps loop, fresh fps metric |
|
||||
| latest | 10 Metal scenes pose-reactive + SMPL-X 127 joints output (finger props) |
|
||||
|
||||
### Environment toggles
|
||||
|
||||
| Env var | Default | Effect |
|
||||
|---------|---------|--------|
|
||||
| `MULTIHMR_BACKEND` | `pytorch` | `pytorch`, `coreml`, `remote` |
|
||||
| `MULTIHMR_REMOTE_HOST` | `127.0.0.1` | macm1 IP (gigabit LAN) |
|
||||
| `MULTIHMR_REMOTE_JPEG` | `1` | JPEG q=80 compression on the wire |
|
||||
| `MULTIHMR_REMOTE_ASYNC` | `1` | client double-buffer queue (maxsize 2/3) |
|
||||
| `MULTIHMR_SERVER_BACKEND` | `pyobjc` | server : `pyobjc` or `coremltools` |
|
||||
| `MULTIHMR_LOOP_FPS` | `30` | Python loop target_fps (formerly capped at 10) |
|
||||
| `AVBODY_HOST` | `127.0.0.1` | route TCP mesh + OSC pose to a remote AVLiveBody |
|
||||
| `MEDIAPIPE_DELEGATE` | `cpu` | `gpu` Metal SRGBA (faster but IOSurface flake on M5) |
|
||||
| `POSE_FILTER` | `median+kalman+lookahead+ik` | toggle filter chain stages |
|
||||
| `MULTIHMR_REID` | `dino` if mlpackage present | `dino` (cosine match) or `iou` |
|
||||
| `MULTIHMR_REID_ALPHA` | `0.5` | IoU vs cosine weight (0=DINO only, 1=IoU only) |
|
||||
|
||||
### Hardware ceilings observed (M5 + M1 Max 32c GPU, LAN gigabit)
|
||||
|
||||
| Path | Predict | Live loop | Mesh rigged perceived |
|
||||
|------|---------|-----------|------------------------|
|
||||
| M5 local (PyTorch MPS) | 270 ms | 3.5 fps | 27 fps |
|
||||
| M5 local (CoreML FP32) | 139 ms | 6.8 fps | 27 fps |
|
||||
| Remote macm1 (idle GPU) | 53 ms | ~18 fps | 27 fps |
|
||||
| Remote macm1 (under MLX contention) | 87 ms | 25 fps loop / 9 fps fresh | 27 fps |
|
||||
| Studio M3 Ultra (80c GPU, *training only*) | est. 35 ms (~30 fps) | — | — |
|
||||
|
||||
Hard ceiling on macm1 ≈ 18 fps fresh predict (unified memory bandwidth + CoreML sync overhead) ; further gains require moving MLX servers off macm1 or quantising the model.
|
||||
|
||||
### 🎬 Demoparties — 15 narrative demos
|
||||
Each demoparty is a multi-act scripted show with unique scroller text, background sequence, scroller style, and SuperCollider album track :
|
||||
@@ -66,7 +208,7 @@ Each demoparty is a multi-act scripted show with unique scroller text, backgroun
|
||||
| `F9` | CUBE STORM | 8 | Q neurofunk_dnb |
|
||||
| `F10` | GRAND FINAL | 8 | L future_garage |
|
||||
|
||||
Each act lasts 12-55 s, transitions between acts trigger flash + glitch postfx + audio swap. Demos loop on their first act after the last.
|
||||
Each act lasts 12–55 s, transitions between acts trigger flash + glitch postfx + audio swap. Demos loop on their first act after the last.
|
||||
|
||||
### 🎛 Live FX control (Mac AZERTY FR friendly)
|
||||
- `a z e r t y u i o p` — 10 background scenes (BoingBall, Tunnel, Metaballs, Voronoi, Twister, KIFS, Fire, Mode7, Octahedron, PlasmaC64)
|
||||
@@ -78,8 +220,32 @@ Each act lasts 12-55 s, transitions between acts trigger flash + glitch postfx +
|
||||
- `Espace` — manual glitch pulse
|
||||
- `F1`–`F5` — fullscreen / GUI / postFx / autoglitch / reload shaders
|
||||
|
||||
### 🌐 Real-world data feeds — `data_feeds/` + `web_realart/` + `data_only_viz/web/`
|
||||
- **20 feed modules** in `data_feeds/feeds/` ingested by `bridge.py` and broadcast as OSC `/data/<source>/<sub>` to SC (`:57121`), oF (`:57123`) and the web bridge (`:57124`) :
|
||||
- **Geophysique** : USGS quakes · Smithsonian GVP volcanoes · Blitzortung lightning
|
||||
- **Meteo / Air** : Open-Meteo (temp/wind/rain) · OpenAQ (PM2.5/PM10/NO2/O3)
|
||||
- **Espace** : NOAA SWPC (solar wind, Bz IMF, Kp, X-ray) · ISS position (wheretheiss.at) · GCN astrophysics
|
||||
- **Mobilite** : OpenSky ADS-B · LiveATC listeners
|
||||
- **Energie / reseau** : Mainsfrequenz.de · RTE eCO2mix · NOAA tides + moon phase
|
||||
- **Social / numerique** : Bluesky firehose · Reddit /r/all + HackerNews top · Wikipedia EventStream · GDELT 2.0 events · GitHub · Bitcoin mempool
|
||||
- **Pose / webcam** : YOLOv8-pose
|
||||
- **SC presets** `sound_algo/examples/16_data_feeds.scd` and `17_data_feeds_more.scd` map each source to synthesis : Schumann cavity drone (foudre), aurora additive pad (Bz/wind/Kp), Netzfrequenz pulse kick, RTE 8-op carbon FM, OpenSky granular swarm
|
||||
- **Web standalone** `web_realart/` ports the visualizers and synths to the browser for `real.art.saillant.cc` :
|
||||
- **WebGPU + three.js TSL** (r171) globe with quake/strike/flight particles (auto-fallback WebGL2)
|
||||
- **Web Audio** ports of 5 SC SynthDef presets (cavity, mix, geo, aurora, pulse)
|
||||
- **Hydra** with 7 data-driven patches (aurora, quake, lightning, flightmap, gridpulse, solarwind, bskyrain)
|
||||
- All three layers share `feeds_client.js` (`window.feeds`) over one WebSocket
|
||||
|
||||
- **Web data-only** `data_only_viz/web/` (port `:3211`, Express + WebSocket) — **bidirectional** OSC bridge for the Data-only mode :
|
||||
- **`/dashboard.html`** — live cards + SVG sparklines for all 19 active feeds (USGS, GDELT, Wiki, ISS, tides + moon, ATC, ...) with severity classes (alert / warn / green)
|
||||
- **`/map.html`** — Leaflet dark fullscreen with markers ephemeres for geocoded feeds (quakes, lightning, planes, volcanoes, GDELT events) + persistent ISS marker
|
||||
- **`/control.html`** — 3-pane control surface : 7 synth/mix sliders (master, cutoff, reso, reverb, delay, tempo), 10 audio scene buttons (`/scene/play`), 9 visual mode buttons routed to oF (`/control/vizMode` → `:57123`), XY pad (`/xy/{x,y}`)
|
||||
- **SC retour** : `sound_algo/control/web_bridge.scd` listens on `:57121` for `/control/*` `/scene/*` `/xy/*` and pushes `/sync/bpm|beat|rms|voices` to web on `:57125` at 4 Hz
|
||||
|
||||
### 📡 Audio reactivity pipeline
|
||||
|
||||
ASCII :
|
||||
|
||||
```
|
||||
Hantek probes → bulk USB → ScopeRing CH1/CH2 (1-48 MS/s, 8-bit)
|
||||
↓
|
||||
@@ -96,6 +262,38 @@ Hantek probes → bulk USB → ScopeRing CH1/CH2 (1-48 MS/s, 8-bit)
|
||||
drives every visualizer parameter
|
||||
```
|
||||
|
||||
Mermaid :
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
H["Hantek 6022BL<br/>2× probes"] -->|bulk USB| R["ScopeRing<br/>CH1/CH2 1–48 MS/s 8-bit"]
|
||||
R --> A["AudioAnalyzer.update()"]
|
||||
A --> D["Downsample box-filter<br/>→ 48 kHz mono"]
|
||||
D --> F["Hann + FFT 2048<br/>~23 Hz/bin"]
|
||||
F --> B["Bands :<br/>bass · lowMid · mid · treble<br/>+ kick & snare transients"]
|
||||
B --> V["Visualizer params<br/>(backgrounds, post-FX, scroller)"]
|
||||
```
|
||||
|
||||
### 🤸 Pose pipeline — `data_only_viz` *(new)*
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
CAM["Webcam<br/>AVCaptureSession"] --> BE{"Backend<br/>(pyproject extras)"}
|
||||
BE -->|holistic| MP["MediaPipe Holistic"]
|
||||
BE -->|pose| YO["YOLOv8-pose"]
|
||||
BE -->|coreml_pose| CM["CoreML ANE"]
|
||||
BE -->|apple_vision_pose| AV["Apple Vision"]
|
||||
BE -->|detrpose| DT["DETRPose"]
|
||||
BE -->|nlf| NLF["NLF SMPL 6890v<br/>CUDA only"]
|
||||
MP & YO & CM & AV & DT & NLF --> KP["Keypoints / Mesh"]
|
||||
KP --> EF["One Euro Filter"]
|
||||
EF --> TR["Tracker IoU<br/>linear_sum_assignment"]
|
||||
TR --> ST["State (lock)<br/>PoseKp · NLFPerson"]
|
||||
ST --> MET["Metal renderer<br/>bg + skel pipelines"]
|
||||
ST --> OSC["OSC /pose/* →<br/>sclang :57121"]
|
||||
SCIN["sclang OSC :57123<br/>/sync /data"] --> ST
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
### macOS install
|
||||
@@ -107,7 +305,7 @@ cd AV-Live
|
||||
|
||||
# 2. Dependencies
|
||||
brew install --cask supercollider
|
||||
brew install libusb node
|
||||
brew install libusb node uv
|
||||
# openFrameworks 0.12 : https://openframeworks.cc/download/ (extract to ~/of)
|
||||
|
||||
# 3. Build the launcher (universal binary arm64+x86_64)
|
||||
@@ -118,19 +316,32 @@ ln -s "$PWD/../oscope-of" ~/of/apps/myApps/oscope-of
|
||||
cd ~/of/apps/myApps/oscope-of
|
||||
make OF_ROOT=$HOME/of -j4 Release
|
||||
|
||||
# 5. Run
|
||||
# 5. (optional) data_only_viz Metal pose viz
|
||||
cd ../../../data_only_viz
|
||||
uv sync # base (Metal + OSC + tracker)
|
||||
uv sync --extra pose # + MediaPipe / YOLOv8 / Ultralytics
|
||||
# uv sync --extra nlf # + NLF SMPL body mesh (CUDA only)
|
||||
uv run python -m data_only_viz.main
|
||||
|
||||
# 6. Run the whole stack
|
||||
open AV-Live/launcher/build/AVLiveLauncher.app
|
||||
```
|
||||
|
||||
The launcher autostarts `sclang` + `oscope-of` + `node` (web UI) on launch. Open the menubar icon for status, logs, restart controls.
|
||||
The launcher autostarts `sclang` + the relevant visualizers + the web bridges depending on the **3 launch modes** offered by the picker at startup :
|
||||
|
||||
- **Full AV-Live** — sclang + oscope-of + sound_algo web UI (`:3000`) + data_feeds (full profile)
|
||||
- **Data-only** — sclang + `data_only_viz` Metal viz + data_feeds (data-only profile) + sound_algo web UI + **data-only dashboard** (`:3211` with `/dashboard.html`, `/map.html`, `/control.html`)
|
||||
- **Body Mesh** — sclang + `data_only_viz` (headless, Multi-HMR worker only) + **`AVLiveBody`** Swift app (RealityKit, SMPL-X mesh + webcam overlay + live `RenderSettings` panel `S`) + data_feeds + dashboard data-only
|
||||
|
||||
Open the menubar icon for per-process Start/Stop, logs, and the "AV-Live-Body" launch button available from any mode.
|
||||
|
||||
### Without Hantek hardware
|
||||
|
||||
oscope-of works without the scope plugged in. It falls back to synthesized signals from the OSC `/sync/*` metadata sent by `sound_algo`. You lose the audio-derived FFT but the demoscene visuals + narrative demos all still play.
|
||||
`oscope-of` works without the scope plugged in. It falls back to synthesized signals from the OSC `/sync/*` metadata sent by `sound_algo`. You lose the audio-derived FFT but the demoscene visuals + narrative demos all still play. `data_only_viz` runs identically since it only needs the OSC bus + a webcam.
|
||||
|
||||
## Demoparty mode
|
||||
|
||||
Press a digit (1-9, 0) or F-key (F6-F10) to launch a 1-3 minute scripted demoparty. Each one is a journey through audio + visual transitions, with greetings to the demoscene legends along the way (Fairlight, Razor 1911, ASD, Conspiracy, Farbrausch, TBL, Mercury, TPOLM, Loonies, Hokuto Force, IQ, Knighty, Smash, Gargaj…).
|
||||
Press a digit (1–9, 0) or F-key (F6–F10) to launch a 1–3 minute scripted demoparty. Each one is a journey through audio + visual transitions, with greetings to the demoscene legends along the way (Fairlight, Razor 1911, ASD, Conspiracy, Farbrausch, TBL, Mercury, TPOLM, Loonies, Hokuto Force, IQ, Knighty, Smash, Gargaj…).
|
||||
|
||||
The `0` GREETINGS demo cycles through 18 acts including Boing Ball, plasma C64, dot tunnel, twister bars, copper bars, fractal KIFS, Möbius topology, then a parade of OS logos (Win 1.0, Lotus 1-2-3, Atari, Apple, OS/2 Warp, IRIX, ZX Spectrum), ending on a roll call of demoscene groups.
|
||||
|
||||
@@ -140,14 +351,21 @@ The `0` GREETINGS demo cycles through 18 acts including Boing Ball, plasma C64,
|
||||
|-----------|------|------|
|
||||
| Sound engine | `sound_algo/` | SuperCollider 3.14+, ~12k LOC `.scd` |
|
||||
| Sound web UI | `sound_algo/web/` | Node Express + WebSocket bridge OSC↔HTTP |
|
||||
| Visualizer | `oscope-of/` | openFrameworks 0.12, GLSL 150, ~6k LOC C++ |
|
||||
| Launcher | `launcher/` | SwiftUI universal app, sentinel-based restart |
|
||||
| Visualizer | `oscope-of/` | openFrameworks 0.12, GLSL 150 GL 3.2 core, ~6k LOC C++ |
|
||||
| Pose / body mesh | `data_only_viz/` | Python 3.11+ / `uv`, Metal natif via pyobjc, 7 pose backends |
|
||||
| Launcher | `launcher/` | SwiftUI universal app, sentinel-based restart, ProcessManager `@MainActor` |
|
||||
| Real-world feeds | `data_feeds/` | Python `uv`, 20 async feed modules, `bridge.py` OSC fan-out (3 targets) |
|
||||
| Data-only web | `data_only_viz/web/` | Node Express `:3211` + WS bidir bridge OSC, dashboard / map / control |
|
||||
| Body mesh app | `launcher/AV-Live-Body/` | Swift 6 + RealityKit, SMPL-X 10 475 verts, LowLevelMesh + wireframe |
|
||||
| Multi-HMR worker | `data_only_viz/multi_hmr_worker.py` | PyTorch MPS, AVCaptureSession native, TCP sender :57130 |
|
||||
| Web standalone | `web_realart/` | Node Express + ws, WebGPU three.js r171 + Web Audio + Hydra |
|
||||
| Audio FFT | `oscope-of/src/AudioAnalyzer.{h,cpp}` | downsample + Hann + FFT 2048 |
|
||||
| 3D models | `oscope-of/bin/data/models/` | 35 PLY parametric meshes |
|
||||
| 3D models | `oscope-of/bin/data/models/` | 27 PLY parametric meshes |
|
||||
| Sprites | `oscope-of/bin/data/sprites/` | 14 PNG OS logos + 5 SVG sources |
|
||||
| Shaders | `oscope-of/bin/data/shaders/` | 33 GLSL frag+vert shaders |
|
||||
| Shaders | `oscope-of/bin/data/shaders/` | 33 GLSL pairs (~65 files) frag+vert |
|
||||
| Metal shaders | `data_only_viz/shaders/` | `.metal`, runtime-compiled |
|
||||
|
||||
OSC ports : `scsynth :57110` · `sclang :57121` · `node web :57122` · `oscope-of :57123` · `http :3000`.
|
||||
OSC ports : `scsynth :57110` · `sclang :57121` · `node web :57122` · `oscope-of :57123` (source: `oscope-of/bin/data/settings.json`) · `web_realart :57124` · `http :3000`.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.uv-cache/
|
||||
uv.lock
|
||||
|
||||
# Config local : peut contenir des creds (RTE OAuth, GCN Kafka).
|
||||
# Le fichier de reference est config.toml.example (tracke).
|
||||
# config.toml et config.data-only.toml restent committes tant qu'ils
|
||||
# ne contiennent que des champs vides ; surveiller pre-commit hook.
|
||||
*.local.toml
|
||||
|
||||
# Modeles YOLO telecharges au runtime
|
||||
*.pt
|
||||
@@ -0,0 +1,134 @@
|
||||
# data_feeds — Pont flux temps réel → OSC
|
||||
|
||||
Worker Python asynchrone qui aspire **20 sources publiques** (sismique,
|
||||
geophysique, meteo, qualite de l'air, espace, energie, foudre, aviation,
|
||||
social, blockchain, evenements monde…) et les rebalance en OSC vers
|
||||
SuperCollider (`:57121`), openFrameworks (`:57123`) et le dashboard web
|
||||
data-only (`:57124`). Le but : nourrir l'engine audio et le visualizer
|
||||
avec des **signaux du monde reel**, sans bricoler du networking dans
|
||||
`sclang`.
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
┌────────────────────────────────┐
|
||||
│ data_feeds/bridge.py │
|
||||
│ ├─ usgs (HTTP 60 s) │
|
||||
│ ├─ swpc (HTTP 60 s) │
|
||||
│ ├─ netzfrequenz(WebSocket) │
|
||||
│ ├─ blitzortung (WebSocket) │
|
||||
│ ├─ opensky (HTTP 15 s) │
|
||||
│ ├─ bluesky (WebSocket) │
|
||||
│ ├─ mempool (WebSocket) │
|
||||
│ ├─ rte_eco2mix (OAuth2) │
|
||||
│ ├─ github (HTTP 30 s) │
|
||||
│ └─ gcn (Kafka) │
|
||||
└─────────────┬──────────────────┘
|
||||
│ OSC broadcast
|
||||
┌─────────────┴────────────┐
|
||||
UDP :57121 UDP :57123
|
||||
┌──▼────────────┐ ┌─────▼────────────┐
|
||||
│ SuperCollider │ │ openFrameworks │
|
||||
│ ~feeds dict │ │ OscClient.data()│
|
||||
└───────────────┘ └──────────────────┘
|
||||
```
|
||||
|
||||
## Démarrage
|
||||
|
||||
```bash
|
||||
cd data_feeds
|
||||
uv sync # créé .venv et installe les deps
|
||||
uv run python bridge.py -v # -v = verbose
|
||||
```
|
||||
|
||||
Côté SC :
|
||||
|
||||
```supercollider
|
||||
"sound_algo/control/data_feeds.scd".loadRelative; // installe les OSCdef
|
||||
~feedDump.value; // affiche l'état
|
||||
```
|
||||
|
||||
Côté oF : automatique dès que `OscClient::update()` tourne (déjà appelé
|
||||
chaque frame). Lecture :
|
||||
|
||||
```cpp
|
||||
float kp = osc_.dataf("swpc", "kp", /*fallback*/ 2.0f);
|
||||
std::vector<float> strike;
|
||||
if (osc_.consumeDataPulse("blitzortung", "strike", strike)) {
|
||||
// strike = [lat, lon, age, mult]
|
||||
}
|
||||
```
|
||||
|
||||
## Schéma OSC
|
||||
|
||||
Toutes les routes sont préfixées `/data/<feed>/<sub>`. Voir
|
||||
[`docs/DATA_FEEDS_OSC.md`](../docs/DATA_FEEDS_OSC.md) pour le schéma
|
||||
complet.
|
||||
|
||||
| Feed | Routes | Cadence |
|
||||
|----------------|---------------------------------------------|-------------|
|
||||
| `usgs` | `event`, `rate` | 60 s |
|
||||
| `swpc` | `wind`, `bz`, `kp`, `xray` | 60 s |
|
||||
| `netzfrequenz` | `freq`, `dev`, `time_dev` | ~200 ms |
|
||||
| `blitzortung` | `strike`, `rate` | event-based |
|
||||
| `opensky` | `count`, `plane` | 15 s |
|
||||
| `bluesky` | `post`, `rate` | event-based |
|
||||
| `mempool` | `tx`, `block` | event-based |
|
||||
| `rte_eco2mix` | `mix`, `co2` | 15 min |
|
||||
| `github` | `event`, `rate` | 30 s |
|
||||
| `gcn` | `alert` | rare |
|
||||
| `pose` | `count`, `person`, `skel`, `bone` | ~20 fps |
|
||||
| `openmeteo` | `now` (temp, hum, wind, press, rain) | 10 min |
|
||||
| `openaq` | `now` (PM2.5, PM10, NO2, O3) | 15 min |
|
||||
| `iss` | `pos` (lat, lon, alt, vel), `pass` | 5 s |
|
||||
| `volcano` | `active`, `eruption` | 1 h |
|
||||
| `social_buzz` | `reddit`, `hn`, `pulse` | 1 min |
|
||||
| `gdelt` | `batch`, `event` (lat, lon, tone, country) | 15 min |
|
||||
| `wikimedia` | `edit`, `rate` | streaming |
|
||||
| `tides` | `level` (obs/pred/residual), `moon` | 6 min |
|
||||
| `atc` | `hub` (icao, listeners), `total` | 5 min |
|
||||
|
||||
## Configuration
|
||||
|
||||
Éditer `config.toml` :
|
||||
|
||||
- `osc.targets` : liste `{host, port}` à arroser. Profil data-only
|
||||
par défaut : SC `:57121` + oF `:57123` + web data-only `:57124`.
|
||||
- `feeds.<name>.enabled` : booléen.
|
||||
- `feeds.<name>.poll_seconds` : période pour les feeds HTTP.
|
||||
- `feeds.opensky.bbox` : `[lamin, lomin, lamax, lomax]` (Lyon par défaut).
|
||||
- `feeds.bluesky.sample_rate` : 0..1, fraction des posts conservée.
|
||||
|
||||
Flux nécessitant des identifiants (désactivés par défaut) :
|
||||
|
||||
- `rte_eco2mix` : créer un client sur
|
||||
<https://data.rte-france.com/> puis renseigner `client_id` /
|
||||
`client_secret`.
|
||||
- `gcn` : <https://gcn.nasa.gov/quickstart> + `uv add gcn-kafka`.
|
||||
- `pose` : install les deps optionnelles avec `uv sync --extra pose`
|
||||
(opencv-python + ultralytics). Sur Mac M5 utiliser `device = "mps"`.
|
||||
Une seule app peut grabber la webcam : si oF tourne `WebcamVis` en
|
||||
capture locale, mettre `feeds.pose.enabled = false` (et inversement).
|
||||
|
||||
## Diagnostic
|
||||
|
||||
```bash
|
||||
# Sniffer les paquets recus cote SC
|
||||
uv run python -c "from pythonosc import osc_server, dispatcher; \
|
||||
d=dispatcher.Dispatcher(); d.set_default_handler(lambda a,*x: print(a,x)); \
|
||||
osc_server.BlockingOSCUDPServer(('127.0.0.1',57121),d).serve_forever()"
|
||||
```
|
||||
|
||||
Côté SC, vérifier le heartbeat :
|
||||
|
||||
```supercollider
|
||||
~feedAlive.value // true si le pont émet depuis < 15 s
|
||||
```
|
||||
|
||||
## Ajout d'un flux
|
||||
|
||||
1. Créer `data_feeds/feeds/<name>.py` exposant `async def run(ctx)`.
|
||||
2. L'enregistrer dans `config.toml` avec `enabled = true`.
|
||||
3. Ajouter les OSCdef correspondants dans
|
||||
`sound_algo/control/data_feeds.scd`.
|
||||
4. Documenter le schéma OSC dans `docs/DATA_FEEDS_OSC.md`.
|
||||
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Orchestrateur du pont data_feeds → OSC.
|
||||
|
||||
Charge `config.toml`, lance un worker async par flux activé, et diffuse
|
||||
en broadcast vers tous les `osc.targets`. Chaque worker doit exposer
|
||||
une coroutine `run(ctx)` qui prend un `Context` et émet via `ctx.send(...)`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import importlib
|
||||
import logging
|
||||
import re
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Whitelist des noms de feed : empeche l'injection de modules arbitraires
|
||||
# via un config.toml malveillant (importlib resolve sur du . ou .. ferait
|
||||
# remonter dans l'arborescence).
|
||||
_FEED_NAME_RE = re.compile(r"^[a-z][a-z0-9_]{0,30}$")
|
||||
|
||||
try:
|
||||
import tomllib # py311+
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib # type: ignore
|
||||
|
||||
from pythonosc.udp_client import SimpleUDPClient
|
||||
|
||||
LOG = logging.getLogger("bridge")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
cfg: dict[str, Any]
|
||||
prefix: str
|
||||
clients: list[SimpleUDPClient]
|
||||
feed_name: str
|
||||
|
||||
def send(self, sub: str, *args: Any) -> None:
|
||||
path = f"{self.prefix}/{self.feed_name}/{sub}"
|
||||
for c in self.clients:
|
||||
try:
|
||||
c.send_message(path, list(args))
|
||||
except OSError as e:
|
||||
LOG.warning("OSC send failed %s: %s", path, e)
|
||||
|
||||
|
||||
def load_config(path: Path) -> dict[str, Any]:
|
||||
with path.open("rb") as f:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
async def run_feed(name: str, cfg: dict[str, Any], ctx: Context) -> None:
|
||||
"""Charge `data_feeds.feeds.<name>` et appelle `run(ctx)`."""
|
||||
if not _FEED_NAME_RE.match(name):
|
||||
LOG.error("invalid feed name %r — must match %s",
|
||||
name, _FEED_NAME_RE.pattern)
|
||||
return
|
||||
try:
|
||||
mod = importlib.import_module(f"data_feeds.feeds.{name}")
|
||||
except ModuleNotFoundError:
|
||||
# Fallback : exécution depuis le dossier data_feeds/
|
||||
mod = importlib.import_module(f"feeds.{name}")
|
||||
LOG.info("starting feed: %s", name)
|
||||
backoff = 1.0
|
||||
while True:
|
||||
try:
|
||||
await mod.run(ctx)
|
||||
# Retour propre : on reset le backoff et on rejoue en boucle.
|
||||
backoff = 1.0
|
||||
await asyncio.sleep(2.0)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("feed %s crashed: %s — retry in %.1fs", name, e, backoff)
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 2, 60.0)
|
||||
|
||||
|
||||
async def heartbeat(clients: list[SimpleUDPClient], prefix: str) -> None:
|
||||
t0 = time.monotonic()
|
||||
while True:
|
||||
for c in clients:
|
||||
c.send_message(f"{prefix}/heartbeat", [time.monotonic() - t0])
|
||||
await asyncio.sleep(5.0)
|
||||
|
||||
|
||||
async def main_async(cfg: dict[str, Any]) -> None:
|
||||
osc_cfg = cfg.get("osc", {})
|
||||
prefix = osc_cfg.get("prefix", "/data")
|
||||
clients = [
|
||||
SimpleUDPClient(t["host"], t["port"])
|
||||
for t in osc_cfg.get("targets", [{"host": "127.0.0.1", "port": 57121}])
|
||||
]
|
||||
LOG.info(
|
||||
"OSC targets: %s",
|
||||
", ".join(f"{c._address}:{c._port}" for c in clients), # noqa: SLF001
|
||||
)
|
||||
|
||||
tasks: list[asyncio.Task[None]] = []
|
||||
for name, fcfg in cfg.get("feeds", {}).items():
|
||||
if not fcfg.get("enabled", False):
|
||||
LOG.info("feed disabled: %s", name)
|
||||
continue
|
||||
ctx = Context(cfg=fcfg, prefix=prefix, clients=clients, feed_name=name)
|
||||
tasks.append(asyncio.create_task(run_feed(name, fcfg, ctx), name=name))
|
||||
|
||||
if not tasks:
|
||||
LOG.warning("no feed enabled — exiting")
|
||||
return
|
||||
|
||||
tasks.append(asyncio.create_task(heartbeat(clients, prefix), name="heartbeat"))
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
stop = loop.create_future()
|
||||
sig_count = 0
|
||||
|
||||
def _on_signal() -> None:
|
||||
nonlocal sig_count
|
||||
sig_count += 1
|
||||
if sig_count > 1:
|
||||
LOG.warning("second signal received — forcing exit")
|
||||
sys.exit(1)
|
||||
if not stop.done():
|
||||
stop.cancel()
|
||||
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
loop.add_signal_handler(sig, _on_signal)
|
||||
try:
|
||||
await stop
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
finally:
|
||||
for t in tasks:
|
||||
t.cancel()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
LOG.info("bridge stopped")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("-c", "--config", type=Path, default=Path(__file__).parent / "config.toml")
|
||||
p.add_argument("-v", "--verbose", action="store_true")
|
||||
args = p.parse_args()
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s — %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
cfg = load_config(args.config)
|
||||
try:
|
||||
asyncio.run(main_async(cfg))
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,149 @@
|
||||
# Profil "data-only" : que les flux open-data + pose YOLO, vers SC + oF.
|
||||
#
|
||||
# Pas de sclang/oscope/web → la chaine consomatrice est libre de
|
||||
# brancher ce qu'elle veut sur les ports 57121 / 57123.
|
||||
#
|
||||
# Lancer avec : uv run python bridge.py -c config.data-only.toml -v
|
||||
|
||||
[osc]
|
||||
targets = [
|
||||
{ host = "127.0.0.1", port = 57121 }, # SuperCollider (si lance)
|
||||
{ host = "127.0.0.1", port = 57123 }, # openFrameworks
|
||||
{ host = "127.0.0.1", port = 57124 }, # Web dashboard data-only
|
||||
]
|
||||
prefix = "/data"
|
||||
|
||||
# -- Pose / webcam --------------------------------------------------------
|
||||
[feeds.pose]
|
||||
# Active uniquement quand le pont tourne dans le bundle launcher (TCC OK).
|
||||
# En CLI le prompt webcam ne peut pas apparaitre -> camera silently refused.
|
||||
enabled = true
|
||||
model = "yolov8n-pose.pt"
|
||||
device = "mps"
|
||||
camera = 0
|
||||
width = 640
|
||||
height = 480
|
||||
target_fps = 20
|
||||
conf_thresh = 0.35
|
||||
max_persons = 4
|
||||
emit_keypoints = true
|
||||
|
||||
# -- Sismique / géophysique ------------------------------------------------
|
||||
[feeds.usgs]
|
||||
enabled = true
|
||||
url = "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson"
|
||||
poll_seconds = 60
|
||||
|
||||
[feeds.swpc]
|
||||
enabled = true
|
||||
url_plasma = "https://services.swpc.noaa.gov/products/solar-wind/plasma-1-day.json"
|
||||
url_mag = "https://services.swpc.noaa.gov/products/solar-wind/mag-1-day.json"
|
||||
url_kp = "https://services.swpc.noaa.gov/products/noaa-planetary-k-index.json"
|
||||
url_xray = "https://services.swpc.noaa.gov/json/goes/primary/xrays-1-day.json"
|
||||
poll_seconds = 60
|
||||
|
||||
# -- Réseau électrique -----------------------------------------------------
|
||||
[feeds.netzfrequenz]
|
||||
# WARN : mainsfrequenz.de a ferme son WS public (NXDOMAIN 2026-05).
|
||||
# Garde a false jusqu'a trouver une source alternative (gridradar ?
|
||||
# swissgrid open data ?).
|
||||
enabled = true
|
||||
ws_url = "wss://www.mainsfrequenz.de/frequenz.socket"
|
||||
|
||||
[feeds.rte_eco2mix]
|
||||
enabled = true
|
||||
client_id = ""
|
||||
client_secret = ""
|
||||
poll_seconds = 900
|
||||
|
||||
# -- Foudre / atmosphère ---------------------------------------------------
|
||||
[feeds.blitzortung]
|
||||
enabled = true
|
||||
ws_url = "wss://ws1.blitzortung.org:443/"
|
||||
|
||||
# -- Aviation / mouvement --------------------------------------------------
|
||||
[feeds.opensky]
|
||||
enabled = true
|
||||
url = "https://opensky-network.org/api/states/all"
|
||||
# API anonyme : 10 req / 1 min credit-based. On reste a 60s pour ne
|
||||
# jamais epuiser le credit en cas de fork de process / restart rapide.
|
||||
# Les 429 vus en 2026-05 etaient dus au poll 15s/20s precedent.
|
||||
poll_seconds = 60
|
||||
bbox = [45.5, 4.6, 46.0, 5.2]
|
||||
|
||||
# -- Pouls numérique -------------------------------------------------------
|
||||
[feeds.bluesky]
|
||||
enabled = true
|
||||
ws_url = "wss://jetstream2.us-east.bsky.network/subscribe?wantedCollections=app.bsky.feed.post"
|
||||
sample_rate = 0.02
|
||||
|
||||
[feeds.mempool]
|
||||
enabled = true
|
||||
ws_url = "wss://mempool.space/api/v1/ws"
|
||||
|
||||
[feeds.github]
|
||||
enabled = true
|
||||
url = "https://api.github.com/events"
|
||||
poll_seconds = 30
|
||||
|
||||
# -- Meteo locale (Open-Meteo, sans cle) ----------------------------------
|
||||
[feeds.openmeteo]
|
||||
enabled = true
|
||||
lat = 48.8566 # Paris par defaut
|
||||
lon = 2.3522
|
||||
poll_seconds = 600
|
||||
|
||||
# -- Qualite de l'air (OpenAQ v3, sans cle) -------------------------------
|
||||
[feeds.openaq]
|
||||
enabled = true
|
||||
lat = 48.8566
|
||||
lon = 2.3522
|
||||
radius_m = 25000
|
||||
poll_seconds = 900
|
||||
|
||||
# -- Station spatiale (ISS / wheretheiss.at) ------------------------------
|
||||
[feeds.iss]
|
||||
enabled = true
|
||||
lat = 48.8566
|
||||
lon = 2.3522
|
||||
pass_radius_km = 1500
|
||||
poll_seconds = 5
|
||||
|
||||
# -- Volcans actifs (Smithsonian GVP 7-jours JSON) ------------------------
|
||||
[feeds.volcano]
|
||||
enabled = true
|
||||
url = "https://volcano.si.edu/feeds/eruptions7days.json"
|
||||
poll_seconds = 3600
|
||||
|
||||
# -- Pouls social (Reddit hot + HackerNews top) ---------------------------
|
||||
[feeds.social_buzz]
|
||||
enabled = true
|
||||
poll_seconds = 60
|
||||
|
||||
# -- GDELT (evenements monde 15-min) --------------------------------------
|
||||
[feeds.gdelt]
|
||||
enabled = true
|
||||
poll_seconds = 900
|
||||
|
||||
# -- Wikipedia recent changes ---------------------------------------------
|
||||
[feeds.wikimedia]
|
||||
enabled = true
|
||||
sample_rate = 0.05
|
||||
rate_window_s = 5
|
||||
|
||||
# -- NOAA tides + lune ----------------------------------------------------
|
||||
[feeds.tides]
|
||||
enabled = true
|
||||
station = "8443970"
|
||||
poll_seconds = 360
|
||||
|
||||
# -- LiveATC hubs ---------------------------------------------------------
|
||||
[feeds.atc]
|
||||
enabled = true
|
||||
hubs = ["KJFK", "KLAX", "KSFO", "KORD", "EGLL", "LFPG"]
|
||||
poll_seconds = 300
|
||||
|
||||
[feeds.gcn]
|
||||
enabled = false
|
||||
client_id = ""
|
||||
client_secret = ""
|
||||
@@ -0,0 +1,135 @@
|
||||
# Configuration du pont data_feeds → OSC.
|
||||
#
|
||||
# - SC écoute par défaut sur 57121 (cf. sound_algo/web_bridge.scd).
|
||||
# - oF écoute sur 57123 (cf. ofApp::setup, oscListenPort_).
|
||||
# Le pont diffuse en broadcast vers TOUS les `osc_targets` listés.
|
||||
#
|
||||
# Activer/désactiver un flux : `enabled = true|false`.
|
||||
# Régler le débit avec `poll_seconds` (HTTP) ou laisser les WS gérer.
|
||||
|
||||
[osc]
|
||||
targets = [
|
||||
{ host = "127.0.0.1", port = 57121 }, # SuperCollider
|
||||
{ host = "127.0.0.1", port = 57123 }, # openFrameworks
|
||||
{ host = "127.0.0.1", port = 57124 }, # web bridge (sound_algo/web + web_realart)
|
||||
]
|
||||
# Préfixe commun. Toutes les routes sont /data/<feed>/...
|
||||
prefix = "/data"
|
||||
|
||||
# -- Pose / webcam --------------------------------------------------------
|
||||
|
||||
[feeds.pose]
|
||||
enabled = true
|
||||
# YOLOv8/v11-pose via ultralytics. Auto-download du .pt au premier run.
|
||||
# Modeles : yolov8n-pose (fast), yolov8s-pose, yolov8m-pose, yolov8l-pose.
|
||||
# Sur Mac M5 prefere `n` ou `s`, device="mps" (Metal).
|
||||
model = "yolov8n-pose.pt"
|
||||
device = "mps" # "cpu", "mps" (Apple Silicon), "cuda:0"
|
||||
camera = 0 # index ofVideoGrabber-style (0 = camera par defaut)
|
||||
width = 640
|
||||
height = 480
|
||||
target_fps = 20 # plafond (le serveur peut faire moins)
|
||||
conf_thresh = 0.35
|
||||
max_persons = 4
|
||||
# Si false, n'emit que `/data/pose/count` et les bbox (pas les 17 kp).
|
||||
emit_keypoints = true
|
||||
|
||||
# Routes :
|
||||
# /data/pose/count <n>
|
||||
# /data/pose/person <idx> <cx> <cy> <w> <h> <conf> (normalises 0..1)
|
||||
# /data/pose/skel <idx> <conf_avg> <x0 y0 c0 ... x16 y16 c16> (17 kp COCO)
|
||||
# /data/pose/bone <idx> <kp_a> <kp_b> (segments du skeleton)
|
||||
|
||||
# -- Sismique / géophysique ------------------------------------------------
|
||||
|
||||
[feeds.usgs]
|
||||
enabled = true
|
||||
url = "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson"
|
||||
poll_seconds = 60
|
||||
# Émet /data/usgs/event <mag> <lon> <lat> <depth> <age_sec>
|
||||
# /data/usgs/rate <events_per_hour>
|
||||
|
||||
[feeds.swpc]
|
||||
enabled = true
|
||||
# Vent solaire (DSCOVR plasma)
|
||||
url_plasma = "https://services.swpc.noaa.gov/products/solar-wind/plasma-1-day.json"
|
||||
url_mag = "https://services.swpc.noaa.gov/products/solar-wind/mag-1-day.json"
|
||||
url_kp = "https://services.swpc.noaa.gov/products/noaa-planetary-k-index.json"
|
||||
url_xray = "https://services.swpc.noaa.gov/json/goes/primary/xrays-1-day.json"
|
||||
poll_seconds = 60
|
||||
# /data/swpc/wind <speed_kms> <density_pcm3> <temp_K>
|
||||
# /data/swpc/bz <Bz_nT> <Bt_nT>
|
||||
# /data/swpc/kp <kp> <a_index>
|
||||
# /data/swpc/xray <short_W_m2> <long_W_m2> <flare_class_norm>
|
||||
|
||||
# -- Réseau électrique -----------------------------------------------------
|
||||
|
||||
[feeds.netzfrequenz]
|
||||
# WARN : mainsfrequenz.de a ferme son WS public (NXDOMAIN 2026-05).
|
||||
# Voir alternatives : gridradar.net (auth requise), swissgrid open data.
|
||||
enabled = false
|
||||
# Mainsfrequenz.de WebSocket (résolution ~200 ms, mesure Karlsruhe)
|
||||
ws_url = "wss://www.mainsfrequenz.de/frequenz.socket"
|
||||
# /data/grid/freq <hz> 50.000 ± 0.200
|
||||
# /data/grid/dev <delta_hz> écart vs 50 Hz
|
||||
# /data/grid/time_dev <sec> dérive intégrée
|
||||
|
||||
[feeds.rte_eco2mix]
|
||||
enabled = false # nécessite token OAuth RTE (gratuit, register)
|
||||
client_id = ""
|
||||
client_secret = ""
|
||||
poll_seconds = 900
|
||||
# /data/rte/mix <nuclear> <gas> <coal> <oil> <hydro> <wind> <solar> <bio>
|
||||
# /data/rte/co2 <gCO2_per_kWh>
|
||||
|
||||
# -- Foudre / atmosphère ---------------------------------------------------
|
||||
|
||||
[feeds.blitzortung]
|
||||
enabled = true
|
||||
# LightningMaps relay (Blitzortung dérivé, public)
|
||||
ws_url = "wss://ws1.blitzortung.org:443/"
|
||||
# /data/lightning/strike <lat> <lon> <age_sec> <multiplicity>
|
||||
# /data/lightning/rate <strikes_per_min>
|
||||
|
||||
# -- Aviation / mouvement --------------------------------------------------
|
||||
|
||||
[feeds.opensky]
|
||||
enabled = true
|
||||
url = "https://opensky-network.org/api/states/all"
|
||||
poll_seconds = 15
|
||||
# Bbox optionnelle (Lyon par défaut : lamin,lomin,lamax,lomax)
|
||||
bbox = [45.5, 4.6, 46.0, 5.2]
|
||||
# /data/aviation/count <n>
|
||||
# /data/aviation/plane <icao> <lon> <lat> <alt_m> <vel_ms> <heading_deg>
|
||||
|
||||
# -- Pouls numérique -------------------------------------------------------
|
||||
|
||||
[feeds.bluesky]
|
||||
enabled = true
|
||||
# Jetstream firehose (posts publics WS, JSON décompressé)
|
||||
ws_url = "wss://jetstream2.us-east.bsky.network/subscribe?wantedCollections=app.bsky.feed.post"
|
||||
sample_rate = 0.02 # garde 2 % des événements pour pas saturer
|
||||
# /data/social/post <text_len> <lang_hash>
|
||||
# /data/social/rate <posts_per_sec>
|
||||
|
||||
[feeds.mempool]
|
||||
enabled = false
|
||||
ws_url = "wss://mempool.space/api/v1/ws"
|
||||
# /data/btc/tx <value_btc> <fee_sat_vb>
|
||||
# /data/btc/block <height> <tx_count> <reward_btc>
|
||||
|
||||
[feeds.github]
|
||||
enabled = false
|
||||
url = "https://api.github.com/events"
|
||||
poll_seconds = 30
|
||||
# /data/github/event <type_hash> <repo_hash>
|
||||
|
||||
# -- Espace / GCN ----------------------------------------------------------
|
||||
|
||||
[feeds.gcn]
|
||||
enabled = false
|
||||
# GCN Classic over Kafka : nécessite credentials.
|
||||
# Voir https://gcn.nasa.gov/quickstart pour générer un token.
|
||||
client_id = ""
|
||||
client_secret = ""
|
||||
# /data/gcn/alert <mission_hash> <ra_deg> <dec_deg> <error_arcmin>
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Helpers communs aux feeds."""
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import time
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
class RateMeter:
|
||||
"""Compte les événements sur une fenêtre glissante (en secondes)."""
|
||||
|
||||
def __init__(self, window: float = 60.0) -> None:
|
||||
self.window = window
|
||||
self._events: collections.deque[float] = collections.deque()
|
||||
|
||||
def tick(self) -> int:
|
||||
now = time.monotonic()
|
||||
self._events.append(now)
|
||||
while self._events and now - self._events[0] > self.window:
|
||||
self._events.popleft()
|
||||
return len(self._events)
|
||||
|
||||
@property
|
||||
def rate(self) -> float:
|
||||
return len(self._events) / max(self.window, 1e-6)
|
||||
|
||||
|
||||
def djb2(s: str) -> int:
|
||||
"""Hash stable 0..65535 pour transformer une string en float OSC."""
|
||||
h = 5381
|
||||
for c in s.encode("utf-8", errors="ignore"):
|
||||
h = ((h << 5) + h + c) & 0xFFFF
|
||||
return h
|
||||
|
||||
|
||||
def fnorm(x: float, lo: float, hi: float) -> float:
|
||||
if hi <= lo:
|
||||
return 0.0
|
||||
return max(0.0, min(1.0, (x - lo) / (hi - lo)))
|
||||
|
||||
|
||||
def safe_get(d: dict, path: Iterable[str], default=None):
|
||||
cur = d
|
||||
for k in path:
|
||||
if isinstance(cur, dict) and k in cur:
|
||||
cur = cur[k]
|
||||
else:
|
||||
return default
|
||||
return cur
|
||||
@@ -0,0 +1,60 @@
|
||||
"""LiveATC feeds metadata — pas de pull audio, juste compteur de
|
||||
streams actifs et indicateur d'activite par hub aeroport.
|
||||
|
||||
LiveATC.net expose un endpoint stats JSON simplifie pour quelques
|
||||
aeroports majeurs (KJFK, KLAX, KSFO, EGLL, LFPG). On polle pour
|
||||
chacun le nombre d'auditeurs courant (proxy pour 'activite ATC').
|
||||
|
||||
OSC out :
|
||||
/data/atc/hub icao listeners
|
||||
/data/atc/total total_listeners n_hubs
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.atc")
|
||||
|
||||
# LiveATC publie un page HTML par feed avec "Listeners: N" — on parse
|
||||
# ca via regex au lieu d'un API officielle (pas disponible).
|
||||
HUB_URL = "https://www.liveatc.net/search/?icao={icao}"
|
||||
LISTENERS_RE = re.compile(r"Listeners:\s*<b>(\d+)</b>", re.I)
|
||||
|
||||
|
||||
async def _fetch_hub(cli: httpx.AsyncClient, icao: str
|
||||
) -> int:
|
||||
r = await cli.get(HUB_URL.format(icao=icao),
|
||||
headers={"User-Agent": "av-live-data-feeds/1.0"})
|
||||
r.raise_for_status()
|
||||
matches = LISTENERS_RE.findall(r.text)
|
||||
if not matches:
|
||||
return 0
|
||||
return sum(int(m) for m in matches)
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
hubs = cfg.get("hubs",
|
||||
["KJFK", "KLAX", "KSFO", "KORD", "EGLL", "LFPG"])
|
||||
period = float(cfg.get("poll_seconds", 300.0)) # 5 min
|
||||
async with httpx.AsyncClient(timeout=20.0) as cli:
|
||||
while True:
|
||||
total = 0
|
||||
active = 0
|
||||
for icao in hubs:
|
||||
try:
|
||||
listeners = await _fetch_hub(cli, icao)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("atc %s failed: %s", icao, e)
|
||||
continue
|
||||
ctx.send("hub", icao, float(listeners))
|
||||
total += listeners
|
||||
if listeners > 0:
|
||||
active += 1
|
||||
await asyncio.sleep(0.5) # polite scrape
|
||||
ctx.send("total", float(total), float(active))
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Blitzortung / LightningMaps — impacts de foudre temps réel.
|
||||
|
||||
Protocole : à la connexion, envoyer `{"a":111}` (handshake LightningMaps).
|
||||
Chaque message JSON contient { time, lat, lon, mds (multiplicity)... }.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
import websockets
|
||||
|
||||
from ._util import RateMeter
|
||||
|
||||
LOG = logging.getLogger("feed.blitzortung")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
url = cfg["ws_url"]
|
||||
rate = RateMeter(window=60.0)
|
||||
while True:
|
||||
try:
|
||||
async with websockets.connect(url, ping_interval=20, max_size=2**20) as ws:
|
||||
await ws.send(json.dumps({"a": 111}))
|
||||
LOG.info("connected %s", url)
|
||||
async for raw in ws:
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
lat = float(d.get("lat", 0.0))
|
||||
lon = float(d.get("lon", 0.0))
|
||||
# time est en ns Unix ; on calcule un age en secondes
|
||||
t_ns = d.get("time", 0)
|
||||
age = 0.0
|
||||
if isinstance(t_ns, (int, float)) and t_ns > 0:
|
||||
age = max(0.0, time.time() - t_ns / 1e9)
|
||||
mult = int(d.get("mds") or 1)
|
||||
ctx.send("strike", lat, lon, age, mult)
|
||||
rate.tick()
|
||||
if rate._events: # noqa: SLF001
|
||||
ctx.send("rate", rate.rate * 60.0)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("ws disconnected: %s — reconnecting", e)
|
||||
await asyncio.sleep(5.0)
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Bluesky Jetstream — firehose des posts publics (WebSocket JSON)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
|
||||
import websockets
|
||||
|
||||
from ._util import RateMeter, djb2
|
||||
|
||||
LOG = logging.getLogger("feed.bluesky")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
url = cfg["ws_url"]
|
||||
sample = float(cfg.get("sample_rate", 0.02))
|
||||
rate = RateMeter(window=10.0)
|
||||
last_rate_emit = 0.0
|
||||
while True:
|
||||
try:
|
||||
async with websockets.connect(url, ping_interval=20, max_size=2**20) as ws:
|
||||
LOG.info("connected jetstream (sample %.0f%%)", sample * 100)
|
||||
async for raw in ws:
|
||||
rate.tick()
|
||||
now = time.monotonic()
|
||||
if now - last_rate_emit > 1.0:
|
||||
ctx.send("rate", rate.rate)
|
||||
last_rate_emit = now
|
||||
if random.random() > sample:
|
||||
continue
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
commit = d.get("commit") or {}
|
||||
rec = commit.get("record") or {}
|
||||
text = rec.get("text") or ""
|
||||
lang = (rec.get("langs") or ["?"])[0]
|
||||
ctx.send("post", float(len(text)), float(djb2(lang)))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("ws disconnected: %s — reconnecting", e)
|
||||
await asyncio.sleep(3.0)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""GCN Classic over Kafka — alertes astrophysiques (GRB, GW, neutrinos).
|
||||
|
||||
Nécessite des credentials Kafka. Voir https://gcn.nasa.gov/quickstart.
|
||||
Cette implémentation est volontairement minimale : on extrait ra/dec/error.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
LOG = logging.getLogger("feed.gcn")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
cid, csec = cfg.get("client_id"), cfg.get("client_secret")
|
||||
if not (cid and csec):
|
||||
LOG.warning("GCN credentials manquants — feed inactif (voir gcn.nasa.gov/quickstart)")
|
||||
await asyncio.Event().wait()
|
||||
return
|
||||
try:
|
||||
from gcn_kafka import Consumer # type: ignore
|
||||
except ModuleNotFoundError:
|
||||
LOG.error("`gcn-kafka` non installé. uv add gcn-kafka")
|
||||
await asyncio.Event().wait()
|
||||
return
|
||||
|
||||
from ._util import djb2
|
||||
|
||||
cons = Consumer(client_id=cid, client_secret=csec)
|
||||
cons.subscribe([
|
||||
"gcn.classic.text.SWIFT_BAT_GRB_POS_ACK",
|
||||
"gcn.classic.text.FERMI_GBM_FLT_POS",
|
||||
"gcn.classic.text.LVC_INITIAL",
|
||||
"gcn.classic.text.ICECUBE_ASTROTRACK_GOLD",
|
||||
])
|
||||
LOG.info("subscribed GCN classic streams")
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _poll():
|
||||
return cons.consume(num_messages=10, timeout=1.0)
|
||||
|
||||
while True:
|
||||
msgs = await loop.run_in_executor(None, _poll)
|
||||
for m in msgs or []:
|
||||
if m.error():
|
||||
continue
|
||||
txt = m.value().decode("utf-8", "ignore")
|
||||
ra, dec, err = _parse(txt)
|
||||
ctx.send("alert", float(djb2(m.topic())), ra, dec, err)
|
||||
|
||||
|
||||
def _parse(txt: str) -> tuple[float, float, float]:
|
||||
ra = dec = err = 0.0
|
||||
for line in txt.splitlines():
|
||||
l = line.lower()
|
||||
try:
|
||||
if "ra:" in l and ra == 0.0:
|
||||
ra = float(line.split(":", 1)[1].split()[0])
|
||||
elif "dec:" in l and dec == 0.0:
|
||||
dec = float(line.split(":", 1)[1].split()[0])
|
||||
elif "error" in l and "arcmin" in l and err == 0.0:
|
||||
err = float(line.split(":", 1)[1].split()[0])
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
return ra, dec, err
|
||||
@@ -0,0 +1,93 @@
|
||||
"""GDELT Project — feed des evenements 'world' 15-min.
|
||||
|
||||
Source : GDELT 2.0 Events CSV ; updates toutes les 15 min.
|
||||
On compte les evenements + extrait les top countries / themes du
|
||||
dernier batch.
|
||||
|
||||
OSC out :
|
||||
/data/gdelt/batch n_events n_countries avg_tone
|
||||
/data/gdelt/event lat lon tone country_code root_event_id
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
import time
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.gdelt")
|
||||
|
||||
# GDELT v2 master file list ; on prend juste le dernier .export.CSV.zip
|
||||
MASTER = "http://data.gdeltproject.org/gdeltv2/lastupdate.txt"
|
||||
|
||||
|
||||
async def _fetch_latest_csv(cli: httpx.AsyncClient) -> list[list[str]]:
|
||||
r = await cli.get(MASTER)
|
||||
r.raise_for_status()
|
||||
# 3 lignes : events, mentions, gkg ; on prend events (premiere ligne)
|
||||
first = (r.text.strip().splitlines() or [""])[0].split(" ")
|
||||
if len(first) < 3:
|
||||
return []
|
||||
url = first[2]
|
||||
rz = await cli.get(url, follow_redirects=True)
|
||||
rz.raise_for_status()
|
||||
with zipfile.ZipFile(BytesIO(rz.content)) as zf:
|
||||
name = zf.namelist()[0]
|
||||
text = zf.read(name).decode("utf-8", errors="ignore")
|
||||
return [line.split("\t") for line in text.splitlines() if line]
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
period = float(cfg.get("poll_seconds", 900.0)) # 15 min
|
||||
seen: collections.OrderedDict[str, None] = collections.OrderedDict()
|
||||
SEEN_MAX = 8192
|
||||
async with httpx.AsyncClient(timeout=60.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
rows = await _fetch_latest_csv(cli)
|
||||
if not rows:
|
||||
LOG.warning("gdelt: empty batch")
|
||||
await asyncio.sleep(period)
|
||||
continue
|
||||
count = 0
|
||||
tones = []
|
||||
countries: collections.Counter[str] = collections.Counter()
|
||||
# GDELT 2.0 events CSV : 61 colonnes.
|
||||
# idx 0 = GLOBALEVENTID, 7 = Actor1CountryCode,
|
||||
# 34 = AvgTone, 39 = ActionGeo_Lat, 40 = ActionGeo_Long
|
||||
for row in rows:
|
||||
if len(row) < 41:
|
||||
continue
|
||||
eid = row[0]
|
||||
if not eid or eid in seen:
|
||||
continue
|
||||
seen[eid] = None
|
||||
if len(seen) > SEEN_MAX:
|
||||
seen.popitem(last=False)
|
||||
count += 1
|
||||
try:
|
||||
tone = float(row[34] or "0")
|
||||
except ValueError:
|
||||
tone = 0.0
|
||||
tones.append(tone)
|
||||
cc = row[7].strip()[:3]
|
||||
if cc:
|
||||
countries[cc] += 1
|
||||
try:
|
||||
lat = float(row[39] or "0")
|
||||
lon = float(row[40] or "0")
|
||||
except ValueError:
|
||||
lat = lon = 0.0
|
||||
if lat or lon:
|
||||
ctx.send("event", lat, lon, tone, cc, eid)
|
||||
avg_tone = (sum(tones) / len(tones)) if tones else 0.0
|
||||
ctx.send("batch", float(count),
|
||||
float(len(countries)), float(avg_tone))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("gdelt fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""GitHub public events — firehose dev mondial (polling REST anonyme)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from ._util import djb2
|
||||
|
||||
LOG = logging.getLogger("feed.github")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
url = cfg["url"]
|
||||
period = float(cfg.get("poll_seconds", 30))
|
||||
# IDs GitHub sont des strings numeriques croissants : on compare en int
|
||||
# (la compare string casse des que le nombre de digits change).
|
||||
last_id: int = 0
|
||||
async with httpx.AsyncClient(timeout=20.0,
|
||||
headers={"Accept": "application/vnd.github+json"}) as cli:
|
||||
while True:
|
||||
try:
|
||||
r = await cli.get(url)
|
||||
r.raise_for_status()
|
||||
for ev in reversed(r.json()):
|
||||
raw = ev.get("id", "")
|
||||
try:
|
||||
eid = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if eid <= last_id:
|
||||
continue
|
||||
ctx.send(
|
||||
"event",
|
||||
float(djb2(ev.get("type", "?"))),
|
||||
float(djb2(((ev.get("repo") or {}).get("name") or "?"))),
|
||||
)
|
||||
last_id = eid
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""ISS position via wheretheiss.at (json).
|
||||
|
||||
Renvoie position lat/lon + altitude + velocity. Polling 5s par defaut.
|
||||
Egalement emet l'event 'pass' (1.0) lorsque la station franchit une
|
||||
zone d'observation autour de l'observateur (configurable lat/lon/radius).
|
||||
|
||||
OSC out :
|
||||
/data/iss/pos lat lon alt_km vel_kmh
|
||||
/data/iss/pass 1 (transient, quand iss enter dans le radius)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.iss")
|
||||
|
||||
URL = "https://api.wheretheiss.at/v1/satellites/25544"
|
||||
|
||||
|
||||
def _great_circle_km(lat1: float, lon1: float,
|
||||
lat2: float, lon2: float) -> float:
|
||||
r = 6371.0
|
||||
p1 = math.radians(lat1)
|
||||
p2 = math.radians(lat2)
|
||||
dp = math.radians(lat2 - lat1)
|
||||
dl = math.radians(lon2 - lon1)
|
||||
a = (math.sin(dp / 2) ** 2
|
||||
+ math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2)
|
||||
return 2 * r * math.asin(math.sqrt(a))
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
period = float(cfg.get("poll_seconds", 5.0))
|
||||
obs_lat = float(cfg.get("lat", 48.8566))
|
||||
obs_lon = float(cfg.get("lon", 2.3522))
|
||||
pass_radius = float(cfg.get("pass_radius_km", 1500.0))
|
||||
inside_prev = False
|
||||
async with httpx.AsyncClient(timeout=10.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
r = await cli.get(URL)
|
||||
r.raise_for_status()
|
||||
j = r.json()
|
||||
lat = float(j.get("latitude", 0.0))
|
||||
lon = float(j.get("longitude", 0.0))
|
||||
alt = float(j.get("altitude", 0.0))
|
||||
vel = float(j.get("velocity", 0.0))
|
||||
ctx.send("pos", lat, lon, alt, vel)
|
||||
dist = _great_circle_km(obs_lat, obs_lon, lat, lon)
|
||||
inside_now = dist < pass_radius
|
||||
if inside_now and not inside_prev:
|
||||
ctx.send("pass", 1.0, dist)
|
||||
inside_prev = inside_now
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("iss fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""mempool.space — Bitcoin txs and blocks (WebSocket)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
import websockets
|
||||
|
||||
LOG = logging.getLogger("feed.mempool")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
url = cfg["ws_url"]
|
||||
while True:
|
||||
try:
|
||||
async with websockets.connect(url, ping_interval=20, max_size=2**21) as ws:
|
||||
await ws.send(json.dumps({"action": "want", "data": ["mempool-blocks", "blocks", "live-2h-chart"]}))
|
||||
LOG.info("connected mempool.space")
|
||||
async for raw in ws:
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
continue
|
||||
if "block" in d:
|
||||
b = d["block"] or {}
|
||||
ctx.send(
|
||||
"block",
|
||||
float(b.get("height", 0)),
|
||||
float(b.get("tx_count", 0)),
|
||||
float(b.get("extras", {}).get("reward", 0) or 0) / 1e8,
|
||||
)
|
||||
if "transactions" in d:
|
||||
for tx in (d["transactions"] or [])[:5]:
|
||||
ctx.send(
|
||||
"tx",
|
||||
float(tx.get("value", 0)) / 1e8,
|
||||
float(tx.get("fee", 0)) / max(1.0, float(tx.get("vsize", 1))),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("ws disconnected: %s — reconnecting", e)
|
||||
await asyncio.sleep(5.0)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Fréquence du réseau électrique européen — WebSocket Mainsfrequenz.de.
|
||||
|
||||
Format payload (texte) : "f=49.987 t=2026-05-11T06:42:00Z" environ.
|
||||
Le serveur peut changer ; on parse defensively et on extrait `f`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
|
||||
import websockets
|
||||
|
||||
LOG = logging.getLogger("feed.netzfrequenz")
|
||||
|
||||
_RE_F = re.compile(r"f\s*=\s*([0-9]+\.[0-9]+)")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
url = cfg["ws_url"]
|
||||
time_dev = 0.0 # dérive intégrée (secondes)
|
||||
last_t = time.monotonic()
|
||||
# Backoff exponentiel cap a 5 minutes pour ne pas spammer un host
|
||||
# mort (mainsfrequenz.de NXDOMAIN depuis 2026-05). On log la
|
||||
# premiere et chaque dixieme reconnect uniquement.
|
||||
backoff = 3.0
|
||||
attempt = 0
|
||||
while True:
|
||||
try:
|
||||
async with websockets.connect(url, ping_interval=20) as ws:
|
||||
LOG.info("connected %s", url)
|
||||
backoff = 3.0 # reset on success
|
||||
attempt = 0
|
||||
async for msg in ws:
|
||||
text = msg if isinstance(msg, str) else msg.decode("utf-8", "ignore")
|
||||
m = _RE_F.search(text)
|
||||
if not m:
|
||||
continue
|
||||
try:
|
||||
f = float(m.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
now = time.monotonic()
|
||||
dt = now - last_t
|
||||
last_t = now
|
||||
delta = f - 50.0
|
||||
# Intégration : 1 s réelle à 49.5 Hz → -0.01 s d'horloge
|
||||
time_dev += (delta / 50.0) * dt
|
||||
ctx.send("freq", f)
|
||||
ctx.send("dev", delta)
|
||||
ctx.send("time_dev", time_dev)
|
||||
except Exception as e: # noqa: BLE001
|
||||
attempt += 1
|
||||
if attempt == 1 or attempt % 10 == 0:
|
||||
LOG.warning("ws disconnected (attempt %d, backoff %ds): %s",
|
||||
attempt, int(backoff), e)
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 1.6, 300.0)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""OpenAQ — qualite de l'air locale.
|
||||
|
||||
Mesures temps reel PM2.5 / PM10 / NO2 / O3 autour d'un point geo.
|
||||
API v3 publique sans cle (rate-limited mais souple).
|
||||
|
||||
OSC out :
|
||||
/data/openaq/now pm25 pm10 no2 o3
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.openaq")
|
||||
|
||||
URL = ("https://api.openaq.org/v3/locations"
|
||||
"?coordinates={lat},{lon}&radius={radius}&limit=20")
|
||||
|
||||
|
||||
def _latest(values: list, param: str) -> float:
|
||||
"""Cherche la mesure la plus recente pour `param` dans la liste
|
||||
de locations OpenAQ v3."""
|
||||
best = 0.0
|
||||
for loc in values:
|
||||
for sensor in loc.get("sensors", []):
|
||||
p = sensor.get("parameter", {})
|
||||
if p.get("name") == param:
|
||||
last = sensor.get("latest", {})
|
||||
v = last.get("value")
|
||||
if v is not None and v > best:
|
||||
best = float(v)
|
||||
return best
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
lat = float(cfg.get("lat", 48.8566))
|
||||
lon = float(cfg.get("lon", 2.3522))
|
||||
radius = int(cfg.get("radius_m", 25000)) # 25 km autour
|
||||
period = float(cfg.get("poll_seconds", 900.0)) # 15 min
|
||||
url = URL.format(lat=lat, lon=lon, radius=radius)
|
||||
async with httpx.AsyncClient(timeout=20.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
r = await cli.get(url)
|
||||
r.raise_for_status()
|
||||
locs = r.json().get("results", [])
|
||||
pm25 = _latest(locs, "pm25")
|
||||
pm10 = _latest(locs, "pm10")
|
||||
no2 = _latest(locs, "no2")
|
||||
o3 = _latest(locs, "o3")
|
||||
ctx.send("now", pm25, pm10, no2, o3)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("openaq fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Open-Meteo — meteo locale (temp / vent / humidite / pression / pluie).
|
||||
|
||||
Pas de cle API. Geolocalisation via lat/lon en config.toml.
|
||||
Update toutes les `poll_seconds` (defaut 600s = 10 min).
|
||||
|
||||
OSC out :
|
||||
/data/openmeteo/now temp_c humidity wind_mps wind_deg pressure_hpa rain_mmh
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.openmeteo")
|
||||
|
||||
URL = ("https://api.open-meteo.com/v1/forecast"
|
||||
"?latitude={lat}&longitude={lon}"
|
||||
"¤t=temperature_2m,relative_humidity_2m,wind_speed_10m,"
|
||||
"wind_direction_10m,pressure_msl,rain"
|
||||
"&wind_speed_unit=ms&timezone=UTC")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
lat = float(cfg.get("lat", 48.8566)) # Paris by default
|
||||
lon = float(cfg.get("lon", 2.3522))
|
||||
period = float(cfg.get("poll_seconds", 600.0))
|
||||
url = URL.format(lat=lat, lon=lon)
|
||||
async with httpx.AsyncClient(timeout=15.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
r = await cli.get(url)
|
||||
r.raise_for_status()
|
||||
cur = r.json().get("current", {})
|
||||
ctx.send("now",
|
||||
float(cur.get("temperature_2m", 0.0)),
|
||||
float(cur.get("relative_humidity_2m", 0.0)),
|
||||
float(cur.get("wind_speed_10m", 0.0)),
|
||||
float(cur.get("wind_direction_10m", 0.0)),
|
||||
float(cur.get("pressure_msl", 1013.0)),
|
||||
float(cur.get("rain", 0.0)))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("openmeteo fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""OpenSky Network — ADS-B aircraft states (REST polling, anon ≤ 15 s)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.opensky")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
base = cfg["url"]
|
||||
period = float(cfg.get("poll_seconds", 15))
|
||||
bbox = cfg.get("bbox") # [lamin, lomin, lamax, lomax]
|
||||
params = None
|
||||
if bbox and len(bbox) == 4:
|
||||
params = {
|
||||
"lamin": bbox[0], "lomin": bbox[1],
|
||||
"lamax": bbox[2], "lomax": bbox[3],
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=20.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
r = await cli.get(base, params=params)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
continue
|
||||
states = data.get("states") or []
|
||||
ctx.send("count", float(len(states)))
|
||||
for s in states:
|
||||
# index: 0=icao24, 5=lon, 6=lat, 7=baro_alt, 9=velocity, 10=heading
|
||||
try:
|
||||
icao = (s[0] or "?")[:8]
|
||||
lon = float(s[5]) if s[5] is not None else 0.0
|
||||
lat = float(s[6]) if s[6] is not None else 0.0
|
||||
alt = float(s[7]) if s[7] is not None else 0.0
|
||||
vel = float(s[9]) if s[9] is not None else 0.0
|
||||
head = float(s[10]) if s[10] is not None else 0.0
|
||||
except (IndexError, TypeError, ValueError):
|
||||
continue
|
||||
ctx.send("plane", icao, lon, lat, alt, vel, head)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Webcam → OpenCV → pose detection (YOLOv8-pose) → OSC.
|
||||
|
||||
Pourquoi YOLOv8-pose plutot qu'OpenPose proper ?
|
||||
- OpenPose officiel = build CUDA, douloureux sur Mac ARM.
|
||||
- YOLOv8-pose : pip install, MPS/Metal accelere, 17 keypoints COCO
|
||||
(proche d'OpenPose BODY_25, suffisant pour de l'AV-live).
|
||||
- Pour un vrai OpenPose, swap simple : remplacer `Detector` par un
|
||||
wrapper autour de pyopenpose ou cmu-openpose et conserver le format
|
||||
keypoints (x_norm, y_norm, conf) emis sur OSC.
|
||||
|
||||
Sortie OSC :
|
||||
/data/pose/count <n>
|
||||
/data/pose/person <idx> <cx> <cy> <w> <h> <conf>
|
||||
/data/pose/skel <idx> <conf_avg> <x0 y0 c0 ... x16 y16 c16>
|
||||
/data/pose/bone <idx> <kp_a> <kp_b> (a la connexion, statique)
|
||||
/data/pose/stats <avg_conf> <avg_size> <cx_bar> <cy_bar> (par batch)
|
||||
|
||||
Toutes les coordonnees sont normalisees 0..1 (origine top-left).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
|
||||
LOG = logging.getLogger("feed.pose")
|
||||
|
||||
# Squelette COCO 17 keypoints (paires d'os).
|
||||
COCO_BONES: list[tuple[int, int]] = [
|
||||
(0, 1), (0, 2), (1, 3), (2, 4), # tete
|
||||
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10),# bras
|
||||
(5, 11), (6, 12), (11, 12), # torse
|
||||
(11, 13), (13, 15), (12, 14), (14, 16), # jambes
|
||||
]
|
||||
|
||||
|
||||
class _Lazy:
|
||||
"""Imports lourds differes pour ne pas casser le pont entier si pose
|
||||
n'est pas demande."""
|
||||
def __init__(self) -> None:
|
||||
self.cv2 = None
|
||||
self.YOLO = None
|
||||
self.np = None
|
||||
|
||||
def load(self) -> None:
|
||||
if self.cv2 is not None:
|
||||
return
|
||||
import cv2 # type: ignore
|
||||
import numpy as np # type: ignore
|
||||
from ultralytics import YOLO # type: ignore
|
||||
self.cv2 = cv2
|
||||
self.np = np
|
||||
self.YOLO = YOLO
|
||||
|
||||
|
||||
_LAZY = _Lazy()
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
try:
|
||||
_LAZY.load()
|
||||
except ModuleNotFoundError as e:
|
||||
LOG.error("dependances manquantes : %s — uv sync --extra pose", e)
|
||||
await asyncio.Event().wait()
|
||||
return
|
||||
|
||||
cv2, np, YOLO = _LAZY.cv2, _LAZY.np, _LAZY.YOLO
|
||||
|
||||
cam_idx = int(cfg.get("camera", 0))
|
||||
width = int(cfg.get("width", 640))
|
||||
height = int(cfg.get("height", 480))
|
||||
target_fps = float(cfg.get("target_fps", 20))
|
||||
conf_thresh = float(cfg.get("conf_thresh", 0.35))
|
||||
max_persons = int(cfg.get("max_persons", 4))
|
||||
emit_kp = bool(cfg.get("emit_keypoints", True))
|
||||
model_name = cfg.get("model", "yolov8n-pose.pt")
|
||||
device = cfg.get("device", "mps")
|
||||
|
||||
LOG.info("loading %s on %s", model_name, device)
|
||||
model = YOLO(model_name)
|
||||
|
||||
cap = cv2.VideoCapture(cam_idx)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index %d indisponible", cam_idx)
|
||||
await asyncio.Event().wait()
|
||||
return
|
||||
|
||||
# Annonce du squelette (statique, une fois)
|
||||
for a, b in COCO_BONES:
|
||||
ctx.send("bone", float(a), float(b))
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
period = 1.0 / max(1.0, target_fps)
|
||||
LOG.info("pose stream up: %dx%d @ %.1f fps target", width, height, target_fps)
|
||||
|
||||
def _grab():
|
||||
ok, frame = cap.read()
|
||||
return frame if ok else None
|
||||
|
||||
# ThreadPoolExecutor dedie : empeche l'inference longue (>20ms sur MPS)
|
||||
# de monopoliser le pool partage et de bloquer les autres feeds.
|
||||
import concurrent.futures
|
||||
pool = concurrent.futures.ThreadPoolExecutor(max_workers=1,
|
||||
thread_name_prefix="pose")
|
||||
|
||||
def _infer(fr):
|
||||
return model.predict(fr, device=device, conf=conf_thresh,
|
||||
verbose=False, max_det=max_persons)
|
||||
|
||||
try:
|
||||
while True:
|
||||
t0 = time.monotonic()
|
||||
frame = await loop.run_in_executor(pool, _grab)
|
||||
if frame is None:
|
||||
await asyncio.sleep(period)
|
||||
continue
|
||||
h, w = frame.shape[:2]
|
||||
try:
|
||||
# Non-bloquant : l'event loop continue de servir les autres
|
||||
# feeds pendant les ~20-80 ms d'inference.
|
||||
results = await loop.run_in_executor(pool, _infer, frame)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("inference failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
continue
|
||||
if not results:
|
||||
ctx.send("count", 0.0)
|
||||
await asyncio.sleep(period)
|
||||
continue
|
||||
|
||||
res = results[0]
|
||||
kp_xy = getattr(res.keypoints, "xy", None)
|
||||
kp_conf = getattr(res.keypoints, "conf", None)
|
||||
boxes = getattr(res, "boxes", None)
|
||||
n = 0 if kp_xy is None else int(len(kp_xy))
|
||||
ctx.send("count", float(n))
|
||||
|
||||
# Agregats pour le dashboard : confiance moyenne globale,
|
||||
# taille moyenne (proxy distance cam), centre du barycentre
|
||||
# des personnes, fraction de l'image occupee.
|
||||
global_conf_sum = 0.0
|
||||
global_conf_n = 0
|
||||
size_sum = 0.0
|
||||
cx_sum = 0.0
|
||||
cy_sum = 0.0
|
||||
|
||||
for i in range(n):
|
||||
# bbox normalisee
|
||||
if boxes is not None and i < len(boxes):
|
||||
b = boxes.xywhn[i].cpu().numpy().tolist() # cx, cy, w, h
|
||||
conf_b = float(boxes.conf[i].item())
|
||||
ctx.send("person", float(i), *b, conf_b)
|
||||
cx_sum += float(b[0])
|
||||
cy_sum += float(b[1])
|
||||
size_sum += float(b[2]) * float(b[3])
|
||||
if not emit_kp or kp_xy is None:
|
||||
continue
|
||||
pts = kp_xy[i].cpu().numpy() # (17, 2) px
|
||||
cfs = kp_conf[i].cpu().numpy() if kp_conf is not None \
|
||||
else np.ones(len(pts), dtype=float)
|
||||
flat: list[float] = []
|
||||
conf_sum = 0.0
|
||||
for (x, y), c in zip(pts, cfs):
|
||||
xn = float(x) / max(1.0, w)
|
||||
yn = float(y) / max(1.0, h)
|
||||
cc = float(c)
|
||||
flat.extend([xn, yn, cc])
|
||||
conf_sum += cc
|
||||
avg = conf_sum / max(1, len(pts))
|
||||
ctx.send("skel", float(i), avg, *flat)
|
||||
global_conf_sum += avg
|
||||
global_conf_n += 1
|
||||
|
||||
# /data/pose/stats avg_conf avg_size cx cy
|
||||
if n > 0:
|
||||
avg_conf = global_conf_sum / max(1, global_conf_n)
|
||||
avg_size = size_sum / n
|
||||
cx_bar = cx_sum / n
|
||||
cy_bar = cy_sum / n
|
||||
ctx.send("stats", float(avg_conf), float(avg_size),
|
||||
float(cx_bar), float(cy_bar))
|
||||
else:
|
||||
ctx.send("stats", 0.0, 0.0, 0.5, 0.5)
|
||||
|
||||
# cadence
|
||||
dt = time.monotonic() - t0
|
||||
if dt < period:
|
||||
await asyncio.sleep(period - dt)
|
||||
finally:
|
||||
cap.release()
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
@@ -0,0 +1,84 @@
|
||||
"""RTE éCO2mix — mix électrique France (OAuth2 client_credentials)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.rte_eco2mix")
|
||||
|
||||
TOKEN_URL = "https://digital.iservices.rte-france.com/token/oauth/"
|
||||
API_URL = "https://digital.iservices.rte-france.com/open_api/actual_generation/v1/actual_generations_per_production_type"
|
||||
|
||||
# Mapping des `production_type` RTE (verbose) vers nos categories courtes.
|
||||
# L'API agrege HYDRO_* et WIND_* pour le contexte musical : on additionne.
|
||||
_RTE_MAP = {
|
||||
"NUCLEAR": "NUCLEAR",
|
||||
"FOSSIL_GAS": "GAS",
|
||||
"FOSSIL_HARD_COAL": "COAL",
|
||||
"FOSSIL_OIL": "OIL",
|
||||
"HYDRO_WATER_RESERVOIR": "HYDRO",
|
||||
"HYDRO_RUN_OF_RIVER_AND_POUNDAGE":"HYDRO",
|
||||
"HYDRO_PUMPED_STORAGE": "HYDRO",
|
||||
"WIND_ONSHORE": "WIND",
|
||||
"WIND_OFFSHORE": "WIND",
|
||||
"SOLAR": "SOLAR",
|
||||
"BIOMASS": "BIOENERGY",
|
||||
"WASTE": "BIOENERGY",
|
||||
}
|
||||
|
||||
|
||||
async def _get_token(cli: httpx.AsyncClient, cid: str, csec: str) -> tuple[str, float]:
|
||||
r = await cli.post(TOKEN_URL, auth=(cid, csec),
|
||||
data={"grant_type": "client_credentials"})
|
||||
r.raise_for_status()
|
||||
j = r.json()
|
||||
return j["access_token"], time.monotonic() + float(j.get("expires_in", 7200)) - 60
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
cid, csec = cfg.get("client_id"), cfg.get("client_secret")
|
||||
period = float(cfg.get("poll_seconds", 900))
|
||||
if not (cid and csec):
|
||||
LOG.warning("client_id/client_secret manquants — feed inactif")
|
||||
await asyncio.Event().wait()
|
||||
return
|
||||
token, exp = "", 0.0
|
||||
async with httpx.AsyncClient(timeout=30.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
if time.monotonic() > exp:
|
||||
token, exp = await _get_token(cli, cid, csec)
|
||||
r = await cli.get(API_URL, headers={"Authorization": f"Bearer {token}"})
|
||||
r.raise_for_status()
|
||||
j = r.json()
|
||||
latest: dict[str, float] = {}
|
||||
for series in (j.get("actual_generations_per_production_type") or []):
|
||||
raw = series.get("production_type", "?")
|
||||
typ = _RTE_MAP.get(raw)
|
||||
if typ is None:
|
||||
continue # type non-mappe, ignore
|
||||
vals = series.get("values") or []
|
||||
if vals:
|
||||
# Aggregation : on additionne les sous-categories
|
||||
# (ex: WIND_ONSHORE + WIND_OFFSHORE -> WIND).
|
||||
latest[typ] = latest.get(typ, 0.0) \
|
||||
+ float(vals[-1].get("value", 0.0))
|
||||
# mapping standard RTE → ordre des args
|
||||
ctx.send(
|
||||
"mix",
|
||||
latest.get("NUCLEAR", 0.0),
|
||||
latest.get("GAS", 0.0),
|
||||
latest.get("COAL", 0.0),
|
||||
latest.get("OIL", 0.0),
|
||||
latest.get("HYDRO", 0.0),
|
||||
latest.get("WIND", 0.0),
|
||||
latest.get("SOLAR", 0.0),
|
||||
latest.get("BIOENERGY", 0.0),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Reddit /r/all + HackerNews top — pulse social pour viz 'social storm'.
|
||||
|
||||
Reddit : /r/all/hot.json — score, num_comments des top posts
|
||||
HN : algolia API search_by_date front_page — points, comments
|
||||
|
||||
OSC out :
|
||||
/data/social_buzz/reddit score_avg comments_avg n
|
||||
/data/social_buzz/hn score_avg comments_avg n
|
||||
/data/social_buzz/pulse combined_score (event tick toutes ~30s)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.social_buzz")
|
||||
|
||||
REDDIT_URL = "https://www.reddit.com/r/all/hot.json?limit=25"
|
||||
HN_URL = "https://hacker-news.firebaseio.com/v0/topstories.json"
|
||||
HN_ITEM = "https://hacker-news.firebaseio.com/v0/item/{}.json"
|
||||
|
||||
|
||||
async def _fetch_reddit(cli: httpx.AsyncClient) -> tuple[float, float, int]:
|
||||
r = await cli.get(REDDIT_URL,
|
||||
headers={"User-Agent": "av-live-data-feeds/1.0"})
|
||||
r.raise_for_status()
|
||||
posts = r.json().get("data", {}).get("children", [])
|
||||
if not posts:
|
||||
return 0.0, 0.0, 0
|
||||
scores = [int(p["data"].get("score", 0)) for p in posts]
|
||||
comments = [int(p["data"].get("num_comments", 0)) for p in posts]
|
||||
n = len(scores)
|
||||
return sum(scores) / n, sum(comments) / n, n
|
||||
|
||||
|
||||
async def _fetch_hn(cli: httpx.AsyncClient, top_n: int = 15
|
||||
) -> tuple[float, float, int]:
|
||||
r = await cli.get(HN_URL)
|
||||
r.raise_for_status()
|
||||
ids = r.json()[:top_n]
|
||||
coros = [cli.get(HN_ITEM.format(i)) for i in ids]
|
||||
resps = await asyncio.gather(*coros, return_exceptions=True)
|
||||
scores, comments = [], []
|
||||
for resp in resps:
|
||||
if isinstance(resp, Exception):
|
||||
continue
|
||||
try:
|
||||
it = resp.json()
|
||||
except Exception:
|
||||
continue
|
||||
scores.append(int(it.get("score", 0)))
|
||||
comments.append(int(it.get("descendants", 0)))
|
||||
n = len(scores) or 1
|
||||
return sum(scores) / n, sum(comments) / n, len(scores)
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
period = float(cfg.get("poll_seconds", 60.0))
|
||||
async with httpx.AsyncClient(timeout=20.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
r_score, r_com, r_n = await _fetch_reddit(cli)
|
||||
ctx.send("reddit", r_score, r_com, float(r_n))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("reddit fetch failed: %s", e)
|
||||
r_score = 0.0
|
||||
try:
|
||||
h_score, h_com, h_n = await _fetch_hn(cli)
|
||||
ctx.send("hn", h_score, h_com, float(h_n))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("hn fetch failed: %s", e)
|
||||
h_score = 0.0
|
||||
# Combined score normalize en [0..1] ; reddit hot ~10k+ posts,
|
||||
# HN front ~300 points. On scale chaque source puis on max.
|
||||
combined = max(min(r_score / 10000.0, 1.0),
|
||||
min(h_score / 500.0, 1.0))
|
||||
ctx.send("pulse", combined)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""NOAA SWPC — vent solaire, IMF Bz, indice Kp, X-ray flux GOES."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.swpc")
|
||||
|
||||
|
||||
def _last_row(data) -> list | None:
|
||||
if not data or len(data) < 2:
|
||||
return None
|
||||
return data[-1]
|
||||
|
||||
|
||||
def _flare_class_norm(long_wm2: float) -> float:
|
||||
"""Mappe le X-ray long band en classe normalisée 0..1.
|
||||
A=1e-8, B=1e-7, C=1e-6, M=1e-5, X=1e-4. log10 → [0..1] sur A→X.
|
||||
"""
|
||||
if long_wm2 <= 0:
|
||||
return 0.0
|
||||
return max(0.0, min(1.0, (math.log10(long_wm2) + 8.0) / 4.0))
|
||||
|
||||
|
||||
async def _fetch_json(cli: httpx.AsyncClient, url: str):
|
||||
r = await cli.get(url)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
period = float(cfg.get("poll_seconds", 60))
|
||||
urls = {
|
||||
"plasma": cfg.get("url_plasma"),
|
||||
"mag": cfg.get("url_mag"),
|
||||
"kp": cfg.get("url_kp"),
|
||||
"xray": cfg.get("url_xray"),
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=20.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
if urls["plasma"]:
|
||||
j = await _fetch_json(cli, urls["plasma"])
|
||||
row = _last_row(j)
|
||||
if row:
|
||||
# ["time_tag","density","speed","temperature"]
|
||||
try:
|
||||
density = float(row[1])
|
||||
speed = float(row[2])
|
||||
temp = float(row[3])
|
||||
ctx.send("wind", speed, density, temp)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if urls["mag"]:
|
||||
j = await _fetch_json(cli, urls["mag"])
|
||||
row = _last_row(j)
|
||||
if row:
|
||||
# ["time_tag","bx_gsm","by_gsm","bz_gsm","lon_gsm","lat_gsm","bt"]
|
||||
try:
|
||||
bz = float(row[3])
|
||||
bt = float(row[6])
|
||||
ctx.send("bz", bz, bt)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if urls["kp"]:
|
||||
j = await _fetch_json(cli, urls["kp"])
|
||||
# NOAA renvoie maintenant une liste de dicts pour Kp
|
||||
# ({"time_tag":..., "Kp":..., "a_running":...}) au lieu
|
||||
# de la liste-de-listes historique. On supporte les deux.
|
||||
if j:
|
||||
last = j[-1]
|
||||
try:
|
||||
if isinstance(last, dict):
|
||||
kp = float(last.get("Kp", 0.0))
|
||||
a = float(last.get("a_running", 0.0))
|
||||
else:
|
||||
kp = float(last[1])
|
||||
a = float(last[2])
|
||||
ctx.send("kp", kp, a)
|
||||
except (TypeError, ValueError, KeyError, IndexError):
|
||||
pass
|
||||
if urls["xray"]:
|
||||
j = await _fetch_json(cli, urls["xray"])
|
||||
# split short/long bands
|
||||
short = next((d for d in reversed(j) if d.get("energy") == "0.05-0.4nm"), None)
|
||||
long_ = next((d for d in reversed(j) if d.get("energy") == "0.1-0.8nm"), None)
|
||||
s = float(short.get("flux", 0.0)) if short else 0.0
|
||||
l = float(long_.get("flux", 0.0)) if long_ else 0.0
|
||||
ctx.send("xray", s, l, _flare_class_norm(l))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("fetch failed: %s: %s", type(e).__name__, e)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,64 @@
|
||||
"""NOAA tides (station configurable) + phase lunaire calculee.
|
||||
|
||||
NOAA CO-OPS API : observed water level + predicted, station codee
|
||||
(defaut Boston). Phase lunaire algorithme Conway approx (sans dep).
|
||||
|
||||
OSC out :
|
||||
/data/tides/level water_level_m predicted_m residual_m
|
||||
/data/tides/moon phase_0_1 illum_0_1
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.tides")
|
||||
|
||||
NOAA_URL = ("https://api.tidesandcurrents.noaa.gov/api/prod/datagetter"
|
||||
"?product={product}&application=AV-Live&format=json"
|
||||
"&time_zone=gmt&datum=MLLW&units=metric"
|
||||
"&date=latest&station={station}")
|
||||
|
||||
|
||||
def _moon_phase(t: float) -> tuple[float, float]:
|
||||
"""t epoch -> (phase 0..1 ou 0 = new, 0.5 = full ; illum 0..1).
|
||||
Approx Conway, +-1 jour de precision suffisant."""
|
||||
# Reference : 2000-01-06 18:14 UTC ~ new moon
|
||||
new = 946755300.0
|
||||
cycle = 29.530588853 * 86400.0
|
||||
phase = ((t - new) % cycle) / cycle
|
||||
illum = (1.0 - math.cos(2 * math.pi * phase)) / 2.0
|
||||
return phase, illum
|
||||
|
||||
|
||||
async def _fetch_level(cli: httpx.AsyncClient, station: str
|
||||
) -> tuple[float, float]:
|
||||
obs_url = NOAA_URL.format(product="water_level", station=station)
|
||||
pred_url = NOAA_URL.format(product="predictions", station=station)
|
||||
obs = await cli.get(obs_url)
|
||||
pred = await cli.get(pred_url)
|
||||
obs.raise_for_status()
|
||||
pred.raise_for_status()
|
||||
o = obs.json().get("data", [{}])[0]
|
||||
p = pred.json().get("predictions", [{}])[0]
|
||||
return float(o.get("v") or 0.0), float(p.get("v") or 0.0)
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
station = str(cfg.get("station", "8443970")) # Boston by default
|
||||
period = float(cfg.get("poll_seconds", 360.0)) # 6 min
|
||||
async with httpx.AsyncClient(timeout=20.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
obs, pred = await _fetch_level(cli, station)
|
||||
ctx.send("level", obs, pred, obs - pred)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("tides fetch failed: %s", e)
|
||||
phase, illum = _moon_phase(time.time())
|
||||
ctx.send("moon", float(phase), float(illum))
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,57 @@
|
||||
"""USGS earthquakes — GeoJSON polling (1 min)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
from ._util import RateMeter
|
||||
|
||||
LOG = logging.getLogger("feed.usgs")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
url = cfg["url"]
|
||||
period = float(cfg.get("poll_seconds", 60))
|
||||
# OrderedDict avec eviction LRU : conserve les 4096 derniers IDs vus
|
||||
# dans l'ORDRE d'arrivee. set() perdait l'ordre au pruning, ce qui
|
||||
# pouvait re-emettre un evenement deja vu.
|
||||
seen: "collections.OrderedDict[str, None]" = collections.OrderedDict()
|
||||
SEEN_MAX = 4096
|
||||
rate = RateMeter(window=3600.0)
|
||||
async with httpx.AsyncClient(timeout=20.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
r = await cli.get(url)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
continue
|
||||
|
||||
now_ms = time.time() * 1000.0
|
||||
for feat in data.get("features", []):
|
||||
fid = feat.get("id")
|
||||
if not fid or fid in seen:
|
||||
continue
|
||||
seen[fid] = None
|
||||
props = feat.get("properties") or {}
|
||||
coords = (feat.get("geometry") or {}).get("coordinates") or [0, 0, 0]
|
||||
mag = float(props.get("mag") or 0.0)
|
||||
t_ms = float(props.get("time") or now_ms)
|
||||
age = max(0.0, (now_ms - t_ms) / 1000.0)
|
||||
ctx.send("event", mag, float(coords[0]), float(coords[1]),
|
||||
float(coords[2]), age)
|
||||
rate.tick()
|
||||
ctx.send("rate", rate.rate * 3600.0)
|
||||
|
||||
# garde la mémoire bornée — evict les plus anciens en preservant
|
||||
# l'ordre d'insertion (LRU front, head most-recent).
|
||||
while len(seen) > SEEN_MAX:
|
||||
seen.popitem(last=False)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Volcans actifs — Smithsonian GVP weekly reports + USGS volcano hazards.
|
||||
|
||||
Source primaire : USGS volcano feed (RSS / GeoJSON), couvre les volcans
|
||||
US actifs. Pour les volcans monde, on parse les CSV publics Smithsonian
|
||||
si configures. Polling 1h.
|
||||
|
||||
OSC out :
|
||||
/data/volcano/active count
|
||||
/data/volcano/eruption lat lon vei region (nouvelle eruption depuis last poll)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import collections
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.volcano")
|
||||
|
||||
USGS_URL = ("https://volcanoes.usgs.gov/hans2/api/volcano/getEvents"
|
||||
"?starttime={start}&endtime={end}")
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
period = float(cfg.get("poll_seconds", 3600.0))
|
||||
url = cfg.get("url",
|
||||
"https://volcano.si.edu/feeds/eruptions7days.json")
|
||||
seen: collections.OrderedDict[str, None] = collections.OrderedDict()
|
||||
SEEN_MAX = 512
|
||||
async with httpx.AsyncClient(timeout=30.0) as cli:
|
||||
while True:
|
||||
try:
|
||||
r = await cli.get(url)
|
||||
r.raise_for_status()
|
||||
ct = r.headers.get("content-type", "")
|
||||
items = []
|
||||
if "json" in ct:
|
||||
data = r.json()
|
||||
items = data.get("features", data.get("items", []))
|
||||
count = 0
|
||||
for it in items:
|
||||
props = it.get("properties", it)
|
||||
eid = str(props.get("id") or props.get("eventid")
|
||||
or props.get("volcanoNumber") or "")
|
||||
if not eid:
|
||||
continue
|
||||
count += 1
|
||||
if eid in seen:
|
||||
continue
|
||||
seen[eid] = None
|
||||
if len(seen) > SEEN_MAX:
|
||||
seen.popitem(last=False)
|
||||
geom = it.get("geometry") or {}
|
||||
coords = geom.get("coordinates") or [0, 0]
|
||||
lon, lat = float(coords[0]), float(coords[1])
|
||||
vei = float(props.get("vei", 0) or 0)
|
||||
region = str(props.get("country")
|
||||
or props.get("region", ""))[:32]
|
||||
ctx.send("eruption", lat, lon, vei, region)
|
||||
ctx.send("active", float(count))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("volcano fetch failed: %s", e)
|
||||
await asyncio.sleep(period)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Wikipedia recent changes — Wikimedia EventStreams SSE.
|
||||
|
||||
Firehose des modifications Wikipedia toutes langues confondues.
|
||||
Tres bavard (~10-30 events/s) — on echantillonne et on emet une
|
||||
fraction + des compteurs.
|
||||
|
||||
OSC out :
|
||||
/data/wikimedia/edit lang title bot (sample)
|
||||
/data/wikimedia/rate per_second
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
LOG = logging.getLogger("feed.wikimedia")
|
||||
|
||||
URL = "https://stream.wikimedia.org/v2/stream/recentchange"
|
||||
|
||||
|
||||
async def run(ctx) -> None:
|
||||
cfg = ctx.cfg
|
||||
sample = float(cfg.get("sample_rate", 0.05)) # emit 5% des edits
|
||||
window = float(cfg.get("rate_window_s", 5.0))
|
||||
# WMF EventStreams refuse l'User-Agent par defaut httpx (403). Il
|
||||
# faut une string descriptive + URL/email pour les abuse reports.
|
||||
headers = {
|
||||
"Accept": "text/event-stream",
|
||||
"User-Agent": "av-live-data-feeds/1.0 (https://github.com/electron-rare/AV-Live)",
|
||||
}
|
||||
while True:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=None) as cli:
|
||||
async with cli.stream("GET", URL, headers=headers) as r:
|
||||
r.raise_for_status()
|
||||
bucket = 0
|
||||
bucket_start = time.monotonic()
|
||||
async for line in r.aiter_lines():
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line[5:].strip())
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
bucket += 1
|
||||
# Sample emit
|
||||
if random.random() < sample:
|
||||
lang = (ev.get("wiki") or "").replace("wiki", "")
|
||||
title = str(ev.get("title", ""))[:64]
|
||||
bot = 1.0 if ev.get("bot") else 0.0
|
||||
ctx.send("edit", lang, title, bot)
|
||||
# Periodic rate
|
||||
now = time.monotonic()
|
||||
if now - bucket_start >= window:
|
||||
per_s = bucket / (now - bucket_start)
|
||||
ctx.send("rate", float(per_s))
|
||||
bucket = 0
|
||||
bucket_start = now
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("wikimedia stream error: %s — reconnect 5s", e)
|
||||
await asyncio.sleep(5.0)
|
||||
@@ -0,0 +1,23 @@
|
||||
[project]
|
||||
name = "av-live-data-feeds"
|
||||
version = "0.1.0"
|
||||
description = "Real-world data → OSC bridge for AV-Live (SuperCollider + openFrameworks)"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"python-osc>=1.8.3",
|
||||
"httpx>=0.27",
|
||||
"websockets>=12.0",
|
||||
"aiomqtt>=2.3",
|
||||
"tomli>=2.0;python_version<'3.11'",
|
||||
"skyfield>=1.49",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
pose = [
|
||||
"opencv-python>=4.10",
|
||||
"ultralytics>=8.3", # YOLOv8/v11-pose (CoreML/MPS sur Apple Silicon)
|
||||
"numpy>=1.26",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
package = false
|
||||
@@ -0,0 +1,4 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.venv/
|
||||
.uv-cache/
|
||||
@@ -0,0 +1,81 @@
|
||||
# data_only_viz
|
||||
|
||||
Visualiseur natif Metal (pyobjc) pour le mode data-only d'AV-Live : capture caméra → détection pose multi-personne → tracker → rendu Metal → OSC out vers `oscope-of`.
|
||||
|
||||
## Environnement
|
||||
|
||||
```bash
|
||||
cd data_only_viz
|
||||
uv sync # base
|
||||
uv sync --extra pose # MediaPipe + YOLO + Ultralytics
|
||||
uv sync --extra nlf # Neural Localizer Fields (SMPL body mesh)
|
||||
uv sync --extra detrpose # DETRPose transformer (clone manuel — voir detrpose.py)
|
||||
uv run python -m data_only_viz.main # lancement standard
|
||||
```
|
||||
|
||||
Python **3.11+** requis. `pyproject.toml` est la source de vérité — ne jamais éditer `uv.lock` à la main.
|
||||
|
||||
## Backends pose disponibles
|
||||
|
||||
| Backend | Fichier | Statut |
|
||||
|---------|---------|--------|
|
||||
| MediaPipe Holistic | `holistic.py` | stable |
|
||||
| MediaPipe multi (Pose+Face+Hand) | `multi.py` | stable ; `MEDIAPIPE_DELEGATE=gpu` (défaut) ou `cpu`. **GPU Metal exige SRGBA 4-ch** (3-ch SRGB crashe `gpu_buffer_storage_cv_pixel_buffer.cc`) — multi.py route auto vers `cv2.COLOR_BGR2RGBA` + `mp.ImageFormat.SRGBA` quand delegate=GPU. Bench M5 image-mode SRGBA : pose 2.9 vs 6.7 ms (GPU/CPU), face 1.0 vs 4.1, hand 3.2 vs 6.1 |
|
||||
| Ultralytics YOLOv8-pose | `pose.py` | stable, modèle `yolov8n-pose.pt` à la racine repo |
|
||||
| Apple Vision (Core ML) | `apple_vision_pose.py`, `coreml_pose.py` | macOS uniquement |
|
||||
| DETRPose | `detrpose.py` | clone manuel + checkpoint, voir docstring |
|
||||
| NLF (SMPL body mesh) | `nlf_worker.py` | TorchScript, **bloqué CPU/MPS** (NotImplementedError 2026-05-13), CUDA-only ; checkpoints via `scripts/setup_nlf.sh` |
|
||||
| Multi-HMR | `multi_hmr_scaffold.md` | scaffold seulement |
|
||||
| SMPLER-X / WHAM-TRAM | `*_scaffold.md` | scaffold seulement |
|
||||
|
||||
## Conventions
|
||||
|
||||
- État partagé multi-thread : `state.py` expose `State.lock()` — toujours mutationner sous lock.
|
||||
- Filtrage temporel : `euro_filter.py` (One Euro Filter) sur les keypoints avant tracker.
|
||||
- Association multi-personne : `tracker.py` IoU-based, `scipy.optimize.linear_sum_assignment`.
|
||||
- Shaders Metal dans `shaders/` (`.metal`), recompilés au runtime ; topologie mesh (SMPL faces) en binaire dans `mesh_topology.py`.
|
||||
- OSC out : `osc_listener.py` / `pose_bridge.py` — destination `oscope-of` sur `:57123`.
|
||||
|
||||
## action-head (classifier action debout/assise/danse)
|
||||
|
||||
Tête de classification d'action streaming au-dessus des j3d SMPL-X (ou body3d MediaPipe en fallback). Implémentée 2026-05-13.
|
||||
|
||||
| Fichier | Rôle |
|
||||
|---|---|
|
||||
| `action_head.py` | `ActionHeadModel` (GRU 1L + MLP, 37 811 params, <2 ms/step M5), `ActionHead.step(pid, j3d) → (label, probs, kin)`, `PerPersonBuffer`, `FeatureExtractor` (201-D : j3d + vel + accel + scalaires) |
|
||||
| `action_head_pub.py` | Publisher thread démarré dans `multi.py` `__init__`. Polle `state.persons_smplx` (préféré) ou `state.persons_body3d` (fallback) à 30 Hz, dédup par timestamp, extrait j3d22 via `SMPLX_JOINT_ANCHOR_VERTS` ou `MEDIAPIPE_TO_22`, émet OSC `/pose/action` + `/pose/kin` + `/pose/enter/leave` |
|
||||
| `training/{dataset,autolabel,augment,train_action_head,eval,review}.py` | Pipeline complet : jsonl IO + sliding windows + by-session split / règles auto-label + glue CLI / 4 augmentations / training MPS AdamW CE-weighted / confusion matrix + latence micro-bench / TUI textuel pour review manuel |
|
||||
| `scripts/capture_actions.py` | Webcam → MP4 + timestamps |
|
||||
| `scripts/extract_j3d_offline.py` | MP4 → jsonl j3d22 via `MultiHMRCoreMLBackend.infer()` directement (pas de refactor worker) |
|
||||
| `scripts/train_on_studio.sh` | rsync grosmac → bastion electron-server → studio M3 Ultra + uv sync `--extra multihmr` + train MPS + ckpt back |
|
||||
|
||||
Pipeline complet de capture à live :
|
||||
```bash
|
||||
uv run python -m data_only_viz.scripts.capture_actions --session sess01 --duration 600
|
||||
uv run python -m data_only_viz.scripts.extract_j3d_offline --session sess01 --video ~/.cache/av-live-action/raw/sess01.mp4
|
||||
uv run python -m data_only_viz.training.autolabel --frames ~/.cache/av-live-action/raw/sess01.jsonl --out ~/.cache/av-live-action/dataset/auto.jsonl
|
||||
uv run python -m data_only_viz.training.review --in ~/.cache/av-live-action/dataset/auto.jsonl --out ~/.cache/av-live-action/dataset/dataset.jsonl
|
||||
./data_only_viz/scripts/train_on_studio.sh --epochs 50
|
||||
uv run python -m data_only_viz.training.eval --ckpt ~/.cache/av-live-action/checkpoints/action_head.pt --dataset ~/.cache/av-live-action/dataset/dataset.jsonl
|
||||
# Live : publisher déjà câblé dans multi.py, aucune action requise
|
||||
```
|
||||
|
||||
Checkpoint par défaut : `~/.cache/av-live-action/checkpoints/action_head.pt`. Absent → random init (warmup retourne `debout`).
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
uv run pytest tests/ -v
|
||||
```
|
||||
|
||||
Tests TDD-first pour `nlf_worker.py` ; valider avant chaque commit qui touche un worker.
|
||||
|
||||
Suite action-head (8 fichiers, 39 tests) : `tests/test_action_head_*.py`, `tests/test_{dataset,autolabel,augment,training_smoke,pose_bridge_action}.py`. Tous doivent rester verts avant chaque commit qui touche `action_head*.py` ou `training/*.py`.
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
- Ne pas charger un modèle ML sans guard `try/except ImportError` — les optional-extras peuvent manquer.
|
||||
- Ne pas committer `*.pt`, `*.ckpt`, `*.safetensors`, `*.mlpackage` (gitignore racine).
|
||||
- Ne pas appeler `state.persons_nlf = ...` hors `with state.lock():`.
|
||||
- Ne pas hardcoder le device (`cuda`/`mps`/`cpu`) : détecter via `torch.backends.mps.is_available()` puis fallback.
|
||||
- Pas de `print` dans la boucle de rendu — utiliser un logger conditionnel.
|
||||
@@ -0,0 +1,131 @@
|
||||
# Multi-HMR + RealityKit — utilisation
|
||||
|
||||
## Setup une fois
|
||||
|
||||
```bash
|
||||
# 1. Clone + checkpoint Multi-HMR (1.28 GB)
|
||||
./data_only_viz/scripts/setup_multihmr.sh
|
||||
|
||||
# 2. SMPL-X NEUTRAL.npz — inscription manuelle MPII
|
||||
# https://smpl-x.is.tue.mpg.de/ → SMPL-X v1.1 (NPZ+PKL)
|
||||
# Extraire SMPLX_NEUTRAL.npz vers
|
||||
# ~/.cache/av-live-multihmr/models/smplx/SMPLX_NEUTRAL.npz
|
||||
|
||||
# 3. Python deps
|
||||
cd data_only_viz && uv sync --extra multihmr
|
||||
|
||||
# 4. Extraire les faces SMPL-X pour Swift (250 896 octets)
|
||||
.venv/bin/python scripts/dump_smplx_faces.py
|
||||
```
|
||||
|
||||
## Lancement
|
||||
|
||||
### Via le launcher (auto)
|
||||
1. Ouvrir AVLiveLauncher.app
|
||||
2. Mode **data-only**, activer le switch *Multi-HMR (mesh SMPL-X dense)*
|
||||
3. Demarrer : le launcher spawn sclang + viz Python + AV-Live-Body Swift
|
||||
|
||||
### Manuel (debug)
|
||||
```bash
|
||||
# Terminal 1 — RealityKit listener
|
||||
cd launcher/AV-Live-Body
|
||||
swift run -c release AVLiveBody
|
||||
|
||||
# Terminal 2 — worker Python
|
||||
cd /Users/electron/Documents/Projets/AV-Live
|
||||
data_only_viz/.venv/bin/python -m data_only_viz.main \
|
||||
-v --pose --multi-hmr --fullscreen
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
webcam Mac (cv2 idx 0, 672x672)
|
||||
|
|
||||
v
|
||||
Multi-HMR ViT-S (PyTorch MPS, ~31.5M params)
|
||||
|
|
||||
v
|
||||
humans = [{v3d (10475,3), j3d, transl, shape, expression, ...}, ...]
|
||||
|
|
||||
v
|
||||
OneEuroFilter (shape, expression) + IoU tracker
|
||||
|
|
||||
v
|
||||
state.persons_smplx : list[SMPLXPerson]
|
||||
|
|
||||
v
|
||||
SMPLXTCPSender : binaire ~126 KB / frame / personne sur :57130
|
||||
|
|
||||
v (TCP)
|
||||
AVLiveBody Swift
|
||||
- OSCServer (Network.framework NWListener)
|
||||
- MeshRenderer (LowLevelMesh, vertex buffer in-place)
|
||||
- BodyView (ARView, perspective cam 2m back)
|
||||
```
|
||||
|
||||
## Decisions cles vs. plan initial
|
||||
|
||||
- **NLF abandonne** : son detecteur YOLO TorchScript a CUDA hardcode
|
||||
(`aten::empty_strided` echoue sur CPU/MPS).
|
||||
- **Multi-HMR `demo.py` bypasse** : depend de `multi_hmr_anny` (package
|
||||
prive `anny`) et `utils.render` (pyrender + OpenGL offscreen lourd).
|
||||
On stubbe ces modules dans `sys.modules` et on construit le `Model`
|
||||
directement depuis `model.py`.
|
||||
- **`v3d` direct** : Multi-HMR renvoie deja les vertices SMPL-X
|
||||
decodes ; on n'utilise pas `SMPLXDecoder` dans le hot path (il
|
||||
reste utile pour les tests neutres).
|
||||
- **macOS 15 requis** : `LowLevelMesh.parts.replaceAll` (API RealityKit
|
||||
release 2024-09) permet le vertex update in-place ; sur macOS 14 il
|
||||
faudrait rebuild la MeshResource a chaque frame (3-4x plus lent).
|
||||
|
||||
## FPS attendu
|
||||
|
||||
- Multi-HMR ViT-S MPS mesure 2026-05-13 :
|
||||
- bench headless (dummy 672x672, 20 iter) : 199 ms median = 5.0 fps
|
||||
- bench camera live (30 s real capture) : 228 ms median = 3.8 fps
|
||||
(overhead capture/pre/tensor ~30 ms)
|
||||
- Pas de speedup significatif vs ViT-L : le bottleneck n'est PAS le
|
||||
backbone DINOv2 sur MPS — probablement la tete SMPL-X (identique
|
||||
entre variantes S/B/L) et l'absence de SDPA fused / xFormers sur MPS.
|
||||
La piste "ViT-S pour gagner du FPS" est invalidee par mesure.
|
||||
- Fallback ViT-L (`multiHMR_896_L.pt`) : ~150-200 ms = 5-7 fps si
|
||||
precision insuffisante avec ViT-S
|
||||
- TCP sender throttle a 12 fps cible
|
||||
- RealityKit render : 60 fps Cocoa, interpole entre frames Multi-HMR
|
||||
|
||||
## Assets caches
|
||||
|
||||
```
|
||||
~/.cache/av-live-multihmr/
|
||||
├── checkpoints/
|
||||
│ └── multiHMR_672_S.pt (124 MB)
|
||||
├── models/
|
||||
│ ├── smplx/
|
||||
│ │ ├── SMPLX_NEUTRAL.npz (108 MB)
|
||||
│ │ ├── SMPLX_MALE.npz (109 MB)
|
||||
│ │ ├── SMPLX_FEMALE.npz (109 MB)
|
||||
│ │ ├── SMPLX_NEUTRAL.pkl (520 MB)
|
||||
│ │ └── smplx_uv_2023.npz (UV mapping, 1 MB)
|
||||
│ └── smpl_mean_params.npz (1.3 KB)
|
||||
└── multi-hmr/ (repo clone)
|
||||
└── models -> ../models (symlink relatif)
|
||||
```
|
||||
|
||||
## Debug
|
||||
|
||||
| Symptome | Verification |
|
||||
|----------|--------------|
|
||||
| Pas de mesh visible | `ls ~/.cache/av-live-multihmr/checkpoints/multiHMR_672_S.pt` |
|
||||
| `Multi-HMR load failed` | `~/.cache/av-live-multihmr/models/smplx/SMPLX_NEUTRAL.npz` present ? |
|
||||
| `TCP refused on :57130` | Lancer l'app Swift AVANT le worker Python |
|
||||
| FPS trop bas | Switcher vers ViT-B en editant `CKPT` dans `multi_hmr_worker.py` |
|
||||
| MPS NotImplementedError | Variable env `PYTORCH_ENABLE_MPS_FALLBACK=1` |
|
||||
|
||||
## Pistes futures
|
||||
|
||||
- Texture webcam projetee sur le mesh via `smplx_uv_2023.npz` (62 724
|
||||
UV coords deja en cache).
|
||||
- Switch dynamique L/B/S selon load CPU (autotuning).
|
||||
- Compression vertices (delta-encoded float16) si la bande passante TCP
|
||||
devient un goulot.
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Capture webcam via AVFoundation natif (pyobjc), sans cv2.
|
||||
|
||||
Resout le mismatch d'indices entre `cv2.VideoCapture(N)` et l'ordre
|
||||
des devices retournes par AVCaptureDeviceDiscoverySession : on
|
||||
selectionne le device par `uniqueID` ou par type
|
||||
(BuiltInWideAngleCamera) au lieu d'un index opaque.
|
||||
|
||||
Pattern :
|
||||
- AVCaptureSession + AVCaptureVideoDataOutput
|
||||
- Delegate avec @objc.python_method pour la copie numpy
|
||||
- Global dispatch queue (libdispatch via ctypes) — pas de main queue
|
||||
pour ne pas bloquer NSApp
|
||||
- Frame BGRA -> BGR HxWx3 uint8 partagee sous lock
|
||||
- API .read() compatible cv2 : (ok, frame_bgr)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import objc
|
||||
|
||||
import AVFoundation as AVF
|
||||
import CoreMedia as CM
|
||||
import Quartz
|
||||
from Foundation import NSObject
|
||||
|
||||
LOG = logging.getLogger("av_capture")
|
||||
|
||||
_DEVICE_TYPES = [
|
||||
"AVCaptureDeviceTypeBuiltInWideAngleCamera",
|
||||
"AVCaptureDeviceTypeContinuityCamera",
|
||||
"AVCaptureDeviceTypeExternal",
|
||||
"AVCaptureDeviceTypeDeskViewCamera",
|
||||
]
|
||||
|
||||
|
||||
def _get_global_queue() -> object:
|
||||
"""Retourne une dispatch_queue_t globale wrappee en pyobjc object."""
|
||||
libdispatch = ctypes.CDLL("/usr/lib/system/libdispatch.dylib")
|
||||
libdispatch.dispatch_get_global_queue.restype = ctypes.c_void_p
|
||||
libdispatch.dispatch_get_global_queue.argtypes = [
|
||||
ctypes.c_long, ctypes.c_ulong]
|
||||
# 0 = DISPATCH_QUEUE_PRIORITY_DEFAULT, 0 = flags
|
||||
q_ptr = libdispatch.dispatch_get_global_queue(0, 0)
|
||||
return objc.objc_object(c_void_p=q_ptr)
|
||||
|
||||
|
||||
def enumerate_devices() -> list[dict]:
|
||||
"""Retourne la liste des devices video disponibles avec metadata
|
||||
: uniqueID, localizedName, deviceType."""
|
||||
session = (AVF.AVCaptureDeviceDiscoverySession
|
||||
.discoverySessionWithDeviceTypes_mediaType_position_(
|
||||
_DEVICE_TYPES, "vide", 0))
|
||||
devices = list(session.devices() or [])
|
||||
out = []
|
||||
for d in devices:
|
||||
out.append({
|
||||
"uniqueID": str(d.uniqueID()),
|
||||
"name": str(d.localizedName()),
|
||||
"type": (str(d.deviceType()) if hasattr(d, "deviceType")
|
||||
else "").split(".")[-1],
|
||||
"_device": d,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
_BANNED_NAME_TOKENS = ("iphone", "gsm", "desk view", "continuity")
|
||||
|
||||
|
||||
def find_builtin_device() -> Optional[dict]:
|
||||
"""Selectionne la webcam Mac integree :
|
||||
- deviceType doit etre BuiltInWideAngleCamera
|
||||
- le nom ne doit contenir aucun de _BANNED_NAME_TOKENS
|
||||
Evite les pieges Continuity/iPhone/Desk View qui peuvent matcher
|
||||
BuiltInWideAngleCamera dans certaines configs."""
|
||||
for info in enumerate_devices():
|
||||
if "BuiltInWideAngleCamera" not in info["type"]:
|
||||
continue
|
||||
name_l = info["name"].lower()
|
||||
if any(tok in name_l for tok in _BANNED_NAME_TOKENS):
|
||||
continue
|
||||
return info
|
||||
return None
|
||||
|
||||
|
||||
class _FrameDelegate(NSObject):
|
||||
"""Delegate AVCaptureVideoDataOutput. Convertit BGRA -> BGR numpy
|
||||
et stocke la derniere frame sous lock."""
|
||||
|
||||
def init(self):
|
||||
self = objc.super(_FrameDelegate, self).init()
|
||||
if self is None:
|
||||
return None
|
||||
self._buf = None
|
||||
self._lock = threading.Lock()
|
||||
self._frame_count = 0
|
||||
return self
|
||||
|
||||
@objc.python_method
|
||||
def get_latest(self):
|
||||
with self._lock:
|
||||
if self._buf is None:
|
||||
return False, None, self._frame_count
|
||||
return True, self._buf.copy(), self._frame_count
|
||||
|
||||
def captureOutput_didOutputSampleBuffer_fromConnection_(
|
||||
self, output, sample_buffer, connection):
|
||||
try:
|
||||
self._handle_sample_buffer(sample_buffer)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("frame conv failed: %s", e)
|
||||
|
||||
@objc.python_method
|
||||
def _handle_sample_buffer(self, sample_buffer):
|
||||
pixel_buffer = CM.CMSampleBufferGetImageBuffer(sample_buffer)
|
||||
if pixel_buffer is None:
|
||||
return
|
||||
w = Quartz.CVPixelBufferGetWidth(pixel_buffer)
|
||||
h = Quartz.CVPixelBufferGetHeight(pixel_buffer)
|
||||
Quartz.CVPixelBufferLockBaseAddress(pixel_buffer, 1) # read-only
|
||||
try:
|
||||
base_addr = Quartz.CVPixelBufferGetBaseAddress(pixel_buffer)
|
||||
stride = Quartz.CVPixelBufferGetBytesPerRow(pixel_buffer)
|
||||
# base_addr est un objc.varlist (void* wrapper). On va via
|
||||
# as_buffer(N) qui renvoie un memoryview de N octets.
|
||||
if base_addr is None:
|
||||
return
|
||||
buf = base_addr.as_buffer(h * stride)
|
||||
arr = np.frombuffer(buf, dtype=np.uint8).reshape(
|
||||
(h, stride // 4, 4))[:, :w, :3] # BGRA -> BGR
|
||||
with self._lock:
|
||||
self._buf = np.ascontiguousarray(arr)
|
||||
self._frame_count += 1
|
||||
finally:
|
||||
Quartz.CVPixelBufferUnlockBaseAddress(pixel_buffer, 1)
|
||||
|
||||
|
||||
class AVCapture:
|
||||
"""API minimale cv2-like sur AVFoundation. Compatible drop-in pour
|
||||
les workers qui font `cap.read()` en boucle.
|
||||
|
||||
Usage :
|
||||
cap = AVCapture(device_info) # ou AVCapture.builtin()
|
||||
cap.start()
|
||||
ok, frame_bgr = cap.read()
|
||||
cap.stop()
|
||||
"""
|
||||
|
||||
def __init__(self, device_info: dict) -> None:
|
||||
self._info = device_info
|
||||
self._session = None
|
||||
self._delegate = None
|
||||
self._queue = None
|
||||
self._last_count = 0
|
||||
|
||||
@classmethod
|
||||
def builtin(cls) -> Optional["AVCapture"]:
|
||||
info = find_builtin_device()
|
||||
return cls(info) if info is not None else None
|
||||
|
||||
def start(self) -> bool:
|
||||
device = self._info["_device"]
|
||||
session = AVF.AVCaptureSession.alloc().init()
|
||||
try:
|
||||
input_, err = (
|
||||
AVF.AVCaptureDeviceInput
|
||||
.deviceInputWithDevice_error_(device, None))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("AVCaptureDeviceInput failed: %s", e)
|
||||
return False
|
||||
if input_ is None:
|
||||
LOG.error("input is None (err=%s)", err)
|
||||
return False
|
||||
if not session.canAddInput_(input_):
|
||||
LOG.error("session refused input")
|
||||
return False
|
||||
session.addInput_(input_)
|
||||
|
||||
output = AVF.AVCaptureVideoDataOutput.alloc().init()
|
||||
settings = {
|
||||
Quartz.kCVPixelBufferPixelFormatTypeKey:
|
||||
Quartz.kCVPixelFormatType_32BGRA,
|
||||
}
|
||||
output.setVideoSettings_(settings)
|
||||
output.setAlwaysDiscardsLateVideoFrames_(True)
|
||||
|
||||
delegate = _FrameDelegate.alloc().init()
|
||||
queue = _get_global_queue()
|
||||
output.setSampleBufferDelegate_queue_(delegate, queue)
|
||||
|
||||
if not session.canAddOutput_(output):
|
||||
LOG.error("session refused output")
|
||||
return False
|
||||
session.addOutput_(output)
|
||||
|
||||
session.startRunning()
|
||||
self._session = session
|
||||
self._delegate = delegate
|
||||
self._queue = queue
|
||||
LOG.info("AV session running on '%s' (%s)",
|
||||
self._info["name"], self._info["type"])
|
||||
return True
|
||||
|
||||
def read(self, timeout_s: float = 0.5) -> tuple[bool, Optional[np.ndarray]]:
|
||||
"""Bloque jusqu'a recevoir une frame NOUVELLE (frame_count
|
||||
different du dernier read), ou timeout. Retourne (ok, BGR
|
||||
HxWx3 uint8)."""
|
||||
if self._delegate is None:
|
||||
return False, None
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
ok, frame, count = self._delegate.get_latest()
|
||||
if ok and count != self._last_count:
|
||||
self._last_count = count
|
||||
return True, frame
|
||||
time.sleep(0.01)
|
||||
# Au timeout, on renvoie quand meme la derniere frame si elle
|
||||
# existe (pour les cas FPS < target)
|
||||
ok, frame, count = self._delegate.get_latest()
|
||||
if ok:
|
||||
self._last_count = count
|
||||
return True, frame
|
||||
return False, None
|
||||
|
||||
def stop(self) -> None:
|
||||
if self._session is not None:
|
||||
try:
|
||||
self._session.stopRunning()
|
||||
except Exception:
|
||||
pass
|
||||
self._session = None
|
||||
self._delegate = None
|
||||
self._queue = None
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Helper de selection de camera macOS : enumere les devices via
|
||||
AVFoundation et retourne l'index OpenCV qui correspond a la webcam
|
||||
built-in (BuiltInWideAngleCamera), en evitant Continuity Camera
|
||||
(iPhone), Desk View, et External."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Iterable
|
||||
|
||||
LOG = logging.getLogger("camera_select")
|
||||
|
||||
|
||||
def list_cameras() -> list[tuple[int, str, str]]:
|
||||
"""Retourne [(index, localized_name, device_type_short), ...]."""
|
||||
try:
|
||||
import objc
|
||||
from Foundation import NSBundle
|
||||
b = NSBundle.bundleWithPath_(
|
||||
"/System/Library/Frameworks/AVFoundation.framework")
|
||||
b.load()
|
||||
ns: dict = {}
|
||||
objc.loadBundle("AVFoundation", ns, b.bundlePath())
|
||||
DiscoverySession = ns["AVCaptureDeviceDiscoverySession"]
|
||||
session = (DiscoverySession
|
||||
.discoverySessionWithDeviceTypes_mediaType_position_(
|
||||
["AVCaptureDeviceTypeBuiltInWideAngleCamera",
|
||||
"AVCaptureDeviceTypeContinuityCamera",
|
||||
"AVCaptureDeviceTypeExternal",
|
||||
"AVCaptureDeviceTypeDeskViewCamera"],
|
||||
"vide", 0))
|
||||
devices = session.devices() or []
|
||||
result = []
|
||||
for i, d in enumerate(devices):
|
||||
name = str(d.localizedName())
|
||||
dtype = str(d.deviceType() if hasattr(d, "deviceType") else "")
|
||||
result.append((i, name, dtype.split(".")[-1]))
|
||||
return result
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("camera enum failed: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def pick_builtin_camera(fallback: int = 0) -> int:
|
||||
"""Retourne l'index du BuiltInWideAngleCamera, sinon fallback."""
|
||||
devices = list_cameras()
|
||||
for i, name, dtype in devices:
|
||||
LOG.info("camera [%d] %s (%s)", i, name, dtype)
|
||||
for i, _, dtype in devices:
|
||||
if "BuiltInWideAngleCamera" in dtype:
|
||||
LOG.info("camera Mac built-in -> index %d", i)
|
||||
return i
|
||||
return fallback
|
||||
|
||||
|
||||
def probe_cv2_indices(max_idx: int = 4) -> list[tuple[int, int, int, float]]:
|
||||
"""Pour chaque index cv2 0..max_idx, retourne (idx, w, h, mean) ou
|
||||
None pour les indices indisponibles. Le 'mean' est la luminance
|
||||
moyenne d'une frame — un capture standby (iPhone verrouille,
|
||||
Continuity inactive) a mean ~0-30 ; une cam active ~80+."""
|
||||
try:
|
||||
import cv2
|
||||
import numpy as np
|
||||
except ImportError:
|
||||
return []
|
||||
result = []
|
||||
for idx in range(max_idx):
|
||||
cap = cv2.VideoCapture(idx, cv2.CAP_AVFOUNDATION)
|
||||
if not cap.isOpened():
|
||||
cap.release()
|
||||
continue
|
||||
w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
# Drain 2 frames pour passer l'auto-exposure
|
||||
for _ in range(2):
|
||||
cap.read()
|
||||
ok, frame = cap.read()
|
||||
mean = float(np.mean(frame)) if (ok and frame is not None) else -1.0
|
||||
result.append((idx, w, h, mean))
|
||||
cap.release()
|
||||
return result
|
||||
|
||||
|
||||
def resolve_camera_index(requested: int, min_mean: float = 50.0) -> int:
|
||||
"""`requested=-1` -> probe cv2, prefere l'index avec frame_mean
|
||||
>= min_mean (rejette les flux noirs / standby). Sinon retourne
|
||||
`requested`."""
|
||||
if requested >= 0:
|
||||
return requested
|
||||
probes = probe_cv2_indices()
|
||||
for idx, w, h, mean in probes:
|
||||
LOG.info("cv2 probe [%d] %dx%d mean=%.1f", idx, w, h, mean)
|
||||
bright = [p for p in probes if p[3] >= min_mean]
|
||||
if bright:
|
||||
idx = bright[0][0]
|
||||
LOG.info("cv2 auto-pick index %d (mean %.1f >= %.1f)",
|
||||
idx, bright[0][3], min_mean)
|
||||
return idx
|
||||
if probes:
|
||||
LOG.warning("no bright cv2 stream — defaulting to index %d",
|
||||
probes[0][0])
|
||||
return probes[0][0]
|
||||
return 0
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Shims mmcv 1.x API surface pour SMPLer-X vendored mmpose.
|
||||
|
||||
mmcv-lite 2.x a migré la plupart des symboles vers mmengine ;
|
||||
SMPLer-X est écrit pour mmcv-full 1.x. Cette module ré-exporte
|
||||
les symboles 1.x sous leur ancien chemin avant qu'on importe la
|
||||
vendored mmpose.
|
||||
|
||||
Appel : `from data_only_viz._smplerx_shims import install_all`
|
||||
puis `install_all()` AVANT tout `import mmpose`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
import warnings
|
||||
|
||||
|
||||
def _no_op_decorator(*args, **kwargs):
|
||||
"""Décorateur no-op : remplace deprecated_api_warning."""
|
||||
if len(args) == 1 and callable(args[0]):
|
||||
return args[0]
|
||||
def wrap(fn):
|
||||
return fn
|
||||
return wrap
|
||||
|
||||
|
||||
class _SimpleTimer:
|
||||
"""Timer minimal compat mmcv.Timer 1.x."""
|
||||
def __init__(self, start: bool = True):
|
||||
self._start = time.perf_counter() if start else None
|
||||
|
||||
def start(self):
|
||||
self._start = time.perf_counter()
|
||||
|
||||
def since_start(self):
|
||||
return time.perf_counter() - (self._start or time.perf_counter())
|
||||
|
||||
def since_last_check(self):
|
||||
now = time.perf_counter()
|
||||
d = now - (self._start or now)
|
||||
self._start = now
|
||||
return d
|
||||
|
||||
|
||||
def _is_seq_of(seq, expected_type, seq_type=None):
|
||||
"""mmcv.is_seq_of -> mmengine.utils.is_seq_of (parfois absent)."""
|
||||
if seq_type is None:
|
||||
exp_seq_type = (list, tuple)
|
||||
else:
|
||||
exp_seq_type = seq_type
|
||||
if not isinstance(seq, exp_seq_type):
|
||||
return False
|
||||
return all(isinstance(item, expected_type) for item in seq)
|
||||
|
||||
|
||||
def install_all() -> None:
|
||||
"""Installer tous les shims sur mmcv pour compat 1.x API."""
|
||||
import mmcv
|
||||
|
||||
# --- top-level symbols ---
|
||||
if not hasattr(mmcv, "Config"):
|
||||
from mmengine.config import Config
|
||||
mmcv.Config = Config
|
||||
|
||||
if not hasattr(mmcv, "deprecated_api_warning"):
|
||||
mmcv.deprecated_api_warning = _no_op_decorator
|
||||
|
||||
if not hasattr(mmcv, "Timer"):
|
||||
mmcv.Timer = _SimpleTimer
|
||||
|
||||
if not hasattr(mmcv, "is_seq_of"):
|
||||
try:
|
||||
from mmengine.utils import is_seq_of
|
||||
mmcv.is_seq_of = is_seq_of
|
||||
except Exception:
|
||||
mmcv.is_seq_of = _is_seq_of
|
||||
|
||||
# --- mmcv.cnn symbols ---
|
||||
import mmcv.cnn as _cnn
|
||||
|
||||
# Init functions migrated to mmengine.model.weight_init
|
||||
try:
|
||||
from mmengine.model import (
|
||||
constant_init, normal_init, kaiming_init,
|
||||
trunc_normal_init, xavier_init, bias_init_with_prob,
|
||||
)
|
||||
except ImportError:
|
||||
# Fallback : implémentations minimales locales
|
||||
import torch.nn as nn
|
||||
|
||||
def constant_init(module, val=0, bias=0):
|
||||
if hasattr(module, 'weight') and module.weight is not None:
|
||||
nn.init.constant_(module.weight, val)
|
||||
if hasattr(module, 'bias') and module.bias is not None:
|
||||
nn.init.constant_(module.bias, bias)
|
||||
|
||||
def normal_init(module, mean=0, std=1, bias=0):
|
||||
if hasattr(module, 'weight') and module.weight is not None:
|
||||
nn.init.normal_(module.weight, mean, std)
|
||||
if hasattr(module, 'bias') and module.bias is not None:
|
||||
nn.init.constant_(module.bias, bias)
|
||||
|
||||
def kaiming_init(module, a=0, mode='fan_out',
|
||||
nonlinearity='relu', bias=0, distribution='normal'):
|
||||
if hasattr(module, 'weight') and module.weight is not None:
|
||||
if distribution == 'normal':
|
||||
nn.init.kaiming_normal_(module.weight, a=a, mode=mode,
|
||||
nonlinearity=nonlinearity)
|
||||
else:
|
||||
nn.init.kaiming_uniform_(module.weight, a=a, mode=mode,
|
||||
nonlinearity=nonlinearity)
|
||||
if hasattr(module, 'bias') and module.bias is not None:
|
||||
nn.init.constant_(module.bias, bias)
|
||||
|
||||
def trunc_normal_init(module, mean=0, std=1, a=-2, b=2, bias=0):
|
||||
if hasattr(module, 'weight') and module.weight is not None:
|
||||
nn.init.trunc_normal_(module.weight, mean, std, a, b)
|
||||
if hasattr(module, 'bias') and module.bias is not None:
|
||||
nn.init.constant_(module.bias, bias)
|
||||
|
||||
def xavier_init(module, gain=1, bias=0, distribution='normal'):
|
||||
if hasattr(module, 'weight') and module.weight is not None:
|
||||
if distribution == 'normal':
|
||||
nn.init.xavier_normal_(module.weight, gain)
|
||||
else:
|
||||
nn.init.xavier_uniform_(module.weight, gain)
|
||||
if hasattr(module, 'bias') and module.bias is not None:
|
||||
nn.init.constant_(module.bias, bias)
|
||||
|
||||
def bias_init_with_prob(prior_prob):
|
||||
import math
|
||||
return float(-math.log((1 - prior_prob) / prior_prob))
|
||||
|
||||
for name, fn in {
|
||||
"constant_init": constant_init,
|
||||
"normal_init": normal_init,
|
||||
"kaiming_init": kaiming_init,
|
||||
"trunc_normal_init": trunc_normal_init,
|
||||
"xavier_init": xavier_init,
|
||||
"bias_init_with_prob": bias_init_with_prob,
|
||||
}.items():
|
||||
if not hasattr(_cnn, name):
|
||||
setattr(_cnn, name, fn)
|
||||
|
||||
# Linear / Conv2d / MaxPool2d : juste re-export torch.nn
|
||||
import torch.nn as _nn
|
||||
for name, cls in {
|
||||
"Linear": _nn.Linear,
|
||||
"Conv2d": _nn.Conv2d,
|
||||
"MaxPool2d": _nn.MaxPool2d,
|
||||
}.items():
|
||||
if not hasattr(_cnn, name):
|
||||
setattr(_cnn, name, cls)
|
||||
|
||||
# build_model_from_cfg : si mmcv.cnn ne l'expose pas, prendre de
|
||||
# mmengine.registry.build_model_from_cfg
|
||||
if not hasattr(_cnn, "build_model_from_cfg"):
|
||||
try:
|
||||
from mmengine.registry import build_model_from_cfg
|
||||
_cnn.build_model_from_cfg = build_model_from_cfg
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# MODELS export : SMPLer-X importe `from mmcv.cnn import MODELS as
|
||||
# MMCV_MODELS`. mmcv 2.x a son MODELS dans mmcv.cnn.
|
||||
if not hasattr(_cnn, "MODELS"):
|
||||
try:
|
||||
from mmengine.registry import MODELS
|
||||
_cnn.MODELS = MODELS
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
__all__ = ["install_all"]
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Action classifier head on top of Multi-HMR j3d.
|
||||
|
||||
Streaming GRU-1-layer + MLP per-person, with a 16-frame ring buffer.
|
||||
Trained windowed (Studio M3 Ultra MPS), inferred streaming (M5 eager CPU).
|
||||
|
||||
Output per step: (label_idx, probs (3,), kin (3,)) where kin is
|
||||
(speed_m_s, accel_m_s2, symmetry_in_minus1_plus1).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
HIDDEN_DIM: int = 48
|
||||
MLP_HIDDEN: int = 32
|
||||
WARMUP_FRAMES: int = 3
|
||||
NAN_SKIP_BUDGET: int = 5
|
||||
|
||||
WINDOW_LEN: int = 16
|
||||
J3D_BODY: int = 22
|
||||
J3D_FINGERS_PER_HAND: int = 5
|
||||
J3D_FINGERS: int = 2 * J3D_FINGERS_PER_HAND # 10
|
||||
J3D_JOINTS: int = J3D_BODY + J3D_FINGERS # 32
|
||||
J3D_DIMS: int = 3
|
||||
NUM_CLASSES: int = 3
|
||||
LABELS: tuple[str, str, str] = ("debout", "assise", "danse")
|
||||
|
||||
EXPR_DIM: int = 10
|
||||
EXTRA_SCALARS: int = 4 # hip_y, knee_angle, sym_score, mouth_open
|
||||
|
||||
# NEW v3 : MediaPipe Hands keypoints block.
|
||||
HANDS_KP_PER_HAND: int = 21
|
||||
HANDS_KP_TOTAL: int = 2 * HANDS_KP_PER_HAND # 42
|
||||
HANDS_KP_DIMS: int = 3
|
||||
HANDS_KP_FLAT: int = HANDS_KP_TOTAL * HANDS_KP_DIMS # 126
|
||||
|
||||
# Layout per step (v3) :
|
||||
# [0 : 96] j3d (32, 3)
|
||||
# [96 : 192] vel (32, 3)
|
||||
# [192 : 288] accel (32, 3)
|
||||
# [288 : 414] hands_kp (42, 3) zero-padded if absent
|
||||
# [414 : 424] expression (10,)
|
||||
# [424 : 428] scalars (hip_y, knee_angle, sym, mouth_open)
|
||||
FEATURE_DIM: int = J3D_JOINTS * J3D_DIMS * 3 + HANDS_KP_FLAT + EXPR_DIM + EXTRA_SCALARS # 428
|
||||
|
||||
# Body joint indices (unchanged from v1, indices 0..21).
|
||||
HIP_LEFT: int = 1
|
||||
HIP_RIGHT: int = 2
|
||||
KNEE_LEFT: int = 4
|
||||
KNEE_RIGHT: int = 5
|
||||
ANKLE_LEFT: int = 7
|
||||
ANKLE_RIGHT: int = 8
|
||||
SHOULDER_LEFT: int = 16
|
||||
SHOULDER_RIGHT: int = 17
|
||||
WRIST_LEFT: int = 20
|
||||
WRIST_RIGHT: int = 21
|
||||
|
||||
# Fingertip indices (new, 22..31), order: L thumb..pinky, R thumb..pinky.
|
||||
FINGERTIP_LEFT_BASE: int = 22
|
||||
FINGERTIP_RIGHT_BASE: int = 27
|
||||
|
||||
|
||||
class FeatureExtractor:
|
||||
"""Stateless feature builder over a list of recent j3d frames.
|
||||
|
||||
Vector layout (FEATURE_DIM = 428, v3):
|
||||
[0 : 96] j3d current frame, flattened (32 joints x 3 dims)
|
||||
[96 : 192] velocity j3d[t] - j3d[t-1] (32 x 3)
|
||||
[192 : 288] acceleration vel[t] - vel[t-1] (32 x 3)
|
||||
[288 : 414] hands_kp (42, 3) MediaPipe Hands, zero-padded if absent
|
||||
[414 : 424] expression PCA coefficients (10,)
|
||||
[424 : 428] kinetics scalars (hip_y, knee_angle, symmetry_score, mouth_open)
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def from_buffer(frames: list[np.ndarray],
|
||||
expr: np.ndarray | None = None,
|
||||
mouth_open: float = 0.0,
|
||||
hands_kp: np.ndarray | None = None) -> np.ndarray:
|
||||
if not frames:
|
||||
return np.zeros(FEATURE_DIM, dtype=np.float32)
|
||||
cur = frames[-1]
|
||||
prev = frames[-2] if len(frames) >= 2 else cur
|
||||
prev2 = frames[-3] if len(frames) >= 3 else prev
|
||||
vel = (cur - prev).astype(np.float32, copy=False)
|
||||
prev_vel = (prev - prev2).astype(np.float32, copy=False)
|
||||
accel = (vel - prev_vel).astype(np.float32, copy=False)
|
||||
hip_y = float((cur[HIP_LEFT, 1] + cur[HIP_RIGHT, 1]) * 0.5)
|
||||
knee_angle = FeatureExtractor._mean_knee_angle(cur)
|
||||
sym = FeatureExtractor._symmetry_score(vel)
|
||||
# hands block (42, 3) -> 126
|
||||
hands_flat = np.zeros(HANDS_KP_FLAT, dtype=np.float32)
|
||||
if hands_kp is not None:
|
||||
hk = np.asarray(hands_kp, dtype=np.float32)
|
||||
if hk.shape == (HANDS_KP_TOTAL, HANDS_KP_DIMS):
|
||||
hands_flat = hk.reshape(-1).astype(np.float32, copy=False)
|
||||
# expression
|
||||
if expr is None:
|
||||
expr_vec = np.zeros(EXPR_DIM, dtype=np.float32)
|
||||
else:
|
||||
expr_vec = np.zeros(EXPR_DIM, dtype=np.float32)
|
||||
n = min(EXPR_DIM, len(expr))
|
||||
expr_vec[:n] = expr[:n]
|
||||
return np.concatenate([
|
||||
cur.reshape(-1),
|
||||
vel.reshape(-1),
|
||||
accel.reshape(-1),
|
||||
hands_flat,
|
||||
expr_vec,
|
||||
np.array([hip_y, knee_angle, sym, float(mouth_open)], dtype=np.float32),
|
||||
]).astype(np.float32, copy=False)
|
||||
|
||||
@staticmethod
|
||||
def kinetics(frames: list[np.ndarray]) -> np.ndarray:
|
||||
"""Return (speed, accel_mag, symmetry) averaged over the buffer."""
|
||||
if len(frames) < 2:
|
||||
return np.zeros(3, dtype=np.float32)
|
||||
arr = np.stack(frames).astype(np.float32, copy=False)
|
||||
diffs = arr[1:] - arr[:-1]
|
||||
speeds = np.linalg.norm(diffs, axis=-1).mean(axis=-1)
|
||||
speed = float(speeds.mean())
|
||||
if len(frames) >= 3:
|
||||
ddiffs = diffs[1:] - diffs[:-1]
|
||||
accel = float(np.linalg.norm(ddiffs, axis=-1).mean())
|
||||
else:
|
||||
accel = 0.0
|
||||
sym = FeatureExtractor._symmetry_score(diffs[-1])
|
||||
return np.array([speed, accel, sym], dtype=np.float32)
|
||||
|
||||
@staticmethod
|
||||
def _mean_knee_angle(j3d: np.ndarray) -> float:
|
||||
"""Angle (rad) at left+right knees, averaged."""
|
||||
def _angle(hip: int, knee: int, ankle: int) -> float:
|
||||
v1 = j3d[hip] - j3d[knee]
|
||||
v2 = j3d[ankle] - j3d[knee]
|
||||
n1 = np.linalg.norm(v1) + 1e-6
|
||||
n2 = np.linalg.norm(v2) + 1e-6
|
||||
cos = float(np.dot(v1, v2) / (n1 * n2))
|
||||
return float(np.arccos(np.clip(cos, -1.0, 1.0)))
|
||||
return 0.5 * (_angle(HIP_LEFT, KNEE_LEFT, ANKLE_LEFT)
|
||||
+ _angle(HIP_RIGHT, KNEE_RIGHT, ANKLE_RIGHT))
|
||||
|
||||
@staticmethod
|
||||
def _symmetry_score(vel: np.ndarray) -> float:
|
||||
"""Cosine sim between left-arm and mirrored right-arm velocity."""
|
||||
left = vel[WRIST_LEFT].copy()
|
||||
right = vel[WRIST_RIGHT].copy()
|
||||
right_mirror = right.copy()
|
||||
right_mirror[0] = -right_mirror[0]
|
||||
n1 = np.linalg.norm(left) + 1e-6
|
||||
n2 = np.linalg.norm(right_mirror) + 1e-6
|
||||
return float(np.dot(left, right_mirror) / (n1 * n2))
|
||||
|
||||
|
||||
class PerPersonBuffer:
|
||||
"""Per-pid ring buffer of j3d frames (deque maxlen=WINDOW_LEN)."""
|
||||
|
||||
__slots__ = ("_buffers",)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._buffers: dict[int, deque[np.ndarray]] = {}
|
||||
|
||||
def append(self, pid: int, j3d: np.ndarray) -> None:
|
||||
if j3d.shape != (J3D_JOINTS, J3D_DIMS):
|
||||
raise ValueError(
|
||||
f"j3d must be ({J3D_JOINTS}, {J3D_DIMS}), got {j3d.shape}"
|
||||
)
|
||||
dq = self._buffers.get(pid)
|
||||
if dq is None:
|
||||
dq = deque(maxlen=WINDOW_LEN)
|
||||
self._buffers[pid] = dq
|
||||
dq.append(j3d.astype(np.float32, copy=False))
|
||||
|
||||
def frames_for(self, pid: int) -> list[np.ndarray]:
|
||||
dq = self._buffers.get(pid)
|
||||
return list(dq) if dq is not None else []
|
||||
|
||||
def forget(self, pid: int) -> None:
|
||||
self._buffers.pop(pid, None)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._buffers)
|
||||
|
||||
def pids(self) -> list[int]:
|
||||
return list(self._buffers.keys())
|
||||
|
||||
|
||||
class ActionHeadModel(nn.Module):
|
||||
"""1-layer GRU + small MLP head.
|
||||
|
||||
Input : (B, FEATURE_DIM) -- single step
|
||||
Hidden : (1, B, HIDDEN_DIM)
|
||||
Output : (B, NUM_CLASSES) logits, new hidden
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.gru = nn.GRU(input_size=FEATURE_DIM,
|
||||
hidden_size=HIDDEN_DIM,
|
||||
num_layers=1,
|
||||
batch_first=True)
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(HIDDEN_DIM, MLP_HIDDEN),
|
||||
nn.ReLU(inplace=True),
|
||||
nn.Linear(MLP_HIDDEN, NUM_CLASSES),
|
||||
)
|
||||
|
||||
def init_hidden(self, batch: int = 1, device: str = "cpu") -> torch.Tensor:
|
||||
return torch.zeros(1, batch, HIDDEN_DIM, device=device)
|
||||
|
||||
def forward(self, x: torch.Tensor,
|
||||
h: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
out, h_new = self.gru(x.unsqueeze(1), h)
|
||||
logits = self.mlp(out.squeeze(1))
|
||||
return logits, h_new
|
||||
|
||||
|
||||
class ActionHead:
|
||||
"""Streaming action classifier per person.
|
||||
|
||||
Use:
|
||||
head = ActionHead(ckpt_path=...)
|
||||
label, probs, kin = head.step(pid, j3d)
|
||||
head.forget(pid)
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
ckpt_path: Path | None = None,
|
||||
device: str = "cpu") -> None:
|
||||
self._device = device
|
||||
self._model = ActionHeadModel().to(device).eval()
|
||||
if ckpt_path is not None:
|
||||
payload = torch.load(ckpt_path, map_location=device,
|
||||
weights_only=True)
|
||||
state = payload.get("model_state_dict", payload)
|
||||
self._model.load_state_dict(state)
|
||||
self._buffers = PerPersonBuffer()
|
||||
self._hidden: dict[int, torch.Tensor] = {}
|
||||
self._nan_streak: dict[int, int] = {}
|
||||
|
||||
def step(self, pid: int, j3d: np.ndarray,
|
||||
expr: np.ndarray | None = None,
|
||||
mouth_open: float = 0.0,
|
||||
hands_kp: np.ndarray | None = None) -> tuple[str, np.ndarray, np.ndarray]:
|
||||
if np.isnan(j3d).any():
|
||||
streak = self._nan_streak.get(pid, 0) + 1
|
||||
self._nan_streak[pid] = streak
|
||||
if streak > NAN_SKIP_BUDGET:
|
||||
self.forget(pid)
|
||||
probs = np.array([1.0, 0.0, 0.0], dtype=np.float32)
|
||||
return LABELS[0], probs, np.zeros(3, dtype=np.float32)
|
||||
self._nan_streak[pid] = 0
|
||||
self._buffers.append(pid, j3d)
|
||||
frames = self._buffers.frames_for(pid)
|
||||
if len(frames) < WARMUP_FRAMES:
|
||||
probs = np.array([1.0, 0.0, 0.0], dtype=np.float32)
|
||||
return LABELS[0], probs, np.zeros(3, dtype=np.float32)
|
||||
feat = FeatureExtractor.from_buffer(frames, expr=expr, mouth_open=mouth_open,
|
||||
hands_kp=hands_kp)
|
||||
kin = FeatureExtractor.kinetics(frames)
|
||||
h = self._hidden.get(pid)
|
||||
if h is None:
|
||||
h = self._model.init_hidden(batch=1, device=self._device)
|
||||
x = torch.from_numpy(feat).unsqueeze(0).to(self._device)
|
||||
with torch.no_grad():
|
||||
logits, h_new = self._model(x, h)
|
||||
probs_t = torch.softmax(logits, dim=-1).squeeze(0)
|
||||
self._hidden[pid] = h_new
|
||||
probs = probs_t.cpu().numpy().astype(np.float32, copy=False)
|
||||
return LABELS[int(np.argmax(probs))], probs, kin
|
||||
|
||||
def forget(self, pid: int) -> None:
|
||||
self._buffers.forget(pid)
|
||||
self._hidden.pop(pid, None)
|
||||
self._nan_streak.pop(pid, None)
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Action-head publisher : reads state.persons_smplx / persons_body3d,
|
||||
runs ActionHead per pid, emits /pose/action and /pose/kin via pose_bridge.
|
||||
|
||||
Stand-alone thread to avoid touching multi_hmr_worker.py while it
|
||||
iterates. Polls state at ~30 Hz, deduplicates by smplx_last_t.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from data_only_viz.action_head import (
|
||||
ActionHead,
|
||||
EXPR_DIM,
|
||||
HANDS_KP_DIMS,
|
||||
HANDS_KP_PER_HAND,
|
||||
HANDS_KP_TOTAL,
|
||||
J3D_FINGERS,
|
||||
J3D_FINGERS_PER_HAND,
|
||||
LABELS,
|
||||
)
|
||||
|
||||
LOG = logging.getLogger("action_head_pub")
|
||||
|
||||
DEFAULT_CKPT = (
|
||||
Path.home() / ".cache" / "av-live-action" / "checkpoints" / "action_head.pt"
|
||||
)
|
||||
|
||||
# Canonical SMPL-X fingertip vertex IDs from smplx.vertex_ids.SMPLX_VERTEX_IDS.
|
||||
# Order : L thumb, L index, L middle, L ring, L pinky,
|
||||
# R thumb, R index, R middle, R ring, R pinky.
|
||||
SMPLX_FINGERTIP_VERTS: tuple[int, ...] = (
|
||||
5361, 4933, 5058, 5169, 5286, # L : lthumb, lindex, lmiddle, lring, lpinky
|
||||
8079, 7669, 7794, 7905, 8022, # R : rthumb, rindex, rmiddle, rring, rpinky
|
||||
)
|
||||
|
||||
# 32 vertex indices on the 10475-vertex SMPL-X mesh:
|
||||
# 22 body (UNCHANGED from v1) + 10 fingertips.
|
||||
# NOTE: approximate vertex anchors -- real SMPL-X joints come from
|
||||
# J_regressor @ v3d, but loading the regressor here is avoided for
|
||||
# live OSC performance. Action-head training must use the same anchors.
|
||||
SMPLX_JOINT_ANCHOR_VERTS: tuple[int, ...] = (
|
||||
# 22 body (UNCHANGED indices, same vertex IDs as before)
|
||||
8204, 3992, 6677, 3500, 3469, 6394, 3279, 3327, 6736, 3074,
|
||||
8846, 8889, 8848, 1300, 4660, 8964, 3013, 6470, 1602, 5083,
|
||||
2114, 5559,
|
||||
# 10 fingertips
|
||||
*SMPLX_FINGERTIP_VERTS,
|
||||
)
|
||||
assert len(SMPLX_JOINT_ANCHOR_VERTS) == 32
|
||||
|
||||
# Mouth-open: distance between two lip vertices on SMPL-X mesh.
|
||||
# vert 8970 (upper outer lip), 8855 (lower outer lip) -- approximate.
|
||||
SMPLX_UPPER_LIP_VERT: int = 8970
|
||||
SMPLX_LOWER_LIP_VERT: int = 8855
|
||||
|
||||
# MediaPipe FaceMesh inner-mouth landmark indices.
|
||||
# 13 = upper inner mid, 14 = lower inner mid.
|
||||
MEDIAPIPE_LIP_UPPER_INNER: int = 13
|
||||
MEDIAPIPE_LIP_LOWER_INNER: int = 14
|
||||
|
||||
# MediaPipe HAND fingertip indices (21-kp hand model).
|
||||
MEDIAPIPE_HAND_FINGERTIPS: tuple[int, ...] = (4, 8, 12, 16, 20)
|
||||
|
||||
# MediaPipe 33-landmark indices mapped into the 22-joint slot order.
|
||||
# NOTE: approximate mapping -- spine joints reuse hip/shoulder anchors.
|
||||
# https://developers.google.com/mediapipe/solutions/vision/pose_landmarker
|
||||
MEDIAPIPE_TO_22: tuple[int, ...] = (
|
||||
24, 23, 24, 23, 25, 26, 11, 27, 28, 11,
|
||||
31, 32, 0, 11, 12, 0, 11, 12, 13, 14, 15, 16,
|
||||
)
|
||||
|
||||
|
||||
class ActionHeadPublisher(threading.Thread):
|
||||
"""Thread that polls state, runs ActionHead per pid, emits OSC."""
|
||||
|
||||
def __init__(self, state: Any, bridge: Any,
|
||||
ckpt_path: Path | None = DEFAULT_CKPT,
|
||||
period_s: float = 1.0 / 30.0) -> None:
|
||||
super().__init__(daemon=True, name="action-head-pub")
|
||||
self.state = state
|
||||
self.bridge = bridge
|
||||
self.period = period_s
|
||||
try:
|
||||
ckpt = ckpt_path if (ckpt_path and ckpt_path.exists()) else None
|
||||
self.head = ActionHead(ckpt_path=ckpt, device="cpu")
|
||||
LOG.info("action_head loaded ckpt=%s",
|
||||
ckpt if ckpt else "<random init>")
|
||||
except Exception as e:
|
||||
LOG.warning("action_head init failed: %s", e)
|
||||
self.head = None
|
||||
self._stop = threading.Event()
|
||||
self._last_smplx_t = 0.0
|
||||
self._last_body_t = 0.0
|
||||
self._last_pids: set[int] = set()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def run(self) -> None:
|
||||
if self.head is None:
|
||||
LOG.warning("publisher exiting: no action_head")
|
||||
return
|
||||
LOG.info("publisher started")
|
||||
while not self._stop.is_set():
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
self._tick(t0)
|
||||
except Exception:
|
||||
LOG.exception("publisher tick failed")
|
||||
dt = time.perf_counter() - t0
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
LOG.info("publisher stopped")
|
||||
|
||||
def _tick(self, t_now: float) -> None:
|
||||
persons32, source_t, source_tag, is_new = self._read_sources()
|
||||
if not is_new:
|
||||
return
|
||||
if "smplx" in source_tag:
|
||||
self._last_smplx_t = source_t
|
||||
else:
|
||||
self._last_body_t = source_t
|
||||
current_pids: set[int] = set()
|
||||
if persons32:
|
||||
for pid, j3d, expr_np, mouth, hands_kp42 in persons32:
|
||||
current_pids.add(pid)
|
||||
label, probs, kin = self.head.step(pid, j3d, expr=expr_np,
|
||||
mouth_open=mouth,
|
||||
hands_kp=hands_kp42)
|
||||
idx = LABELS.index(label)
|
||||
self.bridge.send_action(pid, idx, probs, t_now, force=True)
|
||||
self.bridge.send_kin(pid, kin, t_now, force=True)
|
||||
if pid not in self._last_pids:
|
||||
self.bridge.send_enter(pid=pid)
|
||||
for gone in self._last_pids - current_pids:
|
||||
self.head.forget(gone)
|
||||
self.bridge.send_leave(pid=gone)
|
||||
self._last_pids = current_pids
|
||||
|
||||
def _read_sources(self) -> tuple[
|
||||
list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] | None,
|
||||
float, str, bool,
|
||||
]:
|
||||
"""Return (persons32, source_t, source_tag, is_new).
|
||||
|
||||
Each person entry is (pid, j3d32, expr10, mouth_open, hands_kp42x3).
|
||||
is_new is True when the timestamp advanced (even if person list
|
||||
is empty), so _tick can still run the purge loop.
|
||||
"""
|
||||
with self.state.lock():
|
||||
persons_smplx = getattr(self.state, "persons_smplx", None)
|
||||
t_smplx = getattr(self.state, "smplx_last_t", 0.0)
|
||||
persons_b3d = getattr(self.state, "persons_body3d", None)
|
||||
ids_b3d = getattr(self.state, "persons_body_ids", None)
|
||||
persons_face = getattr(self.state, "persons_face", None)
|
||||
ids_face = getattr(self.state, "persons_face_ids", None)
|
||||
persons_hands = getattr(self.state, "persons_hands", None)
|
||||
ids_hands = getattr(self.state, "persons_hands_ids", None)
|
||||
t_body = getattr(self.state, "pose_last_t", 0.0)
|
||||
|
||||
# Build pid -> hands_kp(42, 3) map from MediaPipe persons_hands.
|
||||
hands_by_pid: dict[int, np.ndarray] = self._build_hands_map(
|
||||
persons_hands or [], ids_hands or [],
|
||||
)
|
||||
# Build pid -> mouth_open scalar from MediaPipe persons_face lips.
|
||||
face_mouth_by_pid: dict[int, float] = self._build_face_mouth_map(
|
||||
persons_face or [], ids_face or [],
|
||||
)
|
||||
|
||||
# SMPL-X path (preferred)
|
||||
if t_smplx > self._last_smplx_t:
|
||||
out: list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] = []
|
||||
for i, p in enumerate(persons_smplx or []):
|
||||
pid = int(p.get("pid", i))
|
||||
v3d = p.get("v3d")
|
||||
if v3d is None:
|
||||
continue
|
||||
# CoreMLArray wraps a numpy array but has no __array__
|
||||
# protocol; unwrap via .numpy() before np.asarray.
|
||||
if hasattr(v3d, "numpy") and not isinstance(v3d, np.ndarray):
|
||||
v3d = v3d.numpy()
|
||||
v3d_np = np.asarray(v3d, dtype=np.float32)
|
||||
if v3d_np.shape[0] < max(SMPLX_JOINT_ANCHOR_VERTS) + 1:
|
||||
continue
|
||||
j3d32 = v3d_np[list(SMPLX_JOINT_ANCHOR_VERTS)].astype(np.float32)
|
||||
# expression
|
||||
expr = p.get("expression")
|
||||
if expr is not None:
|
||||
if hasattr(expr, "numpy") and not isinstance(expr, np.ndarray):
|
||||
expr = expr.numpy()
|
||||
expr_np = np.asarray(expr, dtype=np.float32).flatten()
|
||||
else:
|
||||
expr_np = np.zeros(EXPR_DIM, dtype=np.float32)
|
||||
# mouth_open: prefer MediaPipe face lips, fallback SMPL-X v3d.
|
||||
if pid in face_mouth_by_pid:
|
||||
mouth = face_mouth_by_pid[pid]
|
||||
elif v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT):
|
||||
mouth = float(np.linalg.norm(
|
||||
v3d_np[SMPLX_UPPER_LIP_VERT] - v3d_np[SMPLX_LOWER_LIP_VERT]
|
||||
))
|
||||
else:
|
||||
mouth = 0.0
|
||||
hands_kp42 = hands_by_pid.get(
|
||||
pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32)
|
||||
)
|
||||
out.append((pid, j3d32, expr_np, mouth, hands_kp42))
|
||||
return out or None, t_smplx, "smplx", True
|
||||
|
||||
# MediaPipe body3d fallback
|
||||
if t_body > self._last_body_t:
|
||||
ids = ids_b3d or list(range(len(persons_b3d or [])))
|
||||
out = []
|
||||
for i, body in enumerate(persons_b3d or []):
|
||||
pid = int(ids[i]) if i < len(ids) else i
|
||||
arr = self._kp_list_to_array(body)
|
||||
if arr is None or arr.shape[0] < 33:
|
||||
continue
|
||||
body22 = arr[list(MEDIAPIPE_TO_22)].astype(np.float32)
|
||||
# fingertips from persons_hands if available
|
||||
tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32)
|
||||
hands_kp42 = hands_by_pid.get(
|
||||
pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32)
|
||||
)
|
||||
# extract fingertips from hands_kp42 (idx 4,8,12,16,20 each side)
|
||||
for side_idx in (0, 1):
|
||||
base = side_idx * HANDS_KP_PER_HAND
|
||||
for k, mp_tip in enumerate(MEDIAPIPE_HAND_FINGERTIPS):
|
||||
if base + mp_tip < hands_kp42.shape[0]:
|
||||
tips[side_idx * J3D_FINGERS_PER_HAND + k] = \
|
||||
hands_kp42[base + mp_tip]
|
||||
j3d32 = np.concatenate([body22, tips], axis=0)
|
||||
mouth = face_mouth_by_pid.get(pid, 0.0)
|
||||
expr_np = np.zeros(EXPR_DIM, dtype=np.float32)
|
||||
out.append((pid, j3d32, expr_np, mouth, hands_kp42))
|
||||
return out or None, t_body, "body3d", True
|
||||
return None, 0.0, "", False
|
||||
|
||||
def _build_hands_map(self, persons_hands: list,
|
||||
ids_hands: list) -> dict[int, np.ndarray]:
|
||||
"""Combine left+right hand kp arrays per pid into a single (42, 3) array.
|
||||
|
||||
persons_hands is a flat list ; ids_hands maps each hand-list entry to a
|
||||
pid (and odd/even index indicates which side). When the user's pipeline
|
||||
keeps a different convention, this helper makes the best effort and
|
||||
pads zeros for missing sides.
|
||||
"""
|
||||
out: dict[int, np.ndarray] = {}
|
||||
for hi, hkp in enumerate(persons_hands):
|
||||
if hkp is None:
|
||||
continue
|
||||
pid_raw = ids_hands[hi] if hi < len(ids_hands) else hi
|
||||
try:
|
||||
pid = int(pid_raw)
|
||||
except (TypeError, ValueError):
|
||||
pid = hi
|
||||
side = hi % 2 # 0 = L, 1 = R
|
||||
arr = self._kp_list_to_array(hkp)
|
||||
if arr is None or arr.shape[0] < HANDS_KP_PER_HAND:
|
||||
continue
|
||||
slot = out.setdefault(
|
||||
pid, np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32)
|
||||
)
|
||||
base = side * HANDS_KP_PER_HAND
|
||||
slot[base:base + HANDS_KP_PER_HAND] = arr[:HANDS_KP_PER_HAND]
|
||||
return out
|
||||
|
||||
def _build_face_mouth_map(self, persons_face: list,
|
||||
ids_face: list) -> dict[int, float]:
|
||||
"""Compute mouth_open = norm(upper_inner_lip - lower_inner_lip) per pid."""
|
||||
out: dict[int, float] = {}
|
||||
for fi, fkp in enumerate(persons_face):
|
||||
if fkp is None:
|
||||
continue
|
||||
arr = self._kp_list_to_array(fkp)
|
||||
if arr is None or arr.shape[0] <= MEDIAPIPE_LIP_LOWER_INNER:
|
||||
continue
|
||||
upper = arr[MEDIAPIPE_LIP_UPPER_INNER]
|
||||
lower = arr[MEDIAPIPE_LIP_LOWER_INNER]
|
||||
mouth = float(np.linalg.norm(upper - lower))
|
||||
try:
|
||||
pid = int(ids_face[fi]) if fi < len(ids_face) else fi
|
||||
except (TypeError, ValueError):
|
||||
pid = fi
|
||||
out[pid] = mouth
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _kp_list_to_array(body: Any) -> np.ndarray | None:
|
||||
"""Best-effort conversion of a body keypoint list to (N, 3) array."""
|
||||
if body is None:
|
||||
return None
|
||||
if isinstance(body, np.ndarray):
|
||||
return body
|
||||
try:
|
||||
return np.asarray(
|
||||
[
|
||||
(
|
||||
getattr(kp, "x", kp[0]),
|
||||
getattr(kp, "y", kp[1]),
|
||||
getattr(kp, "z", kp[2] if len(kp) > 2 else 0.0),
|
||||
)
|
||||
for kp in body
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
except (TypeError, IndexError, AttributeError):
|
||||
return None
|
||||
@@ -0,0 +1,628 @@
|
||||
"""Apple Vision body pose — version cv2 + Vision (sans AVCaptureSession).
|
||||
|
||||
Le pipeline AVCaptureSession + dispatch_queue + delegate crashe silencieusement
|
||||
sur Python 3.14 (probablement libdispatch via ctypes). On bypass : capture
|
||||
webcam via cv2.VideoCapture (deja eprouve dans multi.py), inference Vision
|
||||
via VNImageRequestHandler en passant un JPEG bytes.
|
||||
|
||||
Avantages :
|
||||
- Pas de delegate AVF, pas de dispatch_queue ctypes
|
||||
- Pattern thread daemon classique, robuste
|
||||
- VNDetectHumanBodyPoseRequest ANE-accelerated meme via JPEG handler
|
||||
|
||||
Inconvenients vs AVCaptureSession :
|
||||
- Encodage JPEG entre cv2 et Vision (~3 ms overhead)
|
||||
- Pas de zero-copy CVPixelBuffer
|
||||
|
||||
Active : AV_LIVE_APPLE_VISION=1 uv run python -m data_only_viz.main --pose
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
import objc
|
||||
from Foundation import NSBundle, NSData
|
||||
|
||||
from .euro_filter import SkeletonFilter
|
||||
from .fine_analysis import FineAnalyzer
|
||||
from .mesh_topology import FACE_OFFSETS
|
||||
from .pose_bridge import PoseSoundBridge
|
||||
from .state import PoseKp, State
|
||||
from .tracker import IoUTracker
|
||||
|
||||
LOG = logging.getLogger("apple_vision_pose")
|
||||
|
||||
|
||||
# Ordre des 21 joints VNHumanHandPoseObservation (standard MediaPipe).
|
||||
HAND_JOINTS: tuple[str, ...] = (
|
||||
"VNHLKWrist",
|
||||
"VNHLKThumbCMC", "VNHLKThumbMP", "VNHLKThumbIP", "VNHLKThumbTip",
|
||||
"VNHLKIndexMCP", "VNHLKIndexPIP", "VNHLKIndexDIP", "VNHLKIndexTip",
|
||||
"VNHLKMiddleMCP", "VNHLKMiddlePIP", "VNHLKMiddleDIP", "VNHLKMiddleTip",
|
||||
"VNHLKRingMCP", "VNHLKRingPIP", "VNHLKRingDIP", "VNHLKRingTip",
|
||||
"VNHLKLittleMCP", "VNHLKLittlePIP", "VNHLKLittleDIP", "VNHLKLittleTip",
|
||||
)
|
||||
|
||||
# Regions de VNFaceLandmarks2D dans l'ordre attendu par FACE_OFFSETS.
|
||||
FACE_REGIONS: tuple[tuple[str, int], ...] = (
|
||||
("faceContour", 17),
|
||||
("leftEye", 8),
|
||||
("rightEye", 8),
|
||||
("leftEyebrow", 6),
|
||||
("rightEyebrow", 6),
|
||||
("outerLips", 14),
|
||||
("innerLips", 10),
|
||||
("nose", 6),
|
||||
("medianLine", 6),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Charge Vision via loadBundle (pas de pyobjc-framework-Vision sur PyPI)
|
||||
# ---------------------------------------------------------------------------
|
||||
_NS: dict = {}
|
||||
_LOADED = False
|
||||
|
||||
|
||||
def _load_vision() -> dict:
|
||||
"""Charge Vision.framework dans le namespace _NS (lazy, idempotent)."""
|
||||
global _LOADED
|
||||
if _LOADED:
|
||||
return _NS
|
||||
bundle = NSBundle.bundleWithPath_("/System/Library/Frameworks/Vision.framework")
|
||||
if bundle is None or not bundle.load():
|
||||
raise RuntimeError("Impossible de charger Vision.framework")
|
||||
objc.loadBundle("Vision", _NS, bundle.bundlePath())
|
||||
# Enregistrer la metadata explicite : pointAtIndex_ prend un NSUInteger
|
||||
# et retourne un CGPoint struct (pas une id). Sans ca pyobjc voit
|
||||
# un selector 0-arg et plante avec "Need 0 arguments, got 1".
|
||||
try:
|
||||
objc.registerMetaDataForSelector(
|
||||
b'VNFaceLandmarkRegion2D', b'pointAtIndex:',
|
||||
{
|
||||
'arguments': {2: {'type': objc._C_NSUInteger}},
|
||||
'retval': {'type': b'{CGPoint=dd}'},
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
_LOADED = True
|
||||
return _NS
|
||||
|
||||
|
||||
# Mapping joint Apple Vision -> indice MediaPipe POSE_LANDMARKS
|
||||
# (cf https://developers.google.com/mediapipe/solutions/vision/pose_landmarker)
|
||||
JOINT_MAP: dict[str, int] = {
|
||||
"nose": 0,
|
||||
"left_eye": 1,
|
||||
"right_eye": 4,
|
||||
"left_ear": 7,
|
||||
"right_ear": 8,
|
||||
"left_shoulder": 11,
|
||||
"right_shoulder": 12,
|
||||
"left_elbow": 13,
|
||||
"right_elbow": 14,
|
||||
"left_wrist": 15,
|
||||
"right_wrist": 16,
|
||||
"left_hip": 23,
|
||||
"right_hip": 24,
|
||||
"left_knee": 25,
|
||||
"right_knee": 26,
|
||||
"left_ankle": 27,
|
||||
"right_ankle": 28,
|
||||
}
|
||||
|
||||
|
||||
class AppleVisionPoseWorker:
|
||||
"""Worker thread : cv2.VideoCapture + VNDetectHumanBodyPoseRequest."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state: State,
|
||||
camera_index: int = 0,
|
||||
target_fps: float = 30.0,
|
||||
num_persons: int = 4,
|
||||
score_thresh: float = 0.30,
|
||||
) -> None:
|
||||
self.state = state
|
||||
self.camera_index = camera_index
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self.target_fps = target_fps
|
||||
self.num_persons = num_persons
|
||||
self.score_thresh = score_thresh
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
# Lissage + tracking (reutilises de multi.py)
|
||||
self._tracker = IoUTracker(iou_threshold=0.20, max_miss=10)
|
||||
self._smooth = SkeletonFilter(min_cutoff=1.2, beta=0.06)
|
||||
# Pont OSC pose -> sclang (pour piloter du son live)
|
||||
self._sound_bridge = PoseSoundBridge(throttle_hz=30.0)
|
||||
# Analyse fine : crops haute resolution sur visage/mains
|
||||
# (cadence 10 Hz pour ne pas saturer ANE)
|
||||
self._fine_analyzer: FineAnalyzer | None = None
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
"""True si Vision.framework + cv2 + Foundation chargent OK."""
|
||||
try:
|
||||
_load_vision()
|
||||
import cv2 # noqa: F401
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="apple-vision-pose", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _pick_builtin_camera(self) -> int:
|
||||
"""Energe les cameras via AVFoundation, retourne l'index de la
|
||||
BuiltIn (MacBook Pro / FaceTime HD), evite Continuity Camera
|
||||
(iPhone) et Desk View. Fallback sur self.camera_index (0)."""
|
||||
try:
|
||||
from Foundation import NSBundle
|
||||
b = NSBundle.bundleWithPath_(
|
||||
"/System/Library/Frameworks/AVFoundation.framework")
|
||||
b.load()
|
||||
ns = {}
|
||||
objc.loadBundle("AVFoundation", ns, b.bundlePath())
|
||||
DiscoverySession = ns["AVCaptureDeviceDiscoverySession"]
|
||||
session = (DiscoverySession
|
||||
.discoverySessionWithDeviceTypes_mediaType_position_(
|
||||
["AVCaptureDeviceTypeBuiltInWideAngleCamera",
|
||||
"AVCaptureDeviceTypeContinuityCamera",
|
||||
"AVCaptureDeviceTypeExternal",
|
||||
"AVCaptureDeviceTypeDeskViewCamera"],
|
||||
"vide", 0))
|
||||
devices = session.devices() or []
|
||||
for i, d in enumerate(devices):
|
||||
name = str(d.localizedName())
|
||||
dtype = str(d.deviceType() if hasattr(d, "deviceType") else "")
|
||||
LOG.info("camera [%d] %s (%s)", i, name, dtype.split(".")[-1])
|
||||
# Cherche l'index BuiltInWideAngleCamera
|
||||
for i, d in enumerate(devices):
|
||||
dtype = str(d.deviceType() if hasattr(d, "deviceType") else "")
|
||||
if "BuiltInWideAngleCamera" in dtype:
|
||||
LOG.info("camera Mac built-in -> index %d", i)
|
||||
return i
|
||||
except Exception as e:
|
||||
LOG.warning("camera enum failed: %s", e)
|
||||
return self.camera_index
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
import cv2
|
||||
import numpy as np # noqa: F401
|
||||
except ModuleNotFoundError as e:
|
||||
LOG.error("deps manquantes : %s — uv sync --extra pose", e)
|
||||
return
|
||||
try:
|
||||
ns = _load_vision()
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("Vision.framework KO : %s", e)
|
||||
return
|
||||
|
||||
VNImageRequestHandler = ns["VNImageRequestHandler"]
|
||||
VNDetectHumanBodyPoseRequest = ns["VNDetectHumanBodyPoseRequest"]
|
||||
# Face + hands : noms exposes par Vision.framework.
|
||||
VNDetectFaceLandmarksRequest = ns.get("VNDetectFaceLandmarksRequest")
|
||||
VNDetectHumanHandPoseRequest = ns.get("VNDetectHumanHandPoseRequest")
|
||||
if VNDetectFaceLandmarksRequest is None:
|
||||
LOG.warning("VNDetectFaceLandmarksRequest absent — face mesh OFF")
|
||||
if VNDetectHumanHandPoseRequest is None:
|
||||
LOG.warning("VNDetectHumanHandPoseRequest absent — hand mesh OFF")
|
||||
|
||||
# Force cam Mac built-in (evite Continuity Camera iPhone par defaut)
|
||||
cam_idx = self._pick_builtin_camera()
|
||||
cap = cv2.VideoCapture(cam_idx)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index %d indisponible (TCC ?)", cam_idx)
|
||||
return
|
||||
LOG.info("camera ouverte index=%d — VNDetectHumanBodyPoseRequest ANE actif", cam_idx)
|
||||
|
||||
n_frames = 0
|
||||
sum_ms = 0.0
|
||||
while not self._stop.is_set():
|
||||
tA = time.monotonic()
|
||||
ok, frame_bgr = cap.read()
|
||||
if not ok or frame_bgr is None:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
h, w = frame_bgr.shape[:2]
|
||||
|
||||
# Encode JPEG une seule fois (webcam HUD + Vision input)
|
||||
ok2, jpg = cv2.imencode(".jpg", frame_bgr,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), 75])
|
||||
if not ok2:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
jpg_bytes = bytes(jpg)
|
||||
|
||||
# Vision : VNImageRequestHandler accepte des bytes via NSData
|
||||
data = NSData.dataWithBytes_length_(jpg_bytes, len(jpg_bytes))
|
||||
handler = VNImageRequestHandler.alloc().initWithData_options_(
|
||||
data, {})
|
||||
request = VNDetectHumanBodyPoseRequest.alloc().init()
|
||||
# On execute 3 requetes (body + face + hands) sur la MEME
|
||||
# frame : 1 inference ANE pour les 3 (parallelisation interne
|
||||
# Vision). Si une requete est indisponible, on passe.
|
||||
requests = [request]
|
||||
face_request = None
|
||||
hand_request = None
|
||||
if VNDetectFaceLandmarksRequest is not None:
|
||||
face_request = VNDetectFaceLandmarksRequest.alloc().init()
|
||||
requests.append(face_request)
|
||||
if VNDetectHumanHandPoseRequest is not None:
|
||||
hand_request = VNDetectHumanHandPoseRequest.alloc().init()
|
||||
try:
|
||||
hand_request.setMaximumHandCount_(self.num_persons * 2)
|
||||
except Exception:
|
||||
pass
|
||||
requests.append(hand_request)
|
||||
|
||||
try:
|
||||
t_inf = time.monotonic()
|
||||
# pyobjc peut retourner soit bool, soit (bool, error) selon
|
||||
# la version. On normalise.
|
||||
ret = handler.performRequests_error_(requests, None)
|
||||
if isinstance(ret, tuple):
|
||||
ok3, err = ret
|
||||
else:
|
||||
ok3, err = bool(ret), None
|
||||
infer_ms = (time.monotonic() - t_inf) * 1000.0
|
||||
sum_ms += infer_ms
|
||||
n_frames += 1
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("Vision performRequests crash : %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
if not ok3:
|
||||
if n_frames < 5:
|
||||
LOG.warning("Vision request failed: %s", err)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
results = request.results() or []
|
||||
bodies: list[list[PoseKp]] = []
|
||||
n_body_raw = len(results)
|
||||
for obs in results[: self.num_persons * 2]:
|
||||
kps = self._parse_observation(obs)
|
||||
if kps is not None:
|
||||
bodies.append(kps)
|
||||
|
||||
# ---- Face landmarks ------------------------------------
|
||||
faces: list[list[PoseKp]] = []
|
||||
n_face_raw = 0
|
||||
if face_request is not None:
|
||||
try:
|
||||
face_results = face_request.results() or []
|
||||
except Exception:
|
||||
face_results = []
|
||||
n_face_raw = len(face_results)
|
||||
for obs in face_results[: self.num_persons * 2]:
|
||||
fkps = self._parse_face_observation(obs)
|
||||
if fkps is not None:
|
||||
faces.append(fkps)
|
||||
|
||||
# ---- Hand poses ----------------------------------------
|
||||
hands: list[list[PoseKp]] = []
|
||||
n_hand_raw = 0
|
||||
if hand_request is not None:
|
||||
try:
|
||||
hand_results = hand_request.results() or []
|
||||
except Exception:
|
||||
hand_results = []
|
||||
n_hand_raw = len(hand_results)
|
||||
for obs in hand_results[: self.num_persons * 2]:
|
||||
hkps = self._parse_hand_observation(obs)
|
||||
if hkps is not None:
|
||||
hands.append(hkps)
|
||||
|
||||
# Log debug : raw counts vs parsed counts (toutes les ~3s)
|
||||
if n_frames % 90 == 0:
|
||||
LOG.info("Vision raw: body=%d face=%d hand=%d "
|
||||
"parsed: body=%d face=%d hand=%d",
|
||||
n_body_raw, n_face_raw, n_hand_raw,
|
||||
len(bodies), len(faces), len(hands))
|
||||
|
||||
# ---- Analyse fine : re-inference sur crops haute resolution
|
||||
# (visage/mains agrandis 4x avant repasse Vision). Throttle 10 Hz.
|
||||
if self._fine_analyzer is None:
|
||||
self._fine_analyzer = FineAnalyzer(ns, throttle_hz=10.0)
|
||||
t_now = time.monotonic()
|
||||
if self._fine_analyzer.should_refine(t_now):
|
||||
# Wrappers de re-projection : les kps du crop sont en
|
||||
# coordonnees crop normalisees ; on les remap dans l'image
|
||||
# entiere via (x_origin, y_origin) + scale.
|
||||
def _wrap_face(obs, x_origin, y_origin, scale_x, scale_y):
|
||||
kps = self._parse_face_observation(obs)
|
||||
if kps is None:
|
||||
return None
|
||||
return [PoseKp(
|
||||
x=x_origin + k.x * scale_x,
|
||||
y=y_origin + k.y * scale_y,
|
||||
z=k.z, c=k.c,
|
||||
) for k in kps]
|
||||
|
||||
def _wrap_hand(obs, x_origin, y_origin, scale_x, scale_y):
|
||||
kps = self._parse_hand_observation(obs)
|
||||
if kps is None:
|
||||
return None
|
||||
return [PoseKp(
|
||||
x=x_origin + k.x * scale_x,
|
||||
y=y_origin + k.y * scale_y,
|
||||
z=k.z, c=k.c,
|
||||
) for k in kps]
|
||||
|
||||
faces = self._fine_analyzer.refine_face(
|
||||
frame_bgr, faces, _wrap_face)
|
||||
hands = self._fine_analyzer.refine_hands(
|
||||
frame_bgr, hands, _wrap_hand)
|
||||
|
||||
# Tracking + lissage (t_now deja defini au bloc fine analysis)
|
||||
ids = self._tracker.update(bodies)
|
||||
bodies_smooth = []
|
||||
for i, kps in enumerate(bodies):
|
||||
pid = ids[i] if i < len(ids) else -1
|
||||
if pid >= 0:
|
||||
smoothed = []
|
||||
for k, kp in enumerate(kps):
|
||||
if kp.c > 0.0:
|
||||
sx, sy, sz = self._smooth.smooth(
|
||||
pid, k, kp.x, kp.y, kp.z, t_now)
|
||||
smoothed.append(PoseKp(x=sx, y=sy, z=sz, c=kp.c))
|
||||
else:
|
||||
smoothed.append(kp)
|
||||
bodies_smooth.append(smoothed)
|
||||
else:
|
||||
bodies_smooth.append(kps)
|
||||
|
||||
# Pont sonore vers sclang (OSC /pose/* sur 57121)
|
||||
self._sound_bridge.send(bodies_smooth, ids, t_now)
|
||||
|
||||
with self.state.lock():
|
||||
self.state.persons_body = bodies_smooth
|
||||
self.state.persons_body_ids = ids
|
||||
self.state.persons_face = faces
|
||||
# Pas de tracking dedie pour les faces : IDs = index.
|
||||
self.state.persons_face_ids = list(range(len(faces)))
|
||||
self.state.persons_hands = hands
|
||||
self.state.persons_hands_ids = list(range(len(hands)))
|
||||
self.state.body_present = bool(bodies_smooth)
|
||||
self.state.face_present = bool(faces)
|
||||
self.state.hands_present = bool(hands)
|
||||
self.state.pose_count = len(bodies_smooth)
|
||||
self.state.pose_last_t = time.monotonic()
|
||||
self.state.last_webcam_jpeg = jpg_bytes
|
||||
# Compat single-person : copie le 1er body dans pose_kp legacy
|
||||
if bodies_smooth and bodies_smooth[0]:
|
||||
for k in range(min(17, len(bodies_smooth[0]))):
|
||||
self.state.body_kp[k] = bodies_smooth[0][k]
|
||||
|
||||
# Throttle target_fps
|
||||
dt = time.monotonic() - tA
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
|
||||
cap.release()
|
||||
avg = sum_ms / max(1, n_frames)
|
||||
LOG.info("apple-vision stop — %d frames, %.1f ms moy inference",
|
||||
n_frames, avg)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _parse_observation(self, obs) -> list[PoseKp] | None:
|
||||
"""Recupere TOUS les joints via recognizedPointsForGroupKey:
|
||||
retourne un dict {jointName: VNRecognizedPoint}. On mappe les keys
|
||||
retournees (peu importe leur format string exact) sur les indices
|
||||
MediaPipe via un suffixe (case insensitive).
|
||||
"""
|
||||
try:
|
||||
conf = float(obs.confidence())
|
||||
except Exception:
|
||||
conf = 1.0
|
||||
if conf < self.score_thresh:
|
||||
return None
|
||||
|
||||
kps = [PoseKp() for _ in range(33)]
|
||||
# Apple Vision body joints sont nommes selon la hierarchie ARKit
|
||||
# skeleton, pas les COCO/MP names. Les vraies cles :
|
||||
# head_joint, neck_1_joint, root,
|
||||
# left_shoulder_1_joint, right_shoulder_1_joint,
|
||||
# left_forearm_joint, right_forearm_joint,
|
||||
# left_hand_joint, right_hand_joint,
|
||||
# left_upLeg_joint, right_upLeg_joint,
|
||||
# left_leg_joint, right_leg_joint,
|
||||
# left_foot_joint, right_foot_joint
|
||||
APPLE_TO_MP = {
|
||||
"head_joint": 0, # nose (approximation)
|
||||
"left_shoulder_1_joint": 11,
|
||||
"right_shoulder_1_joint":12,
|
||||
"left_forearm_joint": 13, # elbow gauche
|
||||
"right_forearm_joint": 14,
|
||||
"left_hand_joint": 15, # poignet gauche
|
||||
"right_hand_joint": 16,
|
||||
"left_upLeg_joint": 23, # hanche
|
||||
"right_upLeg_joint": 24,
|
||||
"left_leg_joint": 25, # genou
|
||||
"right_leg_joint": 26,
|
||||
"left_foot_joint": 27, # cheville
|
||||
"right_foot_joint": 28,
|
||||
}
|
||||
for apple_name, mp_idx in APPLE_TO_MP.items():
|
||||
try:
|
||||
ret = obs.recognizedPointForJointName_error_(apple_name, None)
|
||||
pt = ret[0] if isinstance(ret, tuple) else ret
|
||||
if pt is None:
|
||||
continue
|
||||
pc = float(pt.confidence())
|
||||
if pc < 0.1:
|
||||
continue
|
||||
loc = pt.location()
|
||||
kps[mp_idx] = PoseKp(
|
||||
x=float(loc.x),
|
||||
y=1.0 - float(loc.y),
|
||||
z=0.0,
|
||||
c=pc,
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Si la 1ere passe ne trouve rien, debug log les vraies keys
|
||||
n_visible = sum(1 for k in kps if k.c > 0.1)
|
||||
if n_visible == 0 and not hasattr(self, "_logged_keys"):
|
||||
try:
|
||||
ret = obs.availableJointNames_error_(None)
|
||||
names = ret[0] if isinstance(ret, tuple) else ret
|
||||
LOG.info("availableJointNames: %s", list(names)[:10] if names else "EMPTY")
|
||||
self._logged_keys = True
|
||||
except Exception as e:
|
||||
LOG.info("availableJointNames KO: %s", e)
|
||||
self._logged_keys = True
|
||||
|
||||
# Verifie qu'on a au moins quelques kp visibles
|
||||
n_visible = sum(1 for k in kps if k.c > 0.1)
|
||||
if n_visible < 4:
|
||||
return None
|
||||
return kps
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _parse_face_observation(self, obs) -> list[PoseKp] | None:
|
||||
"""Parse les face landmarks Apple Vision via `pointAtIndex_(k)`
|
||||
(API stable qui retourne un CGPoint struct, pas un UnsafePointer
|
||||
problematique). Resout le blocage pyobjc PyObjCPointer.
|
||||
"""
|
||||
"""Extrait les landmarks face Apple Vision en liste plate de PoseKp.
|
||||
|
||||
Layout : offsets FACE_OFFSETS (cf mesh_topology.py). Le bbox de
|
||||
l'observation (.boundingBox normalize 0..1) sert a re-projeter
|
||||
les normalizedPoints (normalises DANS le bbox) vers le repere
|
||||
plein cadre normalise (0..1, top-left).
|
||||
"""
|
||||
try:
|
||||
landmarks = obs.landmarks()
|
||||
if landmarks is None:
|
||||
if not hasattr(self, "_face_no_lm_logged"):
|
||||
LOG.info("face: obs.landmarks() == None — face mesh OFF")
|
||||
self._face_no_lm_logged = True
|
||||
return None
|
||||
bb = obs.boundingBox()
|
||||
bx, by = float(bb.origin.x), float(bb.origin.y)
|
||||
bw, bh = float(bb.size.width), float(bb.size.height)
|
||||
except Exception as e:
|
||||
if not hasattr(self, "_face_err_logged"):
|
||||
LOG.info("face parse err: %s", e)
|
||||
self._face_err_logged = True
|
||||
return None
|
||||
|
||||
# Pre-rempli a (0,0,0,0). On comble les regions disponibles.
|
||||
kps: list[PoseKp] = [PoseKp() for _ in range(83)]
|
||||
|
||||
def fill(region_name: str, start: int, end: int) -> None:
|
||||
region = None
|
||||
# Essaye 3 acces : methode obj-c, attribut Python, KVC
|
||||
for fetcher in (
|
||||
lambda: getattr(landmarks, region_name)(),
|
||||
lambda: getattr(landmarks, region_name),
|
||||
lambda: landmarks.valueForKey_(region_name),
|
||||
):
|
||||
try:
|
||||
region = fetcher()
|
||||
if region is not None:
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if region is None:
|
||||
if not hasattr(self, "_logged_face_fail_" + region_name):
|
||||
LOG.info("face: region %s introuvable", region_name)
|
||||
setattr(self, "_logged_face_fail_" + region_name, True)
|
||||
return
|
||||
try:
|
||||
count = int(region.pointCount())
|
||||
except Exception:
|
||||
return
|
||||
if not hasattr(self, "_logged_face_ok_" + region_name):
|
||||
LOG.info("face: region %s count=%d", region_name, count)
|
||||
setattr(self, "_logged_face_ok_" + region_name, True)
|
||||
# pyobjc 11 ne sait pas que pointAtIndex_ prend 1 arg, et
|
||||
# pointsInImageOfSize_ retourne un PyObjCPointer C-array sans
|
||||
# API d'acces simple. Face parsing depuis Apple Vision est
|
||||
# actuellement bloque ; on garde MediaPipe (CPU XNNPACK) pour
|
||||
# face/hand fin tandis que Vision sert body 2D sur ANE.
|
||||
# Skip pour eviter le spam ObjCPointerWarning a 30 fps.
|
||||
return
|
||||
|
||||
# faceContour
|
||||
fill("faceContour", *FACE_OFFSETS["contour"])
|
||||
fill("leftEye", *FACE_OFFSETS["left_eye"])
|
||||
fill("rightEye", *FACE_OFFSETS["right_eye"])
|
||||
fill("leftEyebrow", *FACE_OFFSETS["left_brow"])
|
||||
fill("rightEyebrow",*FACE_OFFSETS["right_brow"])
|
||||
fill("outerLips", *FACE_OFFSETS["outer_lips"])
|
||||
fill("innerLips", *FACE_OFFSETS["inner_lips"])
|
||||
fill("nose", *FACE_OFFSETS["nose"])
|
||||
fill("medianLine", *FACE_OFFSETS["median"])
|
||||
|
||||
# Pupilles : single-point regions ; meme workaround pyobjc.
|
||||
for region_name, idx in (("leftPupil", 81), ("rightPupil", 82)):
|
||||
try:
|
||||
region = getattr(landmarks, region_name)()
|
||||
if region is None or region.pointCount() < 1:
|
||||
continue
|
||||
try:
|
||||
pts = region.pointsInImageOfSize_((1.0, 1.0))
|
||||
except Exception:
|
||||
pts = region.normalizedPoints()
|
||||
if not pts:
|
||||
continue
|
||||
pt = pts[0]
|
||||
try:
|
||||
px, py = float(pt.x), float(pt.y)
|
||||
except (AttributeError, TypeError):
|
||||
px, py = float(pt[0]), float(pt[1])
|
||||
fx = bx + px * bw
|
||||
fy_bl = by + py * bh
|
||||
kps[idx] = PoseKp(x=fx, y=1.0 - fy_bl, z=0.0, c=1.0)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
n_visible = sum(1 for k in kps if k.c > 0.0)
|
||||
if n_visible < 8:
|
||||
return None
|
||||
return kps
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _parse_hand_observation(self, obs) -> list[PoseKp] | None:
|
||||
"""Extrait les 21 joints d'une VNHumanHandPoseObservation."""
|
||||
kps = [PoseKp() for _ in range(21)]
|
||||
n_visible = 0
|
||||
for k, joint_name in enumerate(HAND_JOINTS):
|
||||
try:
|
||||
ret = obs.recognizedPointForJointName_error_(joint_name, None)
|
||||
pt = ret[0] if isinstance(ret, tuple) else ret
|
||||
if pt is None:
|
||||
continue
|
||||
pc = float(pt.confidence())
|
||||
if pc < 0.1:
|
||||
continue
|
||||
loc = pt.location()
|
||||
kps[k] = PoseKp(
|
||||
x=float(loc.x),
|
||||
y=1.0 - float(loc.y),
|
||||
z=0.0,
|
||||
c=pc,
|
||||
)
|
||||
n_visible += 1
|
||||
except Exception:
|
||||
continue
|
||||
if n_visible < 4:
|
||||
return None
|
||||
return kps
|
||||
@@ -0,0 +1,583 @@
|
||||
"""Pipeline pose 100% natif M5 : AVFoundation + Vision + CoreML (ANE).
|
||||
|
||||
Architecture (zero-copy, ANE-first) :
|
||||
|
||||
AVCaptureSession (live webcam)
|
||||
| delegate Python (Obj-C protocol via pyobjc)
|
||||
v
|
||||
CMSampleBuffer -> CVPixelBuffer BGRA
|
||||
| VNImageRequestHandler
|
||||
v
|
||||
VNCoreMLRequest (YOLO11n-pose .mlpackage, ANE)
|
||||
| parse keypoints
|
||||
v
|
||||
IoUTracker (Hungarian) + SkeletonFilter (One Euro)
|
||||
|
|
||||
v
|
||||
state.persons_body / persons_body_ids / last_webcam_jpeg
|
||||
|
||||
Pourquoi cette stack vs MediaPipe / DETRPose ?
|
||||
|
||||
- Decodage video AVFoundation : zero-copy, hardware-accelerated.
|
||||
- Vision wrappe le CVPixelBuffer en MLFeatureValue sans realloc.
|
||||
- YOLO11n-pose tient sur l'Apple Neural Engine M5 (<8 ms / frame
|
||||
en FP16) ; CPU/GPU sont laisses libres pour Metal/SuperCollider.
|
||||
- Pas de cv2 dans le hot path : encodage JPEG via CIImage +
|
||||
CGImageDestination, qui passe aussi par les codecs hardware.
|
||||
|
||||
API publique :
|
||||
CoreMLPoseWorker(state).start() # thread daemon
|
||||
CoreMLPoseWorker(state).stop()
|
||||
CoreMLPoseWorker.is_available() # @staticmethod, check .mlpackage
|
||||
|
||||
Frictions Python 3.14 :
|
||||
- `pyobjc-framework-Vision` n'est PAS publie sur PyPI au moment de
|
||||
cette ecriture. On contourne via `objc.loadBundle()` qui charge
|
||||
Vision.framework directement depuis /System/Library/Frameworks.
|
||||
- Idem pour CoreML : on utilise `objc.loadBundle()`. Les classes
|
||||
MLModel / VNCoreMLRequest deviennent disponibles dans le namespace
|
||||
global du module.
|
||||
- Le delegate AVCaptureVideoDataOutputSampleBufferDelegate est un
|
||||
PROTOCOLE Objective-C ; pyobjc accepte qu'on l'implemente en
|
||||
declarant le selector `captureOutput:didOutputSampleBuffer:fromConnection:`
|
||||
sur une NSObject — il sera resolu par duck typing au runtime.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import objc
|
||||
from Foundation import NSObject, NSURL
|
||||
|
||||
from .euro_filter import SkeletonFilter
|
||||
from .state import PoseKp, State
|
||||
from .tracker import IoUTracker
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
LOG = logging.getLogger("coreml_pose")
|
||||
|
||||
CACHE_DIR = Path.home() / ".cache" / "av-live-coreml"
|
||||
YOLO_MLPACKAGE = CACHE_DIR / "yolo11n-pose.mlpackage"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chargement Vision / CoreML / AVFoundation via objc.loadBundle
|
||||
# ---------------------------------------------------------------------------
|
||||
_FRAMEWORKS_LOADED = False
|
||||
_NS: dict = {}
|
||||
|
||||
|
||||
def _load_frameworks() -> dict:
|
||||
"""Charge Vision + CoreML + AVFoundation dans un namespace partage.
|
||||
|
||||
On le fait une seule fois ; les classes Obj-C sont enregistrees
|
||||
globalement par le runtime, mais on garde un dict pour acceder
|
||||
aux symboles sans polluer le module."""
|
||||
global _FRAMEWORKS_LOADED
|
||||
if _FRAMEWORKS_LOADED:
|
||||
return _NS
|
||||
objc.loadBundle("Vision", _NS,
|
||||
"/System/Library/Frameworks/Vision.framework")
|
||||
objc.loadBundle("CoreML", _NS,
|
||||
"/System/Library/Frameworks/CoreML.framework")
|
||||
objc.loadBundle("AVFoundation", _NS,
|
||||
"/System/Library/Frameworks/AVFoundation.framework")
|
||||
objc.loadBundle("CoreMedia", _NS,
|
||||
"/System/Library/Frameworks/CoreMedia.framework")
|
||||
_FRAMEWORKS_LOADED = True
|
||||
return _NS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers : encodage JPEG via Quartz (CIImage -> CGImageDestination)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _pixelbuffer_to_jpeg(pixel_buffer, quality: float = 0.7) -> bytes | None:
|
||||
"""Encode un CVPixelBuffer en JPEG via le pipeline hardware Quartz."""
|
||||
try:
|
||||
from Quartz import CIImage, CIContext
|
||||
from Foundation import NSMutableData
|
||||
from CoreFoundation import CFDictionaryCreate, kCFTypeDictionaryKeyCallBacks, kCFTypeDictionaryValueCallBacks # noqa: F401
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
try:
|
||||
ci = CIImage.imageWithCVPixelBuffer_(pixel_buffer)
|
||||
if ci is None:
|
||||
return None
|
||||
ctx = CIContext.context()
|
||||
# On utilise jpegRepresentationOfImage:colorSpace:options: (macOS 10.12+)
|
||||
from Quartz import CGColorSpaceCreateDeviceRGB
|
||||
cs = CGColorSpaceCreateDeviceRGB()
|
||||
data = ctx.JPEGRepresentationOfImage_colorSpace_options_(
|
||||
ci, cs, {"kCGImageDestinationLossyCompressionQuality": quality})
|
||||
if data is None:
|
||||
return None
|
||||
return bytes(data)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Delegate AVCaptureVideoDataOutput
|
||||
# ---------------------------------------------------------------------------
|
||||
class _CaptureDelegate(NSObject):
|
||||
"""Delegate Obj-C qui recoit chaque frame webcam.
|
||||
|
||||
Implementation du protocole AVCaptureVideoDataOutputSampleBufferDelegate :
|
||||
pyobjc resout le selector par signature, pas besoin de declarer
|
||||
formellement le protocole."""
|
||||
|
||||
def initWithWorker_(self, worker): # noqa: N802
|
||||
self = objc.super(_CaptureDelegate, self).init()
|
||||
if self is None:
|
||||
return None
|
||||
self._worker = worker
|
||||
return self
|
||||
|
||||
# Signature: -(void)captureOutput:(AVCaptureOutput*)output
|
||||
# didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
# fromConnection:(AVCaptureConnection*)connection
|
||||
def captureOutput_didOutputSampleBuffer_fromConnection_( # noqa: N802
|
||||
self, output, sample_buffer, connection):
|
||||
self._worker._on_frame(sample_buffer)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Queue GCD serielle pour le delegate AVFoundation
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_serial_queue(label: str):
|
||||
"""Cree une dispatch_queue_create(label, DISPATCH_QUEUE_SERIAL) via objc.
|
||||
|
||||
pyobjc expose les fonctions GCD via le module `objc`. La signature
|
||||
Python est : dispatch_queue_create(label_bytes, None) -> dispatch_queue_t.
|
||||
"""
|
||||
try:
|
||||
from libdispatch import dispatch_queue_create # type: ignore
|
||||
return dispatch_queue_create(label.encode("utf-8"), None)
|
||||
except ImportError:
|
||||
pass
|
||||
# Fallback : utiliser le runtime Obj-C via ctypes pour appeler
|
||||
# dispatch_queue_create. Toutefois pyobjc 11+ expose ces fonctions
|
||||
# via Foundation / objc directement.
|
||||
try:
|
||||
import Foundation # noqa: F401
|
||||
# pyobjc enregistre dispatch_queue_create dans `objc` namespace
|
||||
from objc import _objc # type: ignore # noqa: F401
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
# Dernier recours : ctypes wrapper sur libdispatch (toujours present sur macOS)
|
||||
import ctypes
|
||||
libdispatch = ctypes.CDLL("/usr/lib/system/libdispatch.dylib")
|
||||
libdispatch.dispatch_queue_create.restype = ctypes.c_void_p
|
||||
libdispatch.dispatch_queue_create.argtypes = [ctypes.c_char_p, ctypes.c_void_p]
|
||||
q = libdispatch.dispatch_queue_create(label.encode("utf-8"), None)
|
||||
return q
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Worker principal
|
||||
# ===========================================================================
|
||||
class CoreMLPoseWorker:
|
||||
"""Worker pose natif M5 — AVFoundation + Vision + CoreML (YOLO11n-pose).
|
||||
|
||||
start() peut etre appele depuis n'importe quel thread tant que la run
|
||||
loop NSApplication tourne (le delegate AVFoundation est dispatche sur
|
||||
une queue serielle GCD dediee, pas sur le main thread)."""
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
if not YOLO_MLPACKAGE.exists():
|
||||
return False
|
||||
try:
|
||||
_load_frameworks()
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state: State,
|
||||
target_fps: float = 30.0,
|
||||
num_persons: int = 4,
|
||||
score_thresh: float = 0.45,
|
||||
) -> None:
|
||||
self.state = state
|
||||
self.target_fps = float(target_fps)
|
||||
self.num_persons = int(num_persons)
|
||||
self.score_thresh = float(score_thresh)
|
||||
self._frame_period = 1.0 / max(1.0, self.target_fps)
|
||||
self._last_emit = 0.0
|
||||
self._tracker = IoUTracker(iou_threshold=0.20, max_miss=10)
|
||||
self._smooth = SkeletonFilter(min_cutoff=1.2, beta=0.06)
|
||||
# Refs Obj-C
|
||||
self._session = None
|
||||
self._input = None
|
||||
self._output = None
|
||||
self._delegate = None
|
||||
self._queue = None
|
||||
self._vn_model = None
|
||||
self._frame_size: tuple[int, int] = (640, 480)
|
||||
# Metriques
|
||||
self._n_frames = 0
|
||||
self._n_emitted = 0
|
||||
self._sum_infer_ms = 0.0
|
||||
self._started = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def start(self) -> None:
|
||||
if self._started:
|
||||
return
|
||||
try:
|
||||
self._setup_pipeline()
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("setup pipeline echoue : %s", e)
|
||||
return
|
||||
self._started = True
|
||||
# startRunning peut bloquer brievement — on le fait dans un thread
|
||||
# daemon pour ne pas geler le main thread AppKit.
|
||||
threading.Thread(
|
||||
target=self._start_session, name="coreml-session-start",
|
||||
daemon=True).start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._started = False
|
||||
try:
|
||||
if self._session is not None:
|
||||
self._session.stopRunning()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
LOG.info("coreml-pose stop — %d frames, %d emises, %.1f ms moy inference",
|
||||
self._n_frames, self._n_emitted,
|
||||
(self._sum_infer_ms / max(1, self._n_emitted)))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _setup_pipeline(self) -> None:
|
||||
ns = _load_frameworks()
|
||||
AVCaptureSession = ns["AVCaptureSession"]
|
||||
AVCaptureDevice = ns["AVCaptureDevice"]
|
||||
AVCaptureDeviceInput = ns["AVCaptureDeviceInput"]
|
||||
AVCaptureVideoDataOutput = ns["AVCaptureVideoDataOutput"]
|
||||
AVMediaTypeVideo = ns["AVMediaTypeVideo"]
|
||||
MLModel = ns["MLModel"]
|
||||
MLModelConfiguration = ns["MLModelConfiguration"]
|
||||
VNCoreMLModel = ns["VNCoreMLModel"]
|
||||
|
||||
# 1) Charger le modele
|
||||
cfg = MLModelConfiguration.alloc().init()
|
||||
try:
|
||||
cfg.setComputeUnits_(0) # MLComputeUnitsAll
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
url = NSURL.fileURLWithPath_(str(YOLO_MLPACKAGE))
|
||||
ml_model, err = MLModel.modelWithContentsOfURL_configuration_error_(
|
||||
url, cfg, None)
|
||||
if ml_model is None:
|
||||
raise RuntimeError(f"MLModel load: {err}")
|
||||
vn_model, err = VNCoreMLModel.modelForMLModel_error_(ml_model, None)
|
||||
if vn_model is None:
|
||||
raise RuntimeError(f"VNCoreMLModel wrap: {err}")
|
||||
self._vn_model = vn_model
|
||||
LOG.info("CoreML pose worker — modele %s charge (computeUnits=all=ANE+GPU+CPU)",
|
||||
YOLO_MLPACKAGE.name)
|
||||
|
||||
# 2) Session capture
|
||||
session = AVCaptureSession.alloc().init()
|
||||
try:
|
||||
session.setSessionPreset_("AVCaptureSessionPreset640x480")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
device = AVCaptureDevice.defaultDeviceWithMediaType_(AVMediaTypeVideo)
|
||||
if device is None:
|
||||
raise RuntimeError("aucune camera (AVCaptureDevice defaultDevice)")
|
||||
input_, err = AVCaptureDeviceInput.deviceInputWithDevice_error_(
|
||||
device, None)
|
||||
if input_ is None:
|
||||
raise RuntimeError(f"AVCaptureDeviceInput: {err}")
|
||||
if not session.canAddInput_(input_):
|
||||
raise RuntimeError("session.canAddInput == False")
|
||||
session.addInput_(input_)
|
||||
self._input = input_
|
||||
|
||||
# 3) Output BGRA
|
||||
output = AVCaptureVideoDataOutput.alloc().init()
|
||||
BGRA = 0x42475241 # kCVPixelFormatType_32BGRA
|
||||
try:
|
||||
# cle officielle = kCVPixelBufferPixelFormatTypeKey
|
||||
from Quartz import kCVPixelBufferPixelFormatTypeKey
|
||||
output.setVideoSettings_({kCVPixelBufferPixelFormatTypeKey: BGRA})
|
||||
except Exception: # noqa: BLE001
|
||||
# Fallback : nom de cle litteral
|
||||
output.setVideoSettings_({"PixelFormatType": BGRA})
|
||||
output.setAlwaysDiscardsLateVideoFrames_(True)
|
||||
if not session.canAddOutput_(output):
|
||||
raise RuntimeError("session.canAddOutput == False")
|
||||
session.addOutput_(output)
|
||||
self._output = output
|
||||
|
||||
# 4) Delegate + queue serielle GCD
|
||||
delegate = _CaptureDelegate.alloc().initWithWorker_(self)
|
||||
queue = _make_serial_queue("av-live.coreml.pose")
|
||||
output.setSampleBufferDelegate_queue_(delegate, queue)
|
||||
self._delegate = delegate
|
||||
self._queue = queue
|
||||
self._session = session
|
||||
|
||||
def _start_session(self) -> None:
|
||||
if self._session is None:
|
||||
return
|
||||
LOG.info("AVCaptureSession.startRunning ...")
|
||||
self._session.startRunning()
|
||||
LOG.info("session running — fps_target=%.0f num_persons=%d thresh=%.2f",
|
||||
self.target_fps, self.num_persons, self.score_thresh)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Callback delegate — appele par GCD sur la queue serielle "coreml.pose"
|
||||
# ------------------------------------------------------------------
|
||||
def _on_frame(self, sample_buffer) -> None:
|
||||
self._n_frames += 1
|
||||
now = time.monotonic()
|
||||
if now - self._last_emit < self._frame_period:
|
||||
return
|
||||
self._last_emit = now
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
self._process_frame(sample_buffer)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("process_frame: %s", e)
|
||||
return
|
||||
self._sum_infer_ms += (time.monotonic() - t0) * 1000.0
|
||||
self._n_emitted += 1
|
||||
|
||||
def _process_frame(self, sample_buffer) -> None:
|
||||
ns = _NS
|
||||
VNImageRequestHandler = ns["VNImageRequestHandler"]
|
||||
VNCoreMLRequest = ns["VNCoreMLRequest"]
|
||||
# CMSampleBufferGetImageBuffer renvoie un CVPixelBuffer
|
||||
from Quartz import (
|
||||
CMSampleBufferGetImageBuffer,
|
||||
CVPixelBufferGetWidth,
|
||||
CVPixelBufferGetHeight,
|
||||
)
|
||||
pixel_buffer = CMSampleBufferGetImageBuffer(sample_buffer)
|
||||
if pixel_buffer is None:
|
||||
return
|
||||
w = CVPixelBufferGetWidth(pixel_buffer)
|
||||
h = CVPixelBufferGetHeight(pixel_buffer)
|
||||
self._frame_size = (w, h)
|
||||
|
||||
# Construction du request synchrone : on capture les resultats
|
||||
# dans une closure puis on perform().
|
||||
results_box: list = []
|
||||
|
||||
def _handler(request, error):
|
||||
r = request.results()
|
||||
if r is not None:
|
||||
# On copie les pointeurs Obj-C immediatement
|
||||
for obs in r:
|
||||
results_box.append(obs)
|
||||
|
||||
request = VNCoreMLRequest.alloc().initWithModel_completionHandler_(
|
||||
self._vn_model, _handler)
|
||||
try:
|
||||
# Image crop & scale : on laisse Vision faire le scaleFit ;
|
||||
# YOLO11n-pose attend 640x640.
|
||||
request.setImageCropAndScaleOption_(1) # VNImageCropAndScaleOptionScaleFit
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
handler = VNImageRequestHandler.alloc().initWithCVPixelBuffer_options_(
|
||||
pixel_buffer, {})
|
||||
ok, err = handler.performRequests_error_([request], None)
|
||||
if not ok:
|
||||
LOG.debug("perform request error: %s", err)
|
||||
return
|
||||
|
||||
# Parsing : Vision retourne soit VNRecognizedPointsObservation
|
||||
# (modele "pose" classique), soit VNCoreMLFeatureValueObservation
|
||||
# (modele YOLO converti par ultralytics — tenseur brut).
|
||||
bodies = self._parse_results(results_box, w, h)
|
||||
|
||||
# Tracking + lissage
|
||||
ids = self._tracker.update(bodies)
|
||||
t_now = time.monotonic()
|
||||
bodies_smooth = []
|
||||
for i, kps in enumerate(bodies):
|
||||
pid = ids[i] if i < len(ids) else -1
|
||||
if pid < 0:
|
||||
bodies_smooth.append(kps)
|
||||
continue
|
||||
out = []
|
||||
for k, kp in enumerate(kps):
|
||||
sx, sy, sz = self._smooth.smooth(pid, k, kp.x, kp.y, kp.z, t_now)
|
||||
out.append(PoseKp(x=sx, y=sy, z=sz, c=kp.c))
|
||||
bodies_smooth.append(out)
|
||||
|
||||
# JPEG webcam (best effort)
|
||||
jpg = _pixelbuffer_to_jpeg(pixel_buffer, quality=0.65)
|
||||
|
||||
with self.state.lock():
|
||||
self.state.persons_body = bodies_smooth
|
||||
self.state.persons_body_ids = ids
|
||||
# On vide face/hands : YOLO11n-pose ne les fournit pas.
|
||||
self.state.persons_face = []
|
||||
self.state.persons_hands = []
|
||||
self.state.face_present = False
|
||||
self.state.hands_present = False
|
||||
if bodies_smooth:
|
||||
self.state.body_present = True
|
||||
# Compat single-person : 17 kp dans body_kp[0..17],
|
||||
# le reste reste a zero.
|
||||
first = bodies_smooth[0]
|
||||
for k in range(33):
|
||||
self.state.body_kp[k] = (
|
||||
first[k] if k < 17 and k < len(first) else PoseKp())
|
||||
for k in range(17):
|
||||
self.state.pose_kp[k] = (
|
||||
first[k] if k < len(first) else PoseKp())
|
||||
else:
|
||||
self.state.body_present = False
|
||||
self.state.pose_count = len(bodies_smooth)
|
||||
self.state.pose_last_t = t_now
|
||||
if jpg:
|
||||
self.state.last_webcam_jpeg = jpg
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Parsing des observations Vision -> list[list[PoseKp]]
|
||||
# ------------------------------------------------------------------
|
||||
def _parse_results(self, results, w: int, h: int) -> list[list[PoseKp]]:
|
||||
"""Convertit les VN*Observation en keypoints normalises 0..1.
|
||||
|
||||
Deux formats possibles selon la conversion CoreML :
|
||||
(a) VNHumanBodyPoseObservation : API Vision native, 17 kp pre-parses
|
||||
avec joint names (CGPoint normalises + confidence).
|
||||
(b) VNCoreMLFeatureValueObservation : tenseur brut YOLO output
|
||||
(post-NMS si nms=True dans l'export ultralytics) — format
|
||||
(N, 56) = [cx, cy, bw, bh, conf, kp_x1, kp_y1, kp_v1, ..., x17, y17, v17]
|
||||
en pixels du modele (640x640).
|
||||
"""
|
||||
bodies: list[list[PoseKp]] = []
|
||||
if not results:
|
||||
return bodies
|
||||
|
||||
# Cas (a) : Vision pre-parse
|
||||
first = results[0]
|
||||
cls_name = first.className() if hasattr(first, "className") else ""
|
||||
if "BodyPose" in cls_name or "HumanBodyPose" in cls_name:
|
||||
for obs in results[: self.num_persons * 2]:
|
||||
conf = float(obs.confidence()) if hasattr(obs, "confidence") else 1.0
|
||||
if conf < self.score_thresh:
|
||||
continue
|
||||
kps = self._extract_body_pose_obs(obs)
|
||||
if kps:
|
||||
bodies.append(kps)
|
||||
return bodies[: self.num_persons]
|
||||
|
||||
# Cas (b) : tenseur YOLO brut
|
||||
for obs in results:
|
||||
try:
|
||||
fv = obs.featureValue()
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
arr = fv.multiArrayValue() if fv is not None else None
|
||||
if arr is None:
|
||||
continue
|
||||
parsed = self._parse_yolo_tensor(arr)
|
||||
bodies.extend(parsed)
|
||||
# tri par conf decroissant + cap
|
||||
bodies.sort(key=lambda kps: -max((k.c for k in kps), default=0.0))
|
||||
return bodies[: self.num_persons]
|
||||
|
||||
def _extract_body_pose_obs(self, obs) -> list[PoseKp]:
|
||||
"""Extrait 17 kp d'un VNHumanBodyPoseObservation (ordre COCO).
|
||||
|
||||
L'API Vision retourne les points via recognizedPointsForGroupKey:error:
|
||||
OU recognizedPointForJointName:error:. On utilise l'ordre COCO :
|
||||
nose, leye, reye, lear, rear, lsh, rsh, lel, rel, lwr, rwr,
|
||||
lhi, rhi, lkn, rkn, lan, ran.
|
||||
"""
|
||||
joint_names = [
|
||||
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
||||
"left_shoulder", "right_shoulder",
|
||||
"left_elbow", "right_elbow",
|
||||
"left_wrist", "right_wrist",
|
||||
"left_hip", "right_hip",
|
||||
"left_knee", "right_knee",
|
||||
"left_ankle", "right_ankle",
|
||||
]
|
||||
out: list[PoseKp] = []
|
||||
for name in joint_names:
|
||||
try:
|
||||
pt, _err = obs.recognizedPointForJointName_error_(name, None)
|
||||
except Exception: # noqa: BLE001
|
||||
pt = None
|
||||
if pt is None:
|
||||
out.append(PoseKp())
|
||||
continue
|
||||
try:
|
||||
loc = pt.location()
|
||||
cf = float(pt.confidence())
|
||||
# Vision : y origin en bas, on flip pour aligner avec image y-down
|
||||
out.append(PoseKp(x=float(loc.x), y=1.0 - float(loc.y),
|
||||
z=0.0, c=cf))
|
||||
except Exception: # noqa: BLE001
|
||||
out.append(PoseKp())
|
||||
return out
|
||||
|
||||
def _parse_yolo_tensor(self, ml_array) -> list[list[PoseKp]]:
|
||||
"""Parse un MLMultiArray YOLO11n-pose post-NMS -> bodies.
|
||||
|
||||
Shape attendu apres export ultralytics nms=True : (1, N, 56)
|
||||
avec 56 = box(4) + conf(1) + 17 * (x,y,v) = 4+1+51.
|
||||
Sans NMS : (1, 56, M) transpose. On gere les deux."""
|
||||
try:
|
||||
shape = list(ml_array.shape)
|
||||
dims = [int(s) for s in shape]
|
||||
except Exception: # noqa: BLE001
|
||||
return []
|
||||
# Acceder aux donnees via dataPointer() est risque ; on passe par
|
||||
# itemAtIndexedSubscript: ou la conversion en numpy via .float32 view.
|
||||
try:
|
||||
import numpy as np
|
||||
# MLMultiArray expose 'dataPointer' (raw void*) — on prefere
|
||||
# construire un buffer Python via getBytes:length: indirect.
|
||||
count = 1
|
||||
for d in dims:
|
||||
count *= d
|
||||
buf = ml_array.dataPointer()
|
||||
# pyobjc renvoie un objc.varlist ou un voidp — on tente numpy.frombuffer
|
||||
arr = np.frombuffer(buf, dtype=np.float32, count=count).reshape(dims)
|
||||
except Exception: # noqa: BLE001
|
||||
return []
|
||||
|
||||
# Squeeze batch
|
||||
while arr.ndim > 2 and arr.shape[0] == 1:
|
||||
arr = arr[0]
|
||||
if arr.ndim != 2:
|
||||
return []
|
||||
# Si shape (56, M) au lieu de (M, 56) -> transpose
|
||||
if arr.shape[0] == 56 and arr.shape[1] != 56:
|
||||
arr = arr.T
|
||||
elif arr.shape[1] != 56 and arr.shape[0] != 56:
|
||||
return []
|
||||
|
||||
bodies: list[list[PoseKp]] = []
|
||||
for row in arr:
|
||||
conf = float(row[4])
|
||||
if conf < self.score_thresh:
|
||||
continue
|
||||
kps: list[PoseKp] = []
|
||||
for k in range(17):
|
||||
kx = float(row[5 + k * 3 + 0])
|
||||
ky = float(row[5 + k * 3 + 1])
|
||||
kv = float(row[5 + k * 3 + 2])
|
||||
# Coords en pixels du modele 640x640 -> normaliser
|
||||
kps.append(PoseKp(x=kx / 640.0, y=ky / 640.0, z=0.0, c=kv))
|
||||
bodies.append(kps)
|
||||
return bodies
|
||||
@@ -0,0 +1,351 @@
|
||||
"""DETRPose multi-personne — worker alternatif a MediaPipe Multi.
|
||||
|
||||
DETRPose (2025, S. Janampa) : premier transformer end-to-end temps reel
|
||||
pour la detection de pose multi-personne. Sortie COCO 17 keypoints,
|
||||
multi-personne nativement (queries DETR), entraine 5 a 10x plus vite
|
||||
que ses concurrents grace a un denoising base sur OKS.
|
||||
|
||||
- Paper : https://arxiv.org/abs/2506.13027
|
||||
- Repo : https://github.com/SebastianJanampa/DETRPose
|
||||
- Weights : https://github.com/SebastianJanampa/DETRPose/releases/tag/model_weights
|
||||
- Demo HF : https://huggingface.co/spaces/SebasJanampa/DETRPose
|
||||
|
||||
============================================================================
|
||||
INSTALLATION (manuelle — DETRPose n'est PAS pip-installable)
|
||||
============================================================================
|
||||
|
||||
Le repo n'a pas de setup.py / pyproject.toml — il faut le cloner et
|
||||
l'ajouter au PYTHONPATH. Procedure :
|
||||
|
||||
# 1. Cloner dans le cache utilisateur
|
||||
mkdir -p ~/.cache/av-live-detrpose
|
||||
cd ~/.cache/av-live-detrpose
|
||||
git clone https://github.com/SebastianJanampa/DETRPose.git
|
||||
|
||||
# 2. Dependances Python (sans numpy<1.24 — on garde le numpy du venv)
|
||||
cd ~/Documents/Projets/AV-Live/data_only_viz
|
||||
uv pip install torch torchvision transformers omegaconf cloudpickle \
|
||||
pycocotools xtcocotools scipy calflops iopath
|
||||
|
||||
# 3. Telecharger un checkpoint (N = nano, ~16 MB, le plus rapide)
|
||||
cd ~/.cache/av-live-detrpose
|
||||
curl -L -o detrpose_hgnetv2_n.pth \
|
||||
https://github.com/SebastianJanampa/DETRPose/releases/download/model_weights/detrpose_hgnetv2_n.pth
|
||||
|
||||
Sinon, le worker logge une erreur claire et main.py retombe sur MediaPipe.
|
||||
|
||||
============================================================================
|
||||
DEVICE
|
||||
============================================================================
|
||||
|
||||
DETRPose s'appuie sur PyTorch standard — compatible MPS (Apple Silicon),
|
||||
CUDA, CPU. Pas de couche custom CUDA-only. On essaie MPS en premier, on
|
||||
retombe sur CPU si erreur. Inference ~30-50 ms sur M5 avec le modele N.
|
||||
|
||||
============================================================================
|
||||
FORMAT DE SORTIE
|
||||
============================================================================
|
||||
|
||||
COCO 17 keypoints, ordre standard :
|
||||
0: nose, 1-2: eyes, 3-4: ears, 5-6: shoulders, 7-8: elbows,
|
||||
9-10: wrists, 11-12: hips, 13-14: knees, 15-16: ankles.
|
||||
|
||||
Le state AV-Live attend `persons_body` = list[list[PoseKp]] ou chaque
|
||||
PoseKp a x, y normalises 0..1. DETRPose ne fournit pas la profondeur z
|
||||
(modele 2D pur) ni la visibilite par keypoint — on met z=0 et c=score
|
||||
global de la personne.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .state import PoseKp, State
|
||||
|
||||
LOG = logging.getLogger("detrpose")
|
||||
|
||||
CACHE_DIR = Path.home() / ".cache" / "av-live-detrpose"
|
||||
REPO_DIR = CACHE_DIR / "DETRPose"
|
||||
# Modele N (nano) par defaut : 16 MB, le plus rapide.
|
||||
DEFAULT_MODEL_SIZE = "n"
|
||||
_VALID_SIZES = {"n", "s", "l"}
|
||||
DEFAULT_CKPT = CACHE_DIR / f"detrpose_hgnetv2_{DEFAULT_MODEL_SIZE}.pth"
|
||||
DEFAULT_CONFIG_REL = f"configs/detrpose/detrpose_hgnetv2_{DEFAULT_MODEL_SIZE}.py"
|
||||
|
||||
|
||||
def _check_install() -> tuple[bool, str]:
|
||||
"""Verifie que le repo et le checkpoint sont presents. Renvoie (ok, msg)."""
|
||||
if not REPO_DIR.exists():
|
||||
return False, (
|
||||
f"DETRPose repo absent ({REPO_DIR}). Voir docstring du module "
|
||||
"pour la procedure d'install."
|
||||
)
|
||||
if not (REPO_DIR / DEFAULT_CONFIG_REL).exists():
|
||||
return False, f"config manquante : {REPO_DIR / DEFAULT_CONFIG_REL}"
|
||||
if not DEFAULT_CKPT.exists():
|
||||
return False, f"checkpoint manquant : {DEFAULT_CKPT}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""Test rapide : repo + checkpoint presents ET PyTorch importable."""
|
||||
ok, _ = _check_install()
|
||||
if not ok:
|
||||
return False
|
||||
try:
|
||||
import torch # noqa: F401
|
||||
except ImportError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class DETRPoseWorker:
|
||||
"""Worker multi-personne DETRPose (body only, 17 keypoints COCO).
|
||||
|
||||
Suit le meme contrat que MultiWorker : ecrit dans state.persons_body
|
||||
et state.last_webcam_jpeg, thread daemon, stop() propre.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state: State,
|
||||
camera_index: int = 0,
|
||||
target_fps: float = 18.0,
|
||||
num_persons: int = 4,
|
||||
score_thresh: float = 0.5,
|
||||
model_size: str = DEFAULT_MODEL_SIZE,
|
||||
device: str = "auto",
|
||||
) -> None:
|
||||
self.state = state
|
||||
self.camera_index = camera_index
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self.num_persons = num_persons
|
||||
self.score_thresh = score_thresh
|
||||
self._configure_model_size(model_size)
|
||||
self.device_pref = device
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="detrpose", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def _configure_model_size(self, size: str) -> None:
|
||||
"""Validate and set model_size; raise ValueError for unknown sizes."""
|
||||
if size not in _VALID_SIZES:
|
||||
raise ValueError(
|
||||
f"DETRPose model_size must be one of {sorted(_VALID_SIZES)}, got {size!r}"
|
||||
)
|
||||
self.model_size = size
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Chargement modele : on importe le repo DETRPose en ajoutant son
|
||||
# dossier au sys.path, puis on suit le pattern de tools/inference/torch_inf.py.
|
||||
# ------------------------------------------------------------------
|
||||
def _load_model(self):
|
||||
ok, msg = _check_install()
|
||||
if not ok:
|
||||
raise RuntimeError(msg)
|
||||
if str(REPO_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_DIR))
|
||||
# DETRPose utilise des chemins relatifs (configs/, src/) — il
|
||||
# faut chdir dans le repo pour que les imports config marchent.
|
||||
# On preserve le cwd appelant pour ne pas perturber le reste de l'app.
|
||||
prev_cwd = os.getcwd()
|
||||
try:
|
||||
os.chdir(REPO_DIR)
|
||||
import torch
|
||||
from omegaconf import OmegaConf
|
||||
# L'API d'instantiation hydra-like utilisee par DETRPose.
|
||||
try:
|
||||
from src.misc.lazy_config import instantiate # type: ignore
|
||||
except ImportError:
|
||||
from src.core import instantiate # type: ignore
|
||||
|
||||
cfg_path = f"configs/detrpose/detrpose_hgnetv2_{self.model_size}.py"
|
||||
cfg = OmegaConf.load(cfg_path) if cfg_path.endswith(
|
||||
".yaml") else _load_py_config(cfg_path)
|
||||
|
||||
ckpt_path = CACHE_DIR / f"detrpose_hgnetv2_{self.model_size}.pth"
|
||||
ckpt = torch.load(ckpt_path, map_location="cpu")
|
||||
state_dict = ckpt.get("model") or ckpt.get("ema", {}).get(
|
||||
"module") or ckpt
|
||||
|
||||
model = instantiate(cfg.model)
|
||||
model.load_state_dict(state_dict, strict=False)
|
||||
try:
|
||||
model = model.deploy()
|
||||
except AttributeError:
|
||||
pass
|
||||
model.eval()
|
||||
|
||||
postprocessor = instantiate(cfg.postprocessor)
|
||||
try:
|
||||
postprocessor = postprocessor.deploy()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
device = self._pick_device(torch)
|
||||
model = model.to(device)
|
||||
LOG.info("DETRPose %s charge sur %s", self.model_size, device)
|
||||
return model, postprocessor, device, torch
|
||||
finally:
|
||||
os.chdir(prev_cwd)
|
||||
|
||||
def _pick_device(self, torch):
|
||||
pref = self.device_pref
|
||||
if pref == "auto":
|
||||
if torch.backends.mps.is_available():
|
||||
return torch.device("mps")
|
||||
if torch.cuda.is_available():
|
||||
return torch.device("cuda:0")
|
||||
return torch.device("cpu")
|
||||
if pref == "mps" and not torch.backends.mps.is_available():
|
||||
LOG.warning("MPS demande mais indisponible — fallback CPU")
|
||||
return torch.device("cpu")
|
||||
return torch.device(pref)
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
except ModuleNotFoundError as e:
|
||||
LOG.error("deps manquantes : %s", e)
|
||||
return
|
||||
|
||||
try:
|
||||
model, postprocessor, device, _ = self._load_model()
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("chargement DETRPose echoue : %s", e)
|
||||
return
|
||||
|
||||
cap = cv2.VideoCapture(self.camera_index)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index %d indisponible", self.camera_index)
|
||||
return
|
||||
LOG.info("camera ouverte (index %d)", self.camera_index)
|
||||
|
||||
# Tenseur d'input fixe 640x640 (cf torch_inf.py)
|
||||
INPUT_SIZE = 640
|
||||
mean = torch.tensor([0.485, 0.456, 0.406], device=device).view(1, 3, 1, 1)
|
||||
std = torch.tensor([0.229, 0.224, 0.225], device=device).view(1, 3, 1, 1)
|
||||
|
||||
while not self._stop.is_set():
|
||||
tA = time.monotonic()
|
||||
ok, frame_bgr = cap.read()
|
||||
if not ok or frame_bgr is None:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
h, w = frame_bgr.shape[:2]
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Preprocess : resize 640x640, [0,1], NCHW, normalisation ImageNet
|
||||
img = cv2.resize(frame_rgb, (INPUT_SIZE, INPUT_SIZE))
|
||||
tens = torch.from_numpy(img).to(device).float().permute(2, 0, 1)
|
||||
tens = tens.unsqueeze(0) / 255.0
|
||||
tens = (tens - mean) / std
|
||||
orig_sizes = torch.tensor([[w, h]], device=device)
|
||||
|
||||
try:
|
||||
with torch.no_grad():
|
||||
outputs = model(tens)
|
||||
scores, labels, keypoints = postprocessor(outputs, orig_sizes)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("inference: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# scores: (1, N), keypoints: (1, N, 17, 2) en pixels image originale
|
||||
scores0 = scores[0].detach().cpu().numpy()
|
||||
kps0 = keypoints[0].detach().cpu().numpy()
|
||||
idx = scores0 > self.score_thresh
|
||||
sel_scores = scores0[idx]
|
||||
sel_kps = kps0[idx]
|
||||
# Trier par score decroissant et limiter a num_persons
|
||||
order = (-sel_scores).argsort()[: self.num_persons]
|
||||
bodies: list[list[PoseKp]] = []
|
||||
for i in order:
|
||||
conf = float(sel_scores[i])
|
||||
pts = sel_kps[i] # (17, 2) en pixels
|
||||
kp_list = []
|
||||
for kx, ky in pts:
|
||||
kp_list.append(PoseKp(
|
||||
x=float(kx) / max(1, w),
|
||||
y=float(ky) / max(1, h),
|
||||
z=0.0,
|
||||
c=conf,
|
||||
))
|
||||
bodies.append(kp_list)
|
||||
|
||||
# Encode webcam JPEG pour overlay
|
||||
ok2, jpg = cv2.imencode(".jpg", frame_bgr,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), 70])
|
||||
jpg_bytes = bytes(jpg) if ok2 else None
|
||||
|
||||
with self.state.lock():
|
||||
self.state.persons_body = bodies
|
||||
# DETRPose ne fournit pas face/hands — on vide pour
|
||||
# eviter que le renderer dessine des anciennes valeurs.
|
||||
self.state.persons_face = []
|
||||
self.state.persons_hands = []
|
||||
self.state.face_present = False
|
||||
self.state.hands_present = False
|
||||
if bodies:
|
||||
self.state.body_present = True
|
||||
# Compat single-person : on remplit les 17 premiers
|
||||
# slots du buffer body_kp (mediapipe en attend 33,
|
||||
# le reste reste a zero — acceptable).
|
||||
for k in range(33):
|
||||
if k < 17 and k < len(bodies[0]):
|
||||
self.state.body_kp[k] = bodies[0][k]
|
||||
else:
|
||||
self.state.body_kp[k] = PoseKp()
|
||||
# On remplit aussi pose_kp[17] (legacy YOLO COCO).
|
||||
for k in range(17):
|
||||
self.state.pose_kp[k] = (
|
||||
bodies[0][k] if k < len(bodies[0]) else PoseKp())
|
||||
else:
|
||||
self.state.body_present = False
|
||||
self.state.pose_count = len(bodies)
|
||||
self.state.pose_last_t = time.monotonic()
|
||||
if jpg_bytes:
|
||||
self.state.last_webcam_jpeg = jpg_bytes
|
||||
|
||||
dt = time.monotonic() - tA
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
cap.release()
|
||||
LOG.info("detrpose worker stopped")
|
||||
|
||||
|
||||
def _load_py_config(path: str):
|
||||
"""Charge une config DETRPose ecrite en .py (style detectron2/lazy)."""
|
||||
from omegaconf import OmegaConf
|
||||
# Les configs DETRPose sont des fichiers Python qui exposent un dict
|
||||
# `model = LazyCall(...)`. On utilise le helper lazy_config si dispo.
|
||||
try:
|
||||
from src.misc.lazy_config import LazyConfig # type: ignore
|
||||
return LazyConfig.load(path)
|
||||
except ImportError:
|
||||
pass
|
||||
# Fallback minimal : exec + recup des noms cles.
|
||||
ns: dict = {}
|
||||
with open(path) as f:
|
||||
code = compile(f.read(), path, "exec")
|
||||
exec(code, ns)
|
||||
cfg = OmegaConf.create({
|
||||
k: ns[k] for k in ("model", "postprocessor")
|
||||
if k in ns
|
||||
})
|
||||
return cfg
|
||||
@@ -0,0 +1,204 @@
|
||||
"""DINOv2 ViT-S/14 person re-id backend (CoreML via pyobjc).
|
||||
|
||||
Loads the .mlpackage produced by ``scripts/convert_dinov2.py`` and runs
|
||||
inference one crop at a time (pyobjc + MLDictionaryFeatureProvider).
|
||||
Same pattern as ``multihmr_coreml.py`` so Python 3.14 works (no
|
||||
coremltools dependency at runtime).
|
||||
|
||||
Embeddings are L2-normalised inside the CoreML graph, so cosine sim
|
||||
between two outputs is a plain dot product.
|
||||
|
||||
Public API::
|
||||
|
||||
reid = DinoReid(mlpackage_path) # optional path
|
||||
emb = reid.embed_crops(list_of_uint8_HWC) # -> np.ndarray (N, 384)
|
||||
DinoReid.is_available() # bool
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
import numpy as np
|
||||
|
||||
LOG = logging.getLogger("dino_reid")
|
||||
|
||||
DEFAULT_MLPACKAGE = (
|
||||
Path.home() / ".cache" / "av-live-multihmr" / "dinov2_vits14.mlpackage"
|
||||
)
|
||||
|
||||
EMBED_DIM = 384
|
||||
INPUT_SIZE = 224
|
||||
|
||||
# MLMultiArrayDataType raw values (from CoreML headers).
|
||||
ML_DTYPE_FLOAT32 = 65568
|
||||
ML_DTYPE_FLOAT16 = 65552
|
||||
ML_DTYPE_DOUBLE = 65600
|
||||
|
||||
|
||||
def _resize_crop(crop_uint8: np.ndarray) -> np.ndarray:
|
||||
"""Resize an HxWx3 uint8 crop to (3, 224, 224) float32 in [0, 1].
|
||||
|
||||
Uses ``cv2.resize`` when available, falls back to a simple stride
|
||||
sampler otherwise (avoids hard cv2 dep in test envs)."""
|
||||
if crop_uint8.ndim != 3 or crop_uint8.shape[2] != 3:
|
||||
raise ValueError(f"crop must be HxWx3 uint8, got {crop_uint8.shape}")
|
||||
if crop_uint8.shape[0] == INPUT_SIZE and crop_uint8.shape[1] == INPUT_SIZE:
|
||||
rgb = crop_uint8
|
||||
else:
|
||||
try:
|
||||
import cv2
|
||||
rgb = cv2.resize(crop_uint8, (INPUT_SIZE, INPUT_SIZE),
|
||||
interpolation=cv2.INTER_AREA)
|
||||
except ImportError:
|
||||
h, w = crop_uint8.shape[:2]
|
||||
ys = (np.linspace(0, h - 1, INPUT_SIZE)).astype(np.int32)
|
||||
xs = (np.linspace(0, w - 1, INPUT_SIZE)).astype(np.int32)
|
||||
rgb = crop_uint8[ys][:, xs]
|
||||
return (rgb.astype(np.float32) / 255.0).transpose(2, 0, 1)
|
||||
|
||||
|
||||
class DinoReid:
|
||||
"""Forward DINOv2 ViT-S/14 over RGB crops, return L2-normalised
|
||||
embeddings (N, 384)."""
|
||||
|
||||
def __init__(self, mlpackage_path: Path | str | None = None) -> None:
|
||||
self.path = Path(mlpackage_path) if mlpackage_path else DEFAULT_MLPACKAGE
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"mlpackage missing: {self.path}")
|
||||
|
||||
import objc
|
||||
from Foundation import NSURL
|
||||
|
||||
self._objc = objc
|
||||
self._NSURL = NSURL
|
||||
|
||||
ns: dict = {}
|
||||
objc.loadBundle("CoreML", ns,
|
||||
"/System/Library/Frameworks/CoreML.framework")
|
||||
self._ns = ns
|
||||
|
||||
MLModel = ns["MLModel"]
|
||||
MLModelConfiguration = ns["MLModelConfiguration"]
|
||||
cfg = MLModelConfiguration.alloc().init()
|
||||
try:
|
||||
# 2 = MLComputeUnitsAll (CPU+GPU+ANE). DINOv2 ViT-S/14
|
||||
# converts cleanly and ANE serves it well.
|
||||
cfg.setComputeUnits_(2)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
url = NSURL.fileURLWithPath_(str(self.path))
|
||||
compiled = MLModel.compileModelAtURL_error_(url, None)
|
||||
if compiled is None:
|
||||
raise RuntimeError(f"compile failed for {self.path}")
|
||||
model = MLModel.modelWithContentsOfURL_configuration_error_(
|
||||
compiled, cfg, None)
|
||||
if model is None:
|
||||
raise RuntimeError(f"load failed for {compiled}")
|
||||
self._model = model
|
||||
|
||||
# Discover the output feature name (single tensor).
|
||||
desc = model.modelDescription()
|
||||
out_names = [str(n) for n in desc.outputDescriptionsByName().keys()]
|
||||
self._out_name = out_names[0] if out_names else "embedding"
|
||||
LOG.info("dino_reid loaded (%s, out=%s)", self.path.name,
|
||||
self._out_name)
|
||||
|
||||
@classmethod
|
||||
def is_available(cls, mlpackage_path: Path | str | None = None) -> bool:
|
||||
p = Path(mlpackage_path) if mlpackage_path else DEFAULT_MLPACKAGE
|
||||
if not p.exists():
|
||||
return False
|
||||
try:
|
||||
import objc # noqa: F401
|
||||
from Foundation import NSURL # noqa: F401
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# MLMultiArray plumbing — mirrors multihmr_coreml._np_to_mlarray /
|
||||
# _mlarray_to_np. Float32 in, float32-or-float16 out.
|
||||
# ------------------------------------------------------------------
|
||||
def _np_to_mlarray(self, arr: np.ndarray):
|
||||
import ctypes
|
||||
MLMultiArray = self._ns["MLMultiArray"]
|
||||
arr = np.ascontiguousarray(arr, dtype=np.float32)
|
||||
shape = [int(s) for s in arr.shape]
|
||||
ml = MLMultiArray.alloc().initWithShape_dataType_error_(
|
||||
shape, ML_DTYPE_FLOAT32, None)
|
||||
if ml is None:
|
||||
raise RuntimeError("MLMultiArray alloc failed")
|
||||
ptr = ml.dataPointer()
|
||||
addr = int(ptr) if isinstance(ptr, int) else ctypes.cast(
|
||||
ptr, ctypes.c_void_p).value
|
||||
if addr is None:
|
||||
raise RuntimeError("dataPointer null")
|
||||
ctypes.memmove(addr, arr.ctypes.data, arr.nbytes)
|
||||
return ml
|
||||
|
||||
def _mlarray_to_np(self, ml) -> np.ndarray:
|
||||
import ctypes
|
||||
shape = tuple(int(s) for s in ml.shape())
|
||||
dtype_id = int(ml.dataType())
|
||||
count = 1
|
||||
for s in shape:
|
||||
count *= s
|
||||
ptr = ml.dataPointer()
|
||||
addr = int(ptr) if isinstance(ptr, int) else ctypes.cast(
|
||||
ptr, ctypes.c_void_p).value
|
||||
if addr is None:
|
||||
raise RuntimeError("dataPointer null")
|
||||
if dtype_id == ML_DTYPE_FLOAT16:
|
||||
raw = (ctypes.c_uint16 * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).view(np.float16).astype(np.float32)
|
||||
elif dtype_id == ML_DTYPE_FLOAT32:
|
||||
raw = (ctypes.c_float * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).copy()
|
||||
elif dtype_id == ML_DTYPE_DOUBLE:
|
||||
raw = (ctypes.c_double * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).astype(np.float32)
|
||||
else:
|
||||
raise RuntimeError(f"unsupported dtype {dtype_id}")
|
||||
return arr.reshape(shape)
|
||||
|
||||
def _predict_one(self, image_chw: np.ndarray) -> np.ndarray:
|
||||
MLDictionaryFeatureProvider = self._ns["MLDictionaryFeatureProvider"]
|
||||
MLFeatureValue = self._ns["MLFeatureValue"]
|
||||
x4 = image_chw[np.newaxis, ...] if image_chw.ndim == 3 else image_chw
|
||||
img_ml = self._np_to_mlarray(x4)
|
||||
feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml)}
|
||||
provider = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
if provider is None:
|
||||
raise RuntimeError("provider alloc failed")
|
||||
out = self._model.predictionFromFeatures_error_(provider, None)
|
||||
if out is None:
|
||||
raise RuntimeError("predict failed")
|
||||
fv = out.featureValueForName_(self._out_name)
|
||||
ml = fv.multiArrayValue()
|
||||
return self._mlarray_to_np(ml).reshape(-1)
|
||||
|
||||
def embed_crops(
|
||||
self, crops_uint8: Sequence[np.ndarray],
|
||||
) -> np.ndarray:
|
||||
"""Embed a list of HxWx3 uint8 RGB crops -> (N, 384) float32.
|
||||
|
||||
Loops one crop at a time (the CoreML model is traced for B=1).
|
||||
For typical N <= 4 this is still 10-15 ms total on M5."""
|
||||
if not crops_uint8:
|
||||
return np.zeros((0, EMBED_DIM), dtype=np.float32)
|
||||
t0 = time.perf_counter()
|
||||
out = np.zeros((len(crops_uint8), EMBED_DIM), dtype=np.float32)
|
||||
for i, c in enumerate(crops_uint8):
|
||||
chw = _resize_crop(c)
|
||||
out[i] = self._predict_one(chw)
|
||||
dt_ms = (time.perf_counter() - t0) * 1e3
|
||||
if LOG.isEnabledFor(logging.DEBUG) or dt_ms > 50.0:
|
||||
LOG.log(
|
||||
logging.DEBUG if dt_ms <= 50.0 else logging.INFO,
|
||||
"embedded %d crops in %.1f ms", len(crops_uint8), dt_ms)
|
||||
return out
|
||||
@@ -0,0 +1,100 @@
|
||||
"""One Euro filter — lissage adaptatif de keypoints en temps reel.
|
||||
|
||||
Reference : Casiez, Roussel, Vogel (CHI 2012) "1€ Filter: A Simple
|
||||
Speed-based Low-pass Filter for Noisy Input in Interactive Systems".
|
||||
|
||||
Compromis cle : faible latence quand le signal est rapide (cut-off
|
||||
haut), fort lissage quand stable (cut-off bas). Pilote la coupure par
|
||||
la vitesse instantanee.
|
||||
|
||||
Un filtre par scalaire (x, y, z separes). API :
|
||||
f = OneEuroFilter(min_cutoff=1.0, beta=0.05)
|
||||
smooth = f(value, timestamp)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class OneEuroFilter:
|
||||
min_cutoff: float = 1.0 # Hz : coupure quand stable (plus bas = plus lisse)
|
||||
beta: float = 0.05 # gain sur vitesse (plus haut = plus reactif)
|
||||
d_cutoff: float = 1.0 # coupure du derivee
|
||||
_x_prev: float | None = field(default=None, init=False, repr=False)
|
||||
_dx_prev: float = field(default=0.0, init=False, repr=False)
|
||||
_t_prev: float | None = field(default=None, init=False, repr=False)
|
||||
|
||||
def __call__(self, x: float, t: float | None = None) -> float:
|
||||
if t is None:
|
||||
t = time.monotonic()
|
||||
if self._t_prev is None:
|
||||
self._t_prev = t
|
||||
self._x_prev = x
|
||||
return x
|
||||
dt = max(1e-6, t - self._t_prev)
|
||||
# Derivee (estimee) -> filtree -> calcul cut-off adaptatif
|
||||
dx = (x - self._x_prev) / dt
|
||||
a_d = _alpha(self.d_cutoff, dt)
|
||||
dx_hat = a_d * dx + (1 - a_d) * self._dx_prev
|
||||
cutoff = self.min_cutoff + self.beta * abs(dx_hat)
|
||||
a = _alpha(cutoff, dt)
|
||||
x_hat = a * x + (1 - a) * self._x_prev
|
||||
# State
|
||||
self._x_prev = x_hat
|
||||
self._dx_prev = dx_hat
|
||||
self._t_prev = t
|
||||
return x_hat
|
||||
|
||||
def reset(self) -> None:
|
||||
self._x_prev = None
|
||||
self._t_prev = None
|
||||
self._dx_prev = 0.0
|
||||
|
||||
|
||||
def _alpha(cutoff: float, dt: float) -> float:
|
||||
tau = 1.0 / (2.0 * math.pi * cutoff)
|
||||
return 1.0 / (1.0 + tau / dt)
|
||||
|
||||
|
||||
class KpFilter:
|
||||
"""Bundle (x, y, z) pour un keypoint."""
|
||||
|
||||
def __init__(self, min_cutoff: float = 1.0, beta: float = 0.05) -> None:
|
||||
self.fx = OneEuroFilter(min_cutoff, beta)
|
||||
self.fy = OneEuroFilter(min_cutoff, beta)
|
||||
self.fz = OneEuroFilter(min_cutoff, beta)
|
||||
|
||||
def __call__(self, x: float, y: float, z: float, t: float) -> tuple[float, float, float]:
|
||||
return self.fx(x, t), self.fy(y, t), self.fz(z, t)
|
||||
|
||||
def reset(self) -> None:
|
||||
self.fx.reset(); self.fy.reset(); self.fz.reset()
|
||||
|
||||
|
||||
class SkeletonFilter:
|
||||
"""N keypoints x M personnes. Crée des KpFilter à la demande, indexé
|
||||
par (person_id, kp_index)."""
|
||||
|
||||
def __init__(self, min_cutoff: float = 1.2, beta: float = 0.08) -> None:
|
||||
self._min_cutoff = min_cutoff
|
||||
self._beta = beta
|
||||
self._table: dict[tuple[int, int], KpFilter] = {}
|
||||
|
||||
def smooth(self, person_id: int, kp_index: int,
|
||||
x: float, y: float, z: float, t: float) -> tuple[float, float, float]:
|
||||
key = (person_id, kp_index)
|
||||
f = self._table.get(key)
|
||||
if f is None:
|
||||
f = KpFilter(self._min_cutoff, self._beta)
|
||||
self._table[key] = f
|
||||
return f(x, y, z, t)
|
||||
|
||||
def forget(self, person_id: int) -> None:
|
||||
"""Supprime tous les filtres d'une personne (sortie de track)."""
|
||||
self._table = {k: v for k, v in self._table.items() if k[0] != person_id}
|
||||
|
||||
def reset_all(self) -> None:
|
||||
self._table.clear()
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Analyse fine : crops haute resolution sur visage et mains detectes.
|
||||
|
||||
Strategie : la 1ere passe Apple Vision tourne sur la frame 640x480 entiere
|
||||
(rapide, mais visage/mains sont petits → landmarks moins precis). Cette
|
||||
seconde passe identifie les ROIs (bbox visage, bbox main) depuis les
|
||||
detections initiales, **CROP** le frame original a la region, re-encode en
|
||||
JPEG haute resolution et re-execute Vision dessus. Resultat : 3-10× plus
|
||||
de pixels par region → landmarks ultra precis (utile pour mouth shape,
|
||||
eye blink, doigts fins).
|
||||
|
||||
Pour ne pas tuer le fps : cadence reduite (10 Hz vs 30 Hz du worker
|
||||
principal). Les crops sont effectues dans le MEME worker que la pass
|
||||
plein-cadre — c'est `apple_vision_pose.py` qui appelle FineAnalyzer.refine()
|
||||
apres chaque frame ou la 3eme frame seulement.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from Foundation import NSData
|
||||
|
||||
from .state import PoseKp
|
||||
|
||||
LOG = logging.getLogger("fine_analysis")
|
||||
|
||||
|
||||
def _bbox_from_kps(kps: list[PoseKp], pad: float = 0.10
|
||||
) -> tuple[float, float, float, float] | None:
|
||||
"""Calcule la bbox englobant les keypoints visibles, avec padding.
|
||||
Retourne (x_min, y_min, x_max, y_max) en coordonnees normalisees 0..1.
|
||||
None si aucun kp visible."""
|
||||
pts = [(kp.x, kp.y) for kp in kps if kp.c > 0.3]
|
||||
if not pts:
|
||||
return None
|
||||
xs = [p[0] for p in pts]; ys = [p[1] for p in pts]
|
||||
x1, y1, x2, y2 = min(xs), min(ys), max(xs), max(ys)
|
||||
dx, dy = (x2 - x1) * pad, (y2 - y1) * pad
|
||||
return (
|
||||
max(0.0, x1 - dx), max(0.0, y1 - dy),
|
||||
min(1.0, x2 + dx), min(1.0, y2 + dy),
|
||||
)
|
||||
|
||||
|
||||
class FineAnalyzer:
|
||||
"""Re-execute Vision sur des crops haute resolution des ROIs.
|
||||
|
||||
Active automatiquement quand le worker pose detecte un visage ou des
|
||||
mains. Throttle interne pour ne pas saturer ANE.
|
||||
"""
|
||||
|
||||
def __init__(self, ns_vision: dict, throttle_hz: float = 10.0,
|
||||
zoom_max: float = 4.0) -> None:
|
||||
self._ns = ns_vision
|
||||
self._period = 1.0 / max(1.0, throttle_hz)
|
||||
self._last_t = 0.0
|
||||
self._zoom_max = zoom_max
|
||||
self._VNImageRequestHandler = ns_vision.get("VNImageRequestHandler")
|
||||
self._VNDetectFaceLandmarksRequest = ns_vision.get(
|
||||
"VNDetectFaceLandmarksRequest")
|
||||
self._VNDetectHumanHandPoseRequest = ns_vision.get(
|
||||
"VNDetectHumanHandPoseRequest")
|
||||
|
||||
def should_refine(self, t_now: float) -> bool:
|
||||
if t_now - self._last_t < self._period:
|
||||
return False
|
||||
self._last_t = t_now
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def refine_face(self, frame_bgr, persons_face: list[list[PoseKp]],
|
||||
parse_face_fn) -> list[list[PoseKp]]:
|
||||
"""Pour chaque visage detecte, crop la region, re-encode JPEG et
|
||||
relance VNDetectFaceLandmarksRequest. Remplace les kp par les
|
||||
nouveaux (re-projettes en coordonnees image complete).
|
||||
|
||||
`parse_face_fn(obs, x_origin, y_origin, scale_x, scale_y)` : helper
|
||||
fourni par le worker pour parser une VNFaceObservation et retourner
|
||||
une liste de PoseKp.
|
||||
"""
|
||||
if (self._VNImageRequestHandler is None or
|
||||
self._VNDetectFaceLandmarksRequest is None or
|
||||
not persons_face):
|
||||
return persons_face
|
||||
try:
|
||||
import cv2
|
||||
except ImportError:
|
||||
return persons_face
|
||||
|
||||
h, w = frame_bgr.shape[:2]
|
||||
out = []
|
||||
for face_kps in persons_face:
|
||||
bbox = _bbox_from_kps(face_kps, pad=0.20)
|
||||
if bbox is None:
|
||||
out.append(face_kps); continue
|
||||
# Crop pixels
|
||||
x1 = int(bbox[0] * w); y1 = int(bbox[1] * h)
|
||||
x2 = int(bbox[2] * w); y2 = int(bbox[3] * h)
|
||||
cw, ch = x2 - x1, y2 - y1
|
||||
if cw < 60 or ch < 60:
|
||||
out.append(face_kps); continue
|
||||
crop = frame_bgr[y1:y2, x1:x2]
|
||||
# Upscale pour donner plus de pixels au detecteur (ANE accepte
|
||||
# n'importe quelle taille mais plus de detail = plus precis)
|
||||
zoom = min(self._zoom_max, 640.0 / max(cw, ch))
|
||||
if zoom > 1.0:
|
||||
crop = cv2.resize(crop, None, fx=zoom, fy=zoom,
|
||||
interpolation=cv2.INTER_CUBIC)
|
||||
ok, jpg = cv2.imencode(".jpg", crop,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), 85])
|
||||
if not ok:
|
||||
out.append(face_kps); continue
|
||||
jpg_bytes = bytes(jpg)
|
||||
data = NSData.dataWithBytes_length_(jpg_bytes, len(jpg_bytes))
|
||||
handler = self._VNImageRequestHandler.alloc()\
|
||||
.initWithData_options_(data, {})
|
||||
req = self._VNDetectFaceLandmarksRequest.alloc().init()
|
||||
ret = handler.performRequests_error_([req], None)
|
||||
ok2 = ret[0] if isinstance(ret, tuple) else bool(ret)
|
||||
if not ok2:
|
||||
out.append(face_kps); continue
|
||||
results = req.results() or []
|
||||
if not results:
|
||||
out.append(face_kps); continue
|
||||
# Re-projette les coords du crop vers l'image entiere
|
||||
# bbox normalisees: x_origin=bbox[0], scale_x=(bbox[2]-bbox[0])
|
||||
obs = results[0]
|
||||
new_kps = parse_face_fn(
|
||||
obs,
|
||||
x_origin=bbox[0], y_origin=bbox[1],
|
||||
scale_x=(bbox[2] - bbox[0]),
|
||||
scale_y=(bbox[3] - bbox[1]),
|
||||
)
|
||||
out.append(new_kps if new_kps else face_kps)
|
||||
return out
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def refine_hands(self, frame_bgr, persons_hands: list[list[PoseKp]],
|
||||
parse_hand_fn) -> list[list[PoseKp]]:
|
||||
"""Pareil que refine_face mais pour les mains.
|
||||
`parse_hand_fn(obs, x_origin, y_origin, scale_x, scale_y)` →
|
||||
list[PoseKp] de 21 elements."""
|
||||
if (self._VNImageRequestHandler is None or
|
||||
self._VNDetectHumanHandPoseRequest is None or
|
||||
not persons_hands):
|
||||
return persons_hands
|
||||
try:
|
||||
import cv2
|
||||
except ImportError:
|
||||
return persons_hands
|
||||
|
||||
h, w = frame_bgr.shape[:2]
|
||||
out = []
|
||||
for hand_kps in persons_hands:
|
||||
bbox = _bbox_from_kps(hand_kps, pad=0.30)
|
||||
if bbox is None:
|
||||
out.append(hand_kps); continue
|
||||
x1 = int(bbox[0] * w); y1 = int(bbox[1] * h)
|
||||
x2 = int(bbox[2] * w); y2 = int(bbox[3] * h)
|
||||
cw, ch = x2 - x1, y2 - y1
|
||||
if cw < 40 or ch < 40:
|
||||
out.append(hand_kps); continue
|
||||
crop = frame_bgr[y1:y2, x1:x2]
|
||||
zoom = min(self._zoom_max, 320.0 / max(cw, ch))
|
||||
if zoom > 1.0:
|
||||
crop = cv2.resize(crop, None, fx=zoom, fy=zoom,
|
||||
interpolation=cv2.INTER_CUBIC)
|
||||
ok, jpg = cv2.imencode(".jpg", crop,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), 85])
|
||||
if not ok:
|
||||
out.append(hand_kps); continue
|
||||
jpg_bytes = bytes(jpg)
|
||||
data = NSData.dataWithBytes_length_(jpg_bytes, len(jpg_bytes))
|
||||
handler = self._VNImageRequestHandler.alloc()\
|
||||
.initWithData_options_(data, {})
|
||||
req = self._VNDetectHumanHandPoseRequest.alloc().init()
|
||||
try:
|
||||
req.setMaximumHandCount_(1) # un crop = une main
|
||||
except Exception:
|
||||
pass
|
||||
ret = handler.performRequests_error_([req], None)
|
||||
ok2 = ret[0] if isinstance(ret, tuple) else bool(ret)
|
||||
if not ok2:
|
||||
out.append(hand_kps); continue
|
||||
results = req.results() or []
|
||||
if not results:
|
||||
out.append(hand_kps); continue
|
||||
obs = results[0]
|
||||
new_kps = parse_hand_fn(
|
||||
obs,
|
||||
x_origin=bbox[0], y_origin=bbox[1],
|
||||
scale_x=(bbox[2] - bbox[0]),
|
||||
scale_y=(bbox[3] - bbox[1]),
|
||||
)
|
||||
out.append(new_kps if new_kps else hand_kps)
|
||||
return out
|
||||
@@ -0,0 +1,175 @@
|
||||
"""MediaPipe Holistic : capture webcam + landmarks corps/visage/mains.
|
||||
|
||||
Remplace pose.py (YOLO 17 kp) par mediapipe.tasks.HolisticLandmarker :
|
||||
- 33 points POSE_LANDMARKS (body skeleton)
|
||||
- 478 points FACE_LANDMARKS (mesh visage + iris)
|
||||
- 21 × 2 HAND_LANDMARKS (mains droite + gauche)
|
||||
Total ~553 landmarks, ~230 segments de connexion.
|
||||
|
||||
Le modele .task est telecharge dans ~/.cache/mediapipe au premier run.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from .state import PoseKp, State
|
||||
|
||||
LOG = logging.getLogger("holistic")
|
||||
|
||||
MODEL_URL = (
|
||||
"https://storage.googleapis.com/mediapipe-models/"
|
||||
"holistic_landmarker/holistic_landmarker/float16/latest/"
|
||||
"holistic_landmarker.task"
|
||||
)
|
||||
CACHE_DIR = Path.home() / ".cache" / "av-live-mediapipe"
|
||||
MODEL_PATH = CACHE_DIR / "holistic_landmarker.task"
|
||||
|
||||
|
||||
def _ensure_model() -> Path:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if MODEL_PATH.exists() and MODEL_PATH.stat().st_size > 1_000_000:
|
||||
return MODEL_PATH
|
||||
LOG.info("downloading holistic model (%s) -> %s", MODEL_URL, MODEL_PATH)
|
||||
urllib.request.urlretrieve(MODEL_URL, MODEL_PATH)
|
||||
LOG.info("download OK (%d bytes)", MODEL_PATH.stat().st_size)
|
||||
return MODEL_PATH
|
||||
|
||||
|
||||
class HolisticWorker:
|
||||
"""Thread de capture webcam + inference MediaPipe Holistic."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state: State,
|
||||
camera_index: int = 0,
|
||||
target_fps: float = 20.0,
|
||||
min_pose_conf: float = 0.5,
|
||||
min_face_conf: float = 0.5,
|
||||
min_hand_conf: float = 0.4,
|
||||
) -> None:
|
||||
self.state = state
|
||||
self.camera_index = camera_index
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self.min_pose_conf = min_pose_conf
|
||||
self.min_face_conf = min_face_conf
|
||||
self.min_hand_conf = min_hand_conf
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stop = threading.Event()
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="holistic", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
import cv2
|
||||
import numpy as np
|
||||
import mediapipe as mp
|
||||
from mediapipe.tasks.python import BaseOptions
|
||||
from mediapipe.tasks.python.vision import (
|
||||
HolisticLandmarker, HolisticLandmarkerOptions, RunningMode,
|
||||
)
|
||||
except ModuleNotFoundError as e:
|
||||
LOG.error("dependances manquantes : %s — uv sync --extra pose", e)
|
||||
return
|
||||
|
||||
try:
|
||||
model_path = _ensure_model()
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("download model failed: %s", e)
|
||||
return
|
||||
|
||||
opts = HolisticLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(model_path)),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
min_pose_detection_confidence=self.min_pose_conf,
|
||||
min_pose_landmarks_confidence=self.min_pose_conf,
|
||||
min_pose_suppression_threshold=0.5,
|
||||
min_face_detection_confidence=self.min_face_conf,
|
||||
min_face_landmarks_confidence=self.min_face_conf,
|
||||
min_face_suppression_threshold=0.3,
|
||||
min_hand_landmarks_confidence=self.min_hand_conf,
|
||||
)
|
||||
landmarker = HolisticLandmarker.create_from_options(opts)
|
||||
LOG.info("HolisticLandmarker pret")
|
||||
|
||||
cap = cv2.VideoCapture(self.camera_index)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index %d indisponible (TCC ?)", self.camera_index)
|
||||
return
|
||||
LOG.info("camera ouverte (index %d)", self.camera_index)
|
||||
|
||||
t0_ms = int(time.monotonic() * 1000)
|
||||
while not self._stop.is_set():
|
||||
tA = time.monotonic()
|
||||
ok, frame_bgr = cap.read()
|
||||
if not ok or frame_bgr is None:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
h, w = frame_bgr.shape[:2]
|
||||
# MediaPipe attend RGB
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame_rgb)
|
||||
ts_ms = int(time.monotonic() * 1000) - t0_ms
|
||||
try:
|
||||
result = landmarker.detect_for_video(mp_img, ts_ms)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("inference: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# Encode JPEG pour le NSImageView fond
|
||||
ok2, jpg = cv2.imencode(".jpg", frame_bgr,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), 70])
|
||||
jpg_bytes = bytes(jpg) if ok2 else None
|
||||
|
||||
with self.state.lock():
|
||||
# CORPS (33) — liste plate de NormalizedLandmark
|
||||
body = result.pose_landmarks or []
|
||||
self.state.body_present = len(body) > 0
|
||||
for k, lm in enumerate(body[:33]):
|
||||
v = lm.visibility if lm.visibility is not None else 1.0
|
||||
self.state.body_kp[k] = PoseKp(
|
||||
x=float(lm.x), y=float(lm.y), c=float(v))
|
||||
|
||||
# VISAGE (478)
|
||||
face = result.face_landmarks or []
|
||||
self.state.face_present = len(face) > 0
|
||||
for k, lm in enumerate(face[:478]):
|
||||
self.state.face_kp[k] = PoseKp(
|
||||
x=float(lm.x), y=float(lm.y), c=1.0)
|
||||
|
||||
# MAINS (21 + 21)
|
||||
lh = result.left_hand_landmarks or []
|
||||
rh = result.right_hand_landmarks or []
|
||||
for k, lm in enumerate(lh[:21]):
|
||||
self.state.left_hand_kp[k] = PoseKp(
|
||||
x=float(lm.x), y=float(lm.y), c=1.0)
|
||||
for k, lm in enumerate(rh[:21]):
|
||||
self.state.right_hand_kp[k] = PoseKp(
|
||||
x=float(lm.x), y=float(lm.y), c=1.0)
|
||||
self.state.hands_present = bool(lh) or bool(rh)
|
||||
|
||||
# Compatibilite : on remplit pose_count + pose_last_t
|
||||
self.state.pose_count = int(bool(body))
|
||||
self.state.pose_last_t = time.monotonic()
|
||||
if jpg_bytes:
|
||||
self.state.last_webcam_jpeg = jpg_bytes
|
||||
|
||||
dt = time.monotonic() - tA
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
cap.release()
|
||||
landmarker.close()
|
||||
LOG.info("holistic worker stopped")
|
||||
@@ -0,0 +1,653 @@
|
||||
"""Entry point du visualizer Metal pour le mode data-only.
|
||||
|
||||
Lance :
|
||||
- Une fenetre AppKit avec une MTKView plein-cadre
|
||||
- Un thread OSC en arriere-plan qui ecoute :57123
|
||||
- La run loop NSApplication (jamais retournee tant que la window est ouverte)
|
||||
|
||||
Usage :
|
||||
uv run python -m data_only_viz.main
|
||||
uv run python -m data_only_viz.main --port 57123 --fullscreen
|
||||
|
||||
Le main thread DOIT etre la run loop AppKit (regle macOS). Le listener
|
||||
OSC tourne dans un thread daemon.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
|
||||
import objc
|
||||
from AppKit import (
|
||||
NSApp,
|
||||
NSApplication,
|
||||
NSApplicationActivationPolicyRegular,
|
||||
NSBackingStoreBuffered,
|
||||
NSColor,
|
||||
NSEvent,
|
||||
NSEventMaskKeyDown,
|
||||
NSData,
|
||||
NSFont,
|
||||
NSImage,
|
||||
NSImageScaleProportionallyUpOrDown,
|
||||
NSImageView,
|
||||
NSMakeRect,
|
||||
NSObject,
|
||||
NSScreen,
|
||||
NSTextView,
|
||||
NSTimer,
|
||||
NSWindow,
|
||||
NSWindowStyleMaskClosable,
|
||||
NSWindowStyleMaskMiniaturizable,
|
||||
NSWindowStyleMaskResizable,
|
||||
NSWindowStyleMaskTitled,
|
||||
NSViewWidthSizable,
|
||||
NSViewHeightSizable,
|
||||
)
|
||||
from MetalKit import MTKView
|
||||
|
||||
from pythonosc.udp_client import SimpleUDPClient
|
||||
|
||||
from .osc_listener import OscListener
|
||||
from .renderer import MetalRenderer
|
||||
from .state import (
|
||||
KEYMAP_AUDIO, KEYMAP_SOURCE, KEYMAP_SOURCE_NUM, KEYMAP_VIDEO, State,
|
||||
)
|
||||
|
||||
LOG = logging.getLogger("main")
|
||||
|
||||
|
||||
class AppDelegate(NSObject):
|
||||
def initWithOpts_(self, opts): # noqa: N802
|
||||
self = objc.super(AppDelegate, self).init()
|
||||
if self is None:
|
||||
return None
|
||||
self._opts = opts
|
||||
self._state = State()
|
||||
self._listener = OscListener(self._state, host="127.0.0.1", port=opts.port)
|
||||
# Sender vers sclang pour les changements de scene depuis le clavier
|
||||
self._scClient = SimpleUDPClient("127.0.0.1", opts.sclang_port)
|
||||
self._pose_worker = None
|
||||
return self
|
||||
|
||||
def applicationDidFinishLaunching_(self, notification): # noqa: N802
|
||||
LOG.info("applicationDidFinishLaunching: start")
|
||||
# Mode headless : --multi-hmr cede l'affichage a AV-Live-Body
|
||||
# (Swift RealityKit). On garde seulement le worker pose + le
|
||||
# sender TCP, donc pas de NSWindow / Metal renderer / HUD.
|
||||
if getattr(self._opts, "multi_hmr", False):
|
||||
LOG.info("multi-hmr mode: headless (no NSWindow)")
|
||||
self._headless = True
|
||||
from .multi_hmr_worker import MultiHMRWorker
|
||||
if not MultiHMRWorker.is_available():
|
||||
LOG.error(
|
||||
"Multi-HMR requested via --multi-hmr but checkpoint is missing. "
|
||||
"Run scripts/setup_multihmr.sh first, or omit --multi-hmr to use MediaPipe."
|
||||
)
|
||||
sys.exit(2)
|
||||
self._listener.start()
|
||||
self._start_pose_worker()
|
||||
LOG.info("headless ready — OSC :%d, TCP sender :57130",
|
||||
self._opts.port)
|
||||
return
|
||||
self._headless = False
|
||||
# 1) Fenetre
|
||||
screen = NSScreen.mainScreen().frame()
|
||||
w, h = self._opts.width, self._opts.height
|
||||
x = (screen.size.width - w) / 2
|
||||
y = (screen.size.height - h) / 2
|
||||
style = (NSWindowStyleMaskTitled
|
||||
| NSWindowStyleMaskClosable
|
||||
| NSWindowStyleMaskResizable
|
||||
| NSWindowStyleMaskMiniaturizable)
|
||||
self._window = NSWindow.alloc().initWithContentRect_styleMask_backing_defer_(
|
||||
NSMakeRect(x, y, w, h), style, NSBackingStoreBuffered, False)
|
||||
self._window.setTitle_("AV-Live · data-only viz")
|
||||
self._window.setBackgroundColor_(NSColor.blackColor())
|
||||
LOG.info("window created")
|
||||
|
||||
# 2) Container fullscreen avec 2 couches :
|
||||
# z=0 NSImageView : flux webcam live PLEIN-CADRE (sous le Metal)
|
||||
# z=1 MTKView : skeleton + viz effects (transparent par-dessus)
|
||||
from AppKit import NSView
|
||||
self._container = NSView.alloc().initWithFrame_(NSMakeRect(0, 0, w, h))
|
||||
self._container.setWantsLayer_(True)
|
||||
self._container.layer().setBackgroundColor_(
|
||||
NSColor.blackColor().CGColor())
|
||||
|
||||
# 2a) Webcam fond plein-cadre
|
||||
self._cam = NSImageView.alloc().initWithFrame_(NSMakeRect(0, 0, w, h))
|
||||
self._cam.setImageScaling_(NSImageScaleProportionallyUpOrDown)
|
||||
self._cam.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable)
|
||||
# Background visible meme sans frame (debug)
|
||||
self._cam.setWantsLayer_(True)
|
||||
self._cam.layer().setBackgroundColor_(
|
||||
NSColor.colorWithCalibratedRed_green_blue_alpha_(0.05, 0.05, 0.1, 1)
|
||||
.CGColor())
|
||||
self._container.addSubview_(self._cam)
|
||||
|
||||
# 2b) MTKView par-dessus, transparent : on voit la webcam dessous
|
||||
self._renderer = MetalRenderer.alloc().initWithState_(self._state)
|
||||
LOG.info("renderer created")
|
||||
device = self._renderer.device()
|
||||
self._mtkview = MTKView.alloc().initWithFrame_device_(
|
||||
NSMakeRect(0, 0, w, h), device)
|
||||
self._mtkview.setDelegate_(self._renderer)
|
||||
self._mtkview.setPreferredFramesPerSecond_(60)
|
||||
self._mtkview.setColorPixelFormat_(80) # BGRA8Unorm
|
||||
# Transparence : on rend le MTKView en RGBA et on laisse la
|
||||
# CAMetalLayer ne pas etre opaque pour voir le NSImageView dessous.
|
||||
layer = self._mtkview.layer()
|
||||
if layer is not None:
|
||||
layer.setOpaque_(False)
|
||||
# MTKView expose framebufferOnly directement
|
||||
try: self._mtkview.setFramebufferOnly_(False)
|
||||
except Exception: pass
|
||||
self._mtkview.setClearColor_((0.0, 0.0, 0.0, 0.0))
|
||||
self._mtkview.setAutoresizingMask_(NSViewWidthSizable | NSViewHeightSizable)
|
||||
self._container.addSubview_(self._mtkview)
|
||||
LOG.info("mtkview configured (transparent overlay)")
|
||||
|
||||
self._window.setContentView_(self._container)
|
||||
# Window au premier plan : level = floating, always on top jusqu'a
|
||||
# ce qu'on perde le focus. Sinon la fenetre peut etre cachee
|
||||
# derriere l'IDE / le terminal.
|
||||
self._window.setLevel_(3) # NSFloatingWindowLevel
|
||||
self._window.makeKeyAndOrderFront_(None)
|
||||
NSApp().activateIgnoringOtherApps_(True)
|
||||
self._window.makeFirstResponder_(self._container)
|
||||
LOG.info("window shown + key focus forced + floating level")
|
||||
|
||||
# 2b) HUD : overlay NSTextView semi-transparent au-dessus du MTKView.
|
||||
# NSTextView prend en charge le rendu CoreText sans avoir a passer
|
||||
# par un MTLBuffer texte. On le pose comme subview du MTKView.
|
||||
self._hud = NSTextView.alloc().initWithFrame_(
|
||||
NSMakeRect(12, 12, 340, 240))
|
||||
self._hud.setEditable_(False)
|
||||
self._hud.setSelectable_(False)
|
||||
self._hud.setDrawsBackground_(True)
|
||||
self._hud.setBackgroundColor_(
|
||||
NSColor.colorWithCalibratedRed_green_blue_alpha_(0, 0, 0, 0.45))
|
||||
self._hud.setFont_(NSFont.fontWithName_size_("Menlo", 11))
|
||||
self._hud.setTextColor_(NSColor.whiteColor())
|
||||
self._hud.setAutoresizingMask_(NSViewHeightSizable)
|
||||
self._mtkview.addSubview_(self._hud)
|
||||
# Timer 10 Hz, rafraichit le texte avec les valeurs du State.
|
||||
self._hudTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
|
||||
0.1, self, "refreshHud:", None, True)
|
||||
|
||||
# 2c) Timer webcam update (NSImageView fullscreen deja cree en 2a)
|
||||
self._camTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
|
||||
1.0 / 15.0, self, "refreshCam:", None, True)
|
||||
|
||||
# 2d) Auto-engage du mode openpos (#9) quand des personnes sont
|
||||
# detectees. Si l'utilisateur a force un mode au clavier dans
|
||||
# les 8 dernieres secondes, on n'override pas (lock manuel).
|
||||
self._user_viz_lock_t = 0.0 # set par _on_key, lu par autoOpenpos
|
||||
self._autoOpenposTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
|
||||
0.5, self, "autoOpenpos:", None, True)
|
||||
|
||||
if self._opts.fullscreen:
|
||||
self._window.toggleFullScreen_(None)
|
||||
|
||||
# 3) Listener OSC
|
||||
self._listener.start()
|
||||
LOG.info("ready — listening OSC :%d (sc -> :%d)",
|
||||
self._opts.port, self._opts.sclang_port)
|
||||
|
||||
# 3b) Pose worker (optionnel, cv2 + YOLOv8-pose -> State)
|
||||
if self._opts.pose:
|
||||
self._request_camera_and_start_pose()
|
||||
|
||||
def _request_camera_and_start_pose(self):
|
||||
"""macOS exige que la demande TCC camera (AVCaptureDevice.requestAccess)
|
||||
vienne du MAIN THREAD AppKit. OpenCV depuis un worker thread plante
|
||||
avec 'can not spin main run loop from other thread'. On demande ici,
|
||||
sur le main thread, PUIS on lance le pose worker avec
|
||||
OPENCV_AVFOUNDATION_SKIP_AUTH=1 pour qu'il ne tente pas une 2e demande."""
|
||||
import os
|
||||
os.environ["OPENCV_AVFOUNDATION_SKIP_AUTH"] = "1"
|
||||
try:
|
||||
from AVFoundation import (
|
||||
AVCaptureDevice, AVMediaTypeVideo,
|
||||
AVAuthorizationStatusAuthorized,
|
||||
AVAuthorizationStatusNotDetermined,
|
||||
)
|
||||
status = AVCaptureDevice.authorizationStatusForMediaType_(AVMediaTypeVideo)
|
||||
LOG.info("camera TCC status: %s", status)
|
||||
if status == AVAuthorizationStatusAuthorized:
|
||||
self._start_pose_worker()
|
||||
return
|
||||
if status == AVAuthorizationStatusNotDetermined:
|
||||
LOG.info("requesting camera access via AVFoundation...")
|
||||
def _handler(granted):
|
||||
LOG.info("camera access granted=%s", granted)
|
||||
if granted:
|
||||
# Le handler tourne sur un thread arbitraire ; on dispatch
|
||||
# vers le main pour creer le worker proprement.
|
||||
from Foundation import NSOperationQueue
|
||||
NSOperationQueue.mainQueue().addOperationWithBlock_(
|
||||
self._start_pose_worker)
|
||||
AVCaptureDevice.requestAccessForMediaType_completionHandler_(
|
||||
AVMediaTypeVideo, _handler)
|
||||
return
|
||||
LOG.warning("camera access denied — Reglages > Confidentialite")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("AVFoundation TCC check failed (%s) — tentative directe", e)
|
||||
self._start_pose_worker()
|
||||
|
||||
def _start_pose_worker(self):
|
||||
# Priorite (user feedback : on veut FACE + HANDS + BODY toujours
|
||||
# disponibles pour le mapping sonore + viz openpos, donc MediaPipe
|
||||
# Multi devient le default avec ses 33 body + 478 face + 42 hand
|
||||
# par personne, plutot qu'Apple Vision body-only 13 joints) :
|
||||
# 0. Multi-HMR (opt-in via --multi-hmr flag)
|
||||
# 1. MediaPipe Multi (33+478+42 kp × 4 personnes) — DEFAUT
|
||||
# 2. Apple Vision body pose (fallback si MediaPipe casse)
|
||||
# 3. CoreML pose, DETRPose, Holistic, YOLO — fallbacks
|
||||
import os as _os
|
||||
# 0. Multi-HMR (SMPL-X 10475 verts mesh dense) — opt-in via flag
|
||||
if getattr(self._opts, "multi_hmr", False):
|
||||
try:
|
||||
from .multi_hmr_worker import MultiHMRWorker
|
||||
from .smplx_osc_sender import SMPLXTCPSender
|
||||
if MultiHMRWorker.is_available():
|
||||
# target_fps=30 : the worker loop used to self-throttle
|
||||
# at 10 fps (sleep(period - dt)). With the async remote
|
||||
# backend (drop-newest in / latest out queue), we want
|
||||
# the loop to spin at camera rate so we always submit
|
||||
# the freshest frame and drain the freshest result.
|
||||
self._pose_worker = MultiHMRWorker(
|
||||
self._state, num_persons=4,
|
||||
target_fps=float(_os.environ.get(
|
||||
"MULTIHMR_LOOP_FPS", "30.0")),
|
||||
device=getattr(self._opts, "pose_device", "mps"),
|
||||
det_thresh=getattr(self._opts, "det_thresh", 0.15),
|
||||
nms_kernel_size=getattr(
|
||||
self._opts, "nms_kernel_size", 5),
|
||||
motion_gate=getattr(
|
||||
self._opts, "motion_gate", 5.0),
|
||||
camera_index=getattr(self._opts, "camera_index", -1))
|
||||
self._pose_worker.start()
|
||||
# MESH_RIG=0 disables the 30 fps rigid translation
|
||||
# rigger from mesh_rigger.py (used to debug deformation
|
||||
# issues introduced by the hybrid rigging path).
|
||||
self._smplx_tcp = SMPLXTCPSender(
|
||||
self._state,
|
||||
enable_rigging=os.environ.get("MESH_RIG", "1") != "0",
|
||||
)
|
||||
self._smplx_tcp.start()
|
||||
LOG.info("worker: Multi-HMR + SMPL-X (mesh dense)")
|
||||
# Secondary body-pose worker in parallel: AVLiveBody
|
||||
# gets body keypoints on UDP :57126 alongside the mesh
|
||||
# on TCP :57130. Default: Apple Vision (ANE-accel,
|
||||
# body only 19 joints). Set AV_LIVE_PARALLEL_POSE=
|
||||
# mediapipe to swap to MediaPipe Holistic (CPU
|
||||
# XNNPACK but provides face + hand + 3D world).
|
||||
# Defaut: lance BOTH Apple Vision (body 19 joints sur
|
||||
# ANE, ~30 fps) ET MediaPipe Multi (face 468 + hands 21
|
||||
# + pose 3D world sur CPU XNNPACK). Set
|
||||
# AV_LIVE_PARALLEL_POSE=apple_vision pour ne garder que
|
||||
# le path ANE (face/hand fin disparait), ou =mediapipe
|
||||
# pour ne garder que CPU.
|
||||
parallel = _os.environ.get(
|
||||
"AV_LIVE_PARALLEL_POSE", "both")
|
||||
if parallel in ("apple_vision", "both"):
|
||||
try:
|
||||
from .apple_vision_pose import AppleVisionPoseWorker
|
||||
if AppleVisionPoseWorker.is_available():
|
||||
self._av_worker = AppleVisionPoseWorker(
|
||||
self._state, target_fps=30.0,
|
||||
num_persons=4)
|
||||
self._av_worker.start()
|
||||
LOG.info("worker: + Apple Vision body pose "
|
||||
"(ANE) in parallel")
|
||||
else:
|
||||
raise RuntimeError("apple_vision unavailable")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("Apple Vision parallel start failed "
|
||||
"(%s)", e)
|
||||
if parallel in ("mediapipe", "both"):
|
||||
try:
|
||||
from .multi import MultiWorker
|
||||
self._mp_worker = MultiWorker(
|
||||
self._state, num_persons=4)
|
||||
self._mp_worker.start()
|
||||
LOG.info("worker: + MediaPipe Multi (3D pose "
|
||||
"+ face + hand) in parallel")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("MediaPipe parallel start failed "
|
||||
"(%s)", e)
|
||||
return
|
||||
LOG.info("Multi-HMR indisponible (checkpoints manquants) "
|
||||
"— voir scripts/setup_multihmr.sh")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("Multi-HMR failed (%s) — fallback", e)
|
||||
# 1. MediaPipe Multi : DEFAUT pour le mapping sonore + openpos
|
||||
# (33 body + 478 face + 21x2 hands × 4 personnes). Skip via
|
||||
# AV_LIVE_MEDIAPIPE=0 si on prefere body-only ANE-accelere.
|
||||
if _os.environ.get("AV_LIVE_MEDIAPIPE") != "0":
|
||||
try:
|
||||
from .multi import MultiWorker
|
||||
self._pose_worker = MultiWorker(self._state, num_persons=4)
|
||||
self._pose_worker.start()
|
||||
LOG.info("worker: MediaPipe Multi (Pose+Face+Hand × 4)")
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("MediaPipe Multi unavailable (%s) — fallback", e)
|
||||
# 2. Apple Vision body pose : fallback si MediaPipe casse.
|
||||
# Body only, 13 joints, pas de face/hands.
|
||||
if _os.environ.get("AV_LIVE_APPLE_VISION") != "0":
|
||||
try:
|
||||
from .apple_vision_pose import AppleVisionPoseWorker
|
||||
if AppleVisionPoseWorker.is_available():
|
||||
self._pose_worker = AppleVisionPoseWorker(
|
||||
self._state, target_fps=30.0, num_persons=4)
|
||||
self._pose_worker.start()
|
||||
LOG.info("worker: Apple Vision body pose "
|
||||
"(ANE natif, body only, multi-personne)")
|
||||
return
|
||||
LOG.info("Apple Vision body pose indisponible "
|
||||
"(macOS < 11 ?) — fallback")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("Apple Vision pose indisponible (%s) — fallback", e)
|
||||
if _os.environ.get("AV_LIVE_COREML") != "0":
|
||||
try:
|
||||
from .coreml_pose import CoreMLPoseWorker
|
||||
if CoreMLPoseWorker.is_available():
|
||||
self._pose_worker = CoreMLPoseWorker(
|
||||
self._state, target_fps=30.0, num_persons=4)
|
||||
self._pose_worker.start()
|
||||
LOG.info("worker: CoreML pose natif "
|
||||
"(AVFoundation + YOLO11n-pose ANE)")
|
||||
return
|
||||
LOG.info("CoreML pose .mlpackage absent — "
|
||||
"lancer 'uv run python -m data_only_viz.scripts.convert_coreml' "
|
||||
"puis relancer pour activer le pipeline ANE")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("CoreML pose indisponible (%s) — fallback", e)
|
||||
if _os.environ.get("AV_LIVE_DETRPOSE") == "1":
|
||||
try:
|
||||
from .detrpose import DETRPoseWorker, is_available
|
||||
if is_available():
|
||||
self._pose_worker = DETRPoseWorker(
|
||||
self._state, num_persons=4,
|
||||
model_size=getattr(self._opts, "detrpose_model_size", "n"))
|
||||
self._pose_worker.start()
|
||||
LOG.info("worker: DETRPose (transformer, body 17 kp × 4)")
|
||||
return
|
||||
LOG.info("DETRPose pas installe — fallback MediaPipe Multi "
|
||||
"(voir data_only_viz/detrpose.py pour install)")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("detrpose indisponible (%s) — fallback multi", e)
|
||||
# MediaPipe Multi deja tente en priorite 1 ; on saute direct
|
||||
# au fallback holistic puis YOLO.
|
||||
try:
|
||||
from .holistic import HolisticWorker
|
||||
self._pose_worker = HolisticWorker(self._state)
|
||||
self._pose_worker.start()
|
||||
LOG.info("worker: MediaPipe Holistic (mono-personne ~553 lm)")
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("holistic unavailable (%s) — fallback YOLO", e)
|
||||
from .pose import PoseWorker
|
||||
self._pose_worker = PoseWorker(
|
||||
self._state, device=self._opts.pose_device)
|
||||
self._pose_worker.start()
|
||||
|
||||
# 4) Hook clavier : local (app au focus) + global (app au fond).
|
||||
# Le global monitor est read-only mais permet de garder le pilotage
|
||||
# quand l'utilisateur a une autre app au premier plan (IDE,
|
||||
# browser). On ignore le retour pour le global (sinon double-trigger).
|
||||
self._kb_monitor = NSEvent.addLocalMonitorForEventsMatchingMask_handler_(
|
||||
NSEventMaskKeyDown, self._on_key)
|
||||
self._kb_global = NSEvent.addGlobalMonitorForEventsMatchingMask_handler_(
|
||||
NSEventMaskKeyDown, self._on_key_global)
|
||||
# Force le focus initial pour que le local monitor reçoive
|
||||
# tout de suite les touches sans nécessiter un clic.
|
||||
NSApp().activateIgnoringOtherApps_(True)
|
||||
self._window.makeKeyAndOrderFront_(None)
|
||||
self._window.makeFirstResponder_(self._container)
|
||||
|
||||
_cam_log_count = 0
|
||||
|
||||
def refreshCam_(self, _timer): # noqa: N802
|
||||
"""Lit la derniere frame JPEG du pose worker et l'affiche en fond."""
|
||||
with self._state.lock():
|
||||
jpg = self._state.last_webcam_jpeg
|
||||
if not jpg:
|
||||
return
|
||||
try:
|
||||
# NSData : on essaye 2 APIs ; la deuxieme est plus robuste
|
||||
# avec pyobjc 12 pour les Python bytes.
|
||||
data = NSData.dataWithBytes_length_(jpg, len(jpg))
|
||||
except Exception:
|
||||
data = NSData.alloc().initWithBytes_length_(jpg, len(jpg))
|
||||
img = NSImage.alloc().initWithData_(data)
|
||||
if img is not None and img.isValid():
|
||||
self._cam.setImage_(img)
|
||||
self.__class__._cam_log_count += 1
|
||||
if self.__class__._cam_log_count in (1, 30, 150):
|
||||
LOG.info("cam frame #%d displayed (size %dx%d)",
|
||||
self.__class__._cam_log_count,
|
||||
int(img.size().width), int(img.size().height))
|
||||
elif self.__class__._cam_log_count == 0:
|
||||
LOG.warning("cam: NSImage init failed (jpg %d bytes)", len(jpg))
|
||||
self.__class__._cam_log_count = -1
|
||||
|
||||
def refreshHud_(self, _timer): # noqa: N802
|
||||
"""Format le HUD avec la valeur courante des flux + correspondance
|
||||
avec le rendu visuel. Appele a 10 Hz par NSTimer."""
|
||||
s = self._state
|
||||
with s.lock():
|
||||
mode = s.viz_mode
|
||||
mode_name = s.viz_mode_names[mode]
|
||||
preset = s.active_preset
|
||||
scene = s.active_scene
|
||||
kp = s.swpc_kp
|
||||
flare = s.swpc_flare_norm
|
||||
wind = s.swpc_wind_speed
|
||||
bz = s.swpc_bz
|
||||
netz = s.netz_dev
|
||||
lr = s.lightning_rate_min
|
||||
quake = s.usgs_last_mag
|
||||
planes = s.aviation_count
|
||||
social = s.social_rate
|
||||
pose_n = s.pose_count
|
||||
pose_live = s.pose_alive()
|
||||
rms = s.rms
|
||||
bpm = s.bpm
|
||||
beat = s.beat
|
||||
bridge_age = 0.0
|
||||
if s.last_heartbeat:
|
||||
import time as _t
|
||||
bridge_age = _t.monotonic() - s.last_heartbeat
|
||||
# Marqueurs : ce que chaque donnee fait dans le rendu courant
|
||||
kp_norm = min(1.0, kp / 9.0)
|
||||
wind_norm = max(0.0, min(1.0, (wind - 280.0) / 600.0))
|
||||
bz_norm = max(-1.0, min(1.0, bz / 15.0))
|
||||
bridge_ok = "OK" if bridge_age < 15 else "DOWN"
|
||||
bar = lambda v, w=14: "█" * int(max(0.0, min(1.0, v)) * w) + \
|
||||
"·" * (w - int(max(0.0, min(1.0, v)) * w))
|
||||
bits = []
|
||||
if preset: bits.append(f"source:{preset}")
|
||||
if scene: bits.append(f"scène:{scene}")
|
||||
active_line = (" " + " · ".join(bits)) if bits else ""
|
||||
bridge_label = "OK" if bridge_age < 15 else "ARRÊTÉ"
|
||||
pose_label = "OUI" if pose_live else "non"
|
||||
txt = (
|
||||
f"AV-Live data-only · vidéo:[{mode}]{mode_name}{active_line}\n"
|
||||
f"────────────────────────────────────────\n"
|
||||
f" Touches vidéo:azertyuiop audio:qsdfghjklm source:wxcvbn 0-9\n"
|
||||
f"────────────────────────────────────────\n"
|
||||
f" Audio rms {rms:5.2f} bpm {bpm:5.1f} beat {beat:>4}\n"
|
||||
f" Pont {bridge_label} ({bridge_age:4.1f}s)\n"
|
||||
f"────────────────────────────────────────\n"
|
||||
f" Soleil Kp {kp:4.1f} {bar(kp_norm)} → palette orage\n"
|
||||
f" éruption {flare:4.2f} {bar(flare)} → flash orange\n"
|
||||
f" vent {wind:4.0f} {bar(wind_norm)} → vitesse fond\n"
|
||||
f" Bz {bz:+5.1f} {bar((bz_norm+1)/2)} → reverb\n"
|
||||
f" Réseau Δf {netz:+.3f}Hz {bar(abs(netz)*10)} → vibrato\n"
|
||||
f" Foudre {lr:4.1f}/min → percussions\n"
|
||||
f" Séisme M {quake:4.1f} → sub-bass\n"
|
||||
f" Avions {planes:>4} → voix FM\n"
|
||||
f" Social {social:4.1f}/s → kick\n"
|
||||
f" Pose n={pose_n:>2} live={pose_label} → squelette\n"
|
||||
)
|
||||
self._hud.setString_(txt)
|
||||
|
||||
def autoOpenpos_(self, _timer): # noqa: N802
|
||||
"""Si des personnes sont detectees ET l'utilisateur n'a pas pose
|
||||
un mode au clavier dans les 8 dernieres secondes, on passe
|
||||
automatiquement en mode openpos (#9) pour mettre la pose en
|
||||
valeur. Lache la main si lock recent ou si plus personne."""
|
||||
import time as _t
|
||||
s = self._state
|
||||
with s.lock():
|
||||
has_persons = bool(s.persons_body)
|
||||
mode = s.viz_mode
|
||||
now = _t.monotonic()
|
||||
if (now - self._user_viz_lock_t) < 8.0:
|
||||
return # lock utilisateur recent
|
||||
if has_persons and mode != 9:
|
||||
with s.lock():
|
||||
s.viz_mode = 9
|
||||
LOG.info("[auto] openpos engaged (persons detectees)")
|
||||
elif not has_persons and mode == 9:
|
||||
# plus de personne, on revient au mode storm par defaut
|
||||
with s.lock():
|
||||
s.viz_mode = 0
|
||||
LOG.info("[auto] storm engaged (plus de pose)")
|
||||
|
||||
def _on_key_global(self, ev):
|
||||
# Global monitor : read-only, on appelle _on_key mais on ne
|
||||
# retourne pas l'event (interdit par AppKit pour les globaux).
|
||||
try:
|
||||
self._on_key(ev)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("global key handler failed: %s", e)
|
||||
|
||||
def _on_key(self, ev):
|
||||
key = ev.charactersIgnoringModifiers()
|
||||
if not key:
|
||||
return ev
|
||||
k = key.lower()
|
||||
# Debug : log toutes les touches recues pour diagnostic
|
||||
LOG.debug("[key] raw=%r lower=%r", key, k)
|
||||
if key == "\x1b":
|
||||
NSApp().terminate_(self); return None
|
||||
if key == " ":
|
||||
self._scClient.send_message("/control/doScene", ["stop"])
|
||||
return None
|
||||
# Cmd+F geree par macOS pour fullscreen ; on garde shift+F en raccourci
|
||||
if key == "F":
|
||||
self._window.toggleFullScreen_(None); return None
|
||||
if key == "H":
|
||||
self._hud.setHidden_(not self._hud.isHidden()); return None
|
||||
if key == "C": # Shift+C : toggle webcam overlay
|
||||
self._cam.setHidden_(not self._cam.isHidden()); return None
|
||||
# azertyuiop -> video (viz mode)
|
||||
for kk, name in KEYMAP_VIDEO:
|
||||
if k == kk:
|
||||
names = list(self._state.viz_mode_names)
|
||||
if name in names:
|
||||
idx = names.index(name)
|
||||
with self._state.lock():
|
||||
self._state.viz_mode = idx
|
||||
# Lock l'auto-openpos pendant 8s : l'utilisateur a
|
||||
# explicitement choisi un mode, on respecte.
|
||||
import time as _t
|
||||
self._user_viz_lock_t = _t.monotonic()
|
||||
LOG.info("[video] viz -> %s (%d) (lock 8s)", name, idx)
|
||||
return None
|
||||
# qsdfghjklm -> audio (scene SC)
|
||||
for kk, scene in KEYMAP_AUDIO:
|
||||
if k == kk:
|
||||
self._scClient.send_message("/control/doScene", [scene])
|
||||
with self._state.lock():
|
||||
self._state.active_scene = scene
|
||||
LOG.info("[audio] scene -> %s", scene)
|
||||
return None
|
||||
# wxcvbn + 0-9 -> preset bundle (source + scene audio + viz video)
|
||||
for kk, source, scene, viz in (*KEYMAP_SOURCE, *KEYMAP_SOURCE_NUM):
|
||||
if key == kk or k == kk:
|
||||
# Audio : envoie a sclang
|
||||
self._scClient.send_message("/control/doScene", [scene])
|
||||
# Video : applique le viz mode
|
||||
names = list(self._state.viz_mode_names)
|
||||
idx = names.index(viz) if viz in names else 0
|
||||
with self._state.lock():
|
||||
self._state.viz_mode = idx
|
||||
self._state.active_preset = source
|
||||
self._state.active_scene = scene
|
||||
LOG.info("[preset] %s : scene=%s viz=%s", source, scene, viz)
|
||||
return None
|
||||
return ev
|
||||
|
||||
def applicationWillTerminate_(self, _): # noqa: N802
|
||||
self._listener.stop()
|
||||
if self._pose_worker is not None:
|
||||
self._pose_worker.stop()
|
||||
LOG.info("bye")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(prog="data_only_viz")
|
||||
p.add_argument("--port", type=int, default=57123,
|
||||
help="UDP port OSC entrant (default 57123)")
|
||||
p.add_argument("--sclang-port", type=int, default=57121,
|
||||
help="UDP port SC sortant pour /control/doScene (default 57121)")
|
||||
p.add_argument("--width", type=int, default=1280)
|
||||
p.add_argument("--height", type=int, default=720)
|
||||
p.add_argument("--fullscreen", action="store_true")
|
||||
p.add_argument("--pose", action="store_true",
|
||||
help="Active la captation pose YOLO (cv2 + ultralytics)")
|
||||
p.add_argument("--multi-hmr", dest="multi_hmr", action="store_true",
|
||||
help="Active Multi-HMR worker pour mesh SMPL-X dense "
|
||||
"(necessite setup_multihmr.sh + SMPLX_NEUTRAL.npz)")
|
||||
p.add_argument("--camera-index", dest="camera_index", type=int,
|
||||
default=-1,
|
||||
help="Index camera OpenCV (-1 = auto built-in Mac)")
|
||||
p.add_argument("--pose-device", default="mps",
|
||||
choices=("cpu", "mps", "cuda:0"),
|
||||
help="Device YOLO inference (default mps)")
|
||||
p.add_argument("--det-thresh", dest="det_thresh", type=float,
|
||||
default=0.15,
|
||||
help="Multi-HMR detection threshold (default 0.15)")
|
||||
p.add_argument("--nms-kernel-size", dest="nms_kernel_size", type=int,
|
||||
default=5,
|
||||
help="Multi-HMR NMS kernel (odd, >=3; default 5)")
|
||||
p.add_argument("--motion-gate", dest="motion_gate", type=float,
|
||||
default=5.0,
|
||||
help="Skip Multi-HMR si diff caméra <X (0-255 ; "
|
||||
"0=desactive ; default 5.0)")
|
||||
p.add_argument("--detrpose-model-size",
|
||||
choices=["n", "s", "l"],
|
||||
default="n",
|
||||
help="DETRPose model size (default: n)")
|
||||
p.add_argument("-v", "--verbose", action="store_true")
|
||||
opts = p.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if opts.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s — %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
app = NSApplication.sharedApplication()
|
||||
app.setActivationPolicy_(NSApplicationActivationPolicyRegular)
|
||||
delegate = AppDelegate.alloc().initWithOpts_(opts)
|
||||
app.setDelegate_(delegate)
|
||||
|
||||
# Ctrl-C en console termine proprement
|
||||
signal.signal(signal.SIGINT, lambda *_: app.terminate_(None))
|
||||
|
||||
app.run()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,435 @@
|
||||
"""Mesh rigging hybride keyframe (Multi-HMR) + delta Apple Vision.
|
||||
|
||||
Multi-HMR produit un mesh SMPL-X dense (10475 verts) tous les ~300 ms
|
||||
sur M5 (PyTorch MPS ~3.5 fps). Entre deux keyframes, Apple Vision sur
|
||||
ANE produit 30 fps de body keypoints 2D. On exploite le pelvis 2D de
|
||||
Vision pour translater rigidement le mesh keyframe et donner une
|
||||
perception fluide a 30 fps cote launcher RealityKit.
|
||||
|
||||
Limitations connues (premiere iteration) :
|
||||
- Translation rigide uniquement (pas de rotation, pas de LBS articule)
|
||||
- Pelvis 2D delta projete en X/Y a profondeur constante (z keyframe)
|
||||
- Pas de matching d'identite Vision <-> Multi-HMR : on prend la
|
||||
personne Vision la plus proche du pelvis projete keyframe
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import logging
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
_HAVE_SCIPY = True
|
||||
except ImportError: # noqa: BLE001
|
||||
_HAVE_SCIPY = False
|
||||
|
||||
from .state import PoseKp, SMPLXPerson, State
|
||||
|
||||
LOG = logging.getLogger("mesh_rigger")
|
||||
|
||||
|
||||
# Indices MediaPipe POSE_LANDMARKS pour les hanches (pelvis 2D = midpoint).
|
||||
_LEFT_HIP = 23
|
||||
_RIGHT_HIP = 24
|
||||
|
||||
# Focale par defaut Multi-HMR (camera intrinsics typiques utilisees
|
||||
# dans multi_hmr_worker : focal = IMG_SIZE).
|
||||
_IMG_SIZE = 672
|
||||
_FOCAL = float(_IMG_SIZE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Keyframe:
|
||||
"""Snapshot d'un mesh Multi-HMR + reference Vision au moment T."""
|
||||
pid: int
|
||||
t: float
|
||||
# Mesh world coords (10475, 3) float32 incluant la translation
|
||||
vertices_3d: np.ndarray
|
||||
translation: np.ndarray # (3,) world pelvis
|
||||
vision_pelvis_2d: tuple[float, float] | None # (cx, cy) normalises 0..1
|
||||
|
||||
|
||||
def _pelvis_2d_from_body(body: list[PoseKp]) -> tuple[float, float] | None:
|
||||
"""Midpoint des deux hanches MediaPipe si confidence > 0."""
|
||||
if not body or len(body) <= _RIGHT_HIP:
|
||||
return None
|
||||
lh, rh = body[_LEFT_HIP], body[_RIGHT_HIP]
|
||||
if lh.c <= 0.1 or rh.c <= 0.1:
|
||||
return None
|
||||
return (0.5 * (lh.x + rh.x), 0.5 * (lh.y + rh.y))
|
||||
|
||||
|
||||
def _body_bbox_norm(
|
||||
body: list[PoseKp],
|
||||
) -> tuple[float, float, float, float] | None:
|
||||
"""Bbox image-normalized [0,1] from a list of body landmarks
|
||||
(Vision 19 joints OR MediaPipe 33). None if not enough confident
|
||||
points."""
|
||||
if not body:
|
||||
return None
|
||||
xs = [kp.x for kp in body if kp.c > 0.05]
|
||||
ys = [kp.y for kp in body if kp.c > 0.05]
|
||||
if len(xs) < 4 or len(ys) < 4:
|
||||
return None
|
||||
x0, x1 = max(0.0, min(xs)), min(1.0, max(xs))
|
||||
y0, y1 = max(0.0, min(ys)), min(1.0, max(ys))
|
||||
# Pad 10% to capture full body silhouette.
|
||||
dx = (x1 - x0) * 0.10
|
||||
dy = (y1 - y0) * 0.10
|
||||
x0 = max(0.0, x0 - dx); x1 = min(1.0, x1 + dx)
|
||||
y0 = max(0.0, y0 - dy); y1 = min(1.0, y1 + dy)
|
||||
if x1 - x0 < 0.02 or y1 - y0 < 0.02:
|
||||
return None
|
||||
return (x0, y0, x1, y1)
|
||||
|
||||
|
||||
def _mesh_bbox_norm(p: SMPLXPerson) -> tuple[float, float, float, float] | None:
|
||||
"""Project SMPL-X mesh vertices to image-normalized bbox.
|
||||
|
||||
Multi-HMR uses focal = IMG_SIZE camera intrinsics. World verts
|
||||
have z>0 (in front of camera)."""
|
||||
v = np.asarray(p.vertices_3d, dtype=np.float32)
|
||||
if v.size == 0 or v.shape[0] < 100:
|
||||
return None
|
||||
z = v[:, 2]
|
||||
valid = z > 1e-3
|
||||
if not np.any(valid):
|
||||
return None
|
||||
x_img = (v[valid, 0] * _FOCAL / z[valid]) / _IMG_SIZE + 0.5
|
||||
y_img = (v[valid, 1] * _FOCAL / z[valid]) / _IMG_SIZE + 0.5
|
||||
x0, x1 = float(x_img.min()), float(x_img.max())
|
||||
y0, y1 = float(y_img.min()), float(y_img.max())
|
||||
x0 = max(0.0, x0); x1 = min(1.0, x1)
|
||||
y0 = max(0.0, y0); y1 = min(1.0, y1)
|
||||
if x1 - x0 < 0.02 or y1 - y0 < 0.02:
|
||||
return None
|
||||
return (x0, y0, x1, y1)
|
||||
|
||||
|
||||
def _iou_norm(
|
||||
a: tuple[float, float, float, float],
|
||||
b: tuple[float, float, float, float],
|
||||
) -> float:
|
||||
ax0, ay0, ax1, ay1 = a
|
||||
bx0, by0, bx1, by1 = b
|
||||
ix0 = max(ax0, bx0); iy0 = max(ay0, by0)
|
||||
ix1 = min(ax1, bx1); iy1 = min(ay1, by1)
|
||||
iw = max(0.0, ix1 - ix0); ih = max(0.0, iy1 - iy0)
|
||||
inter = iw * ih
|
||||
if inter <= 0:
|
||||
return 0.0
|
||||
a_area = (ax1 - ax0) * (ay1 - ay0)
|
||||
b_area = (bx1 - bx0) * (by1 - by0)
|
||||
return float(inter / (a_area + b_area - inter + 1e-9))
|
||||
|
||||
|
||||
def _vision_pid_match(
|
||||
keyframe_pelvis_2d: tuple[float, float] | None,
|
||||
vision_bodies: list[list[PoseKp]],
|
||||
vision_ids: list[int],
|
||||
) -> int | None:
|
||||
"""Retourne le pid Vision dont le pelvis 2D est le plus proche du
|
||||
keyframe pelvis projete. None si rien."""
|
||||
if keyframe_pelvis_2d is None or not vision_bodies:
|
||||
return None
|
||||
kx, ky = keyframe_pelvis_2d
|
||||
best_pid: int | None = None
|
||||
best_d2 = float("inf")
|
||||
for body, vpid in zip(vision_bodies, vision_ids):
|
||||
p = _pelvis_2d_from_body(body)
|
||||
if p is None:
|
||||
continue
|
||||
d2 = (p[0] - kx) ** 2 + (p[1] - ky) ** 2
|
||||
if d2 < best_d2:
|
||||
best_d2 = d2
|
||||
best_pid = int(vpid)
|
||||
return best_pid
|
||||
|
||||
|
||||
class MeshRigger:
|
||||
"""Rig le mesh SMPL-X keyframe via le delta pelvis Vision.
|
||||
|
||||
Usage :
|
||||
rigger = MeshRigger(state)
|
||||
rigged_persons = rigger.apply(state.persons_smplx,
|
||||
state.persons_body,
|
||||
t_now)
|
||||
Thread-safe : ne mute pas le state, retourne une nouvelle liste.
|
||||
"""
|
||||
|
||||
def __init__(self, state: State, hold_window_s: float = 1.5,
|
||||
dino_weight: float = 0.5,
|
||||
dino_reid=None) -> None:
|
||||
self.state = state
|
||||
self.hold_window_s = hold_window_s
|
||||
self.dino_weight = float(dino_weight)
|
||||
self.dino_reid = dino_reid
|
||||
self._lock = threading.Lock()
|
||||
# pid Multi-HMR -> keyframe
|
||||
self._keyframes: dict[int, _Keyframe] = {}
|
||||
# pid Multi-HMR -> pid Vision matched (sticky across frames)
|
||||
self._vision_pid_map: dict[int, int] = {}
|
||||
# pid Multi-HMR -> recent DINO embeddings (mean -> reid signature)
|
||||
self._pid_embeddings: dict[int, collections.deque] = {}
|
||||
# Cached log throttle
|
||||
self._next_dino_log = 0.0
|
||||
|
||||
def apply(
|
||||
self,
|
||||
persons_smplx: list[SMPLXPerson],
|
||||
persons_body: list[list[PoseKp]],
|
||||
persons_body_ids: list[int],
|
||||
t_now: float,
|
||||
) -> list[SMPLXPerson]:
|
||||
"""Retourne une liste SMPLXPerson translatee par delta Vision."""
|
||||
# 1) Detect new keyframes (timestamp tracked via state.smplx_last_t)
|
||||
with self._lock:
|
||||
current_pids = {p.pid for p in persons_smplx}
|
||||
# Drop stale keyframes (person disparue)
|
||||
for old_pid in list(self._keyframes):
|
||||
if old_pid not in current_pids:
|
||||
self._keyframes.pop(old_pid, None)
|
||||
self._vision_pid_map.pop(old_pid, None)
|
||||
self._pid_embeddings.pop(old_pid, None)
|
||||
|
||||
# 2) DINO fusion: if a reid backend is wired, try Hungarian
|
||||
# over (mesh keyframe pids) x (Vision body pids) using
|
||||
# alpha*IoU + (1-alpha)*cosine. This only kicks in when a
|
||||
# keyframe is detected this call AND we have an RGB frame.
|
||||
self._dino_match(persons_smplx, persons_body,
|
||||
persons_body_ids)
|
||||
|
||||
out: list[SMPLXPerson] = []
|
||||
for person in persons_smplx:
|
||||
kf = self._keyframes.get(person.pid)
|
||||
# Detect keyframe refresh : translation differs from kf
|
||||
is_new_kf = (kf is None or not np.allclose(
|
||||
kf.translation, person.translation, atol=1e-4))
|
||||
if is_new_kf:
|
||||
# Trouver le pid Vision le plus proche pour ce mesh.
|
||||
# On projette le pelvis world en 2D image-normalized :
|
||||
# x_img = (X / Z) * focal / IMG_SIZE + 0.5
|
||||
pelvis_2d = self._project_pelvis(person.translation)
|
||||
matched = _vision_pid_match(
|
||||
pelvis_2d, persons_body, persons_body_ids)
|
||||
if matched is None:
|
||||
matched = self._vision_pid_map.get(person.pid)
|
||||
if matched is not None:
|
||||
self._vision_pid_map[person.pid] = matched
|
||||
# Capture du pelvis 2D Vision au moment du keyframe
|
||||
vp = None
|
||||
if matched is not None:
|
||||
try:
|
||||
i = persons_body_ids.index(matched)
|
||||
vp = _pelvis_2d_from_body(persons_body[i])
|
||||
except (ValueError, IndexError):
|
||||
vp = None
|
||||
self._keyframes[person.pid] = _Keyframe(
|
||||
pid=person.pid,
|
||||
t=t_now,
|
||||
vertices_3d=person.vertices_3d.copy(),
|
||||
translation=person.translation.copy(),
|
||||
vision_pelvis_2d=vp,
|
||||
)
|
||||
out.append(person)
|
||||
continue
|
||||
|
||||
# Entre keyframes : applique delta translation depuis
|
||||
# Vision pelvis 2D actuel vs keyframe pelvis 2D.
|
||||
if t_now - kf.t > self.hold_window_s:
|
||||
# Trop ancien, on lache le rig (mesh statique)
|
||||
out.append(person)
|
||||
continue
|
||||
matched_pid = self._vision_pid_map.get(person.pid)
|
||||
if matched_pid is None or kf.vision_pelvis_2d is None:
|
||||
out.append(person)
|
||||
continue
|
||||
try:
|
||||
i = persons_body_ids.index(matched_pid)
|
||||
except ValueError:
|
||||
out.append(person)
|
||||
continue
|
||||
current_vp = _pelvis_2d_from_body(persons_body[i])
|
||||
if current_vp is None:
|
||||
out.append(person)
|
||||
continue
|
||||
|
||||
# Image-normalized 2D delta -> world XY delta a depth z_kf.
|
||||
# Pour un pelvis aux coords image (px in [0,1] centre 0.5),
|
||||
# X_world = (px - 0.5) * IMG_SIZE * Z / focal = (px-0.5)*Z
|
||||
# (focal=IMG_SIZE). Delta image -> Delta world a Z fixe.
|
||||
z_kf = float(kf.translation[2]) if abs(
|
||||
kf.translation[2]) > 1e-3 else 1.0
|
||||
dx_img = current_vp[0] - kf.vision_pelvis_2d[0]
|
||||
dy_img = current_vp[1] - kf.vision_pelvis_2d[1]
|
||||
dx_world = dx_img * _IMG_SIZE * z_kf / _FOCAL
|
||||
dy_world = dy_img * _IMG_SIZE * z_kf / _FOCAL
|
||||
|
||||
# Applique a tous les vertices + a translation.
|
||||
new_verts = kf.vertices_3d.copy()
|
||||
new_verts[:, 0] += np.float32(dx_world)
|
||||
new_verts[:, 1] += np.float32(dy_world)
|
||||
new_transl = kf.translation.copy()
|
||||
new_transl[0] += np.float32(dx_world)
|
||||
new_transl[1] += np.float32(dy_world)
|
||||
|
||||
out.append(SMPLXPerson(
|
||||
pid=person.pid,
|
||||
vertices_3d=new_verts,
|
||||
translation=new_transl,
|
||||
confidence=person.confidence,
|
||||
betas=person.betas,
|
||||
expression=person.expression,
|
||||
))
|
||||
return out
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# DINOv2 reid hooks
|
||||
# ------------------------------------------------------------------
|
||||
def _dino_match(
|
||||
self,
|
||||
persons_smplx: list[SMPLXPerson],
|
||||
persons_body: list[list[PoseKp]],
|
||||
persons_body_ids: list[int],
|
||||
) -> None:
|
||||
"""Update self._vision_pid_map and self._pid_embeddings by
|
||||
matching mesh pids against Vision pids on alpha*IoU +
|
||||
(1-alpha)*DINO cosine. No-op if any prerequisite missing.
|
||||
|
||||
Caller must hold self._lock."""
|
||||
if self.dino_reid is None or not _HAVE_SCIPY:
|
||||
return
|
||||
if not persons_smplx or not persons_body:
|
||||
return
|
||||
# Need at least one new keyframe to be worth running DINO.
|
||||
new_kf_pids: list[int] = []
|
||||
for p in persons_smplx:
|
||||
kf = self._keyframes.get(p.pid)
|
||||
if kf is None or not np.allclose(
|
||||
kf.translation, p.translation, atol=1e-4):
|
||||
new_kf_pids.append(int(p.pid))
|
||||
if not new_kf_pids:
|
||||
return
|
||||
|
||||
# Acquire current RGB frame (best effort, no double lock).
|
||||
frame = self.state.last_frame_rgb
|
||||
if frame is None:
|
||||
return
|
||||
H, W = frame.shape[:2]
|
||||
|
||||
# Build Vision bboxes (image-normalized) and pixel crops.
|
||||
v_bboxes_norm: list[tuple[float, float, float, float]] = []
|
||||
v_crops: list[np.ndarray] = []
|
||||
v_pids: list[int] = []
|
||||
for body, vpid in zip(persons_body, persons_body_ids):
|
||||
bb = _body_bbox_norm(body)
|
||||
if bb is None:
|
||||
continue
|
||||
x0, y0, x1, y1 = bb
|
||||
px0 = max(0, int(x0 * W))
|
||||
py0 = max(0, int(y0 * H))
|
||||
px1 = min(W, int(x1 * W))
|
||||
py1 = min(H, int(y1 * H))
|
||||
if px1 <= px0 + 4 or py1 <= py0 + 4:
|
||||
continue
|
||||
v_bboxes_norm.append(bb)
|
||||
v_crops.append(frame[py0:py1, px0:px1].copy())
|
||||
v_pids.append(int(vpid))
|
||||
|
||||
if not v_crops:
|
||||
return
|
||||
|
||||
# Build mesh bboxes (image-normalized) from world pelvis proj.
|
||||
m_bboxes_norm: list[tuple[float, float, float, float]] = []
|
||||
m_pids_keep: list[int] = []
|
||||
m_crops: list[np.ndarray] = []
|
||||
for p in persons_smplx:
|
||||
bb = _mesh_bbox_norm(p)
|
||||
if bb is None:
|
||||
continue
|
||||
m_bboxes_norm.append(bb)
|
||||
m_pids_keep.append(int(p.pid))
|
||||
x0, y0, x1, y1 = bb
|
||||
px0 = max(0, int(x0 * W))
|
||||
py0 = max(0, int(y0 * H))
|
||||
px1 = min(W, int(x1 * W))
|
||||
py1 = min(H, int(y1 * H))
|
||||
if px1 > px0 + 4 and py1 > py0 + 4:
|
||||
m_crops.append(frame[py0:py1, px0:px1].copy())
|
||||
else:
|
||||
m_crops.append(None) # type: ignore[arg-type]
|
||||
|
||||
if not m_bboxes_norm:
|
||||
return
|
||||
|
||||
# Embed Vision crops in one batch (still loops internally).
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
v_emb = self.dino_reid.embed_crops(v_crops)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("dino_reid embed failed: %s", e)
|
||||
return
|
||||
|
||||
# Build cost matrix mesh x vision : 1 - (alpha*IoU + (1-alpha)*cos)
|
||||
n_m = len(m_bboxes_norm)
|
||||
n_v = len(v_bboxes_norm)
|
||||
alpha = float(np.clip(self.dino_weight, 0.0, 1.0))
|
||||
cost = np.ones((n_m, n_v), dtype=np.float32)
|
||||
for i, mbb in enumerate(m_bboxes_norm):
|
||||
hist = self._pid_embeddings.get(m_pids_keep[i])
|
||||
mean_emb = None
|
||||
if hist:
|
||||
stack = np.stack(list(hist), axis=0)
|
||||
mean_emb = stack.mean(axis=0)
|
||||
n = np.linalg.norm(mean_emb) + 1e-8
|
||||
mean_emb = mean_emb / n
|
||||
for j, vbb in enumerate(v_bboxes_norm):
|
||||
iou = _iou_norm(mbb, vbb)
|
||||
if mean_emb is not None:
|
||||
cos = float(np.dot(mean_emb, v_emb[j]))
|
||||
else:
|
||||
cos = iou # no history -> trust IoU
|
||||
score = alpha * iou + (1.0 - alpha) * max(0.0, cos)
|
||||
cost[i, j] = 1.0 - score
|
||||
|
||||
rr, cc = linear_sum_assignment(cost)
|
||||
for i, j in zip(rr, cc):
|
||||
if cost[i, j] >= 0.95:
|
||||
continue # weak match, ignore
|
||||
mpid = m_pids_keep[i]
|
||||
self._vision_pid_map[mpid] = v_pids[j]
|
||||
# Update embedding history for THIS mesh pid using the
|
||||
# Vision crop (most recent visual evidence).
|
||||
dq = self._pid_embeddings.setdefault(
|
||||
mpid, collections.deque(maxlen=10))
|
||||
dq.append(v_emb[j].copy())
|
||||
|
||||
now = time.monotonic()
|
||||
dt_ms = (time.perf_counter() - t0) * 1e3
|
||||
if now >= self._next_dino_log:
|
||||
LOG.info(
|
||||
"dino_reid: embedded %d crops in %.1f ms (alpha=%.2f, "
|
||||
"matched %d mesh<->vision pairs)",
|
||||
len(v_crops), dt_ms, alpha, min(n_m, n_v))
|
||||
self._next_dino_log = now + 5.0
|
||||
|
||||
@staticmethod
|
||||
def _project_pelvis(
|
||||
translation: np.ndarray,
|
||||
) -> tuple[float, float] | None:
|
||||
"""World pelvis (X,Y,Z) -> image-normalized 2D pelvis."""
|
||||
z = float(translation[2])
|
||||
if abs(z) < 1e-3:
|
||||
return None
|
||||
x_img = (float(translation[0]) * _FOCAL / z) / _IMG_SIZE + 0.5
|
||||
y_img = (float(translation[1]) * _FOCAL / z) / _IMG_SIZE + 0.5
|
||||
# Clamp en [0,1]
|
||||
if not (0.0 <= x_img <= 1.0 and 0.0 <= y_img <= 1.0):
|
||||
return None
|
||||
return (x_img, y_img)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Topologie de triangles pour le rendu mesh face/main/corps.
|
||||
|
||||
Trois listes statiques d'indices :
|
||||
- FACE_TRIANGLES : visage Apple Vision (~76 landmarks, layout plat dans
|
||||
state.persons_face[i]). Generee dynamiquement via scipy.Delaunay au
|
||||
premier frame (cache global). Voir build_face_triangles_dynamic().
|
||||
- HAND_TRIANGLES : 21 landmarks main (paume fan + strips doigts).
|
||||
- BODY_TRIANGLES : 33 landmarks MediaPipe POSE_LANDMARKS (tronc, bras,
|
||||
jambes, tete) ; reutilise tel quel pour Apple Vision body 17 kp
|
||||
mappes sur les memes 33 indices.
|
||||
|
||||
Format : list[tuple[int, int, int]] (a, b, c) indices dans la liste de
|
||||
keypoints correspondante.
|
||||
|
||||
Convention Apple Vision FaceLandmarks2D — offsets par region tels
|
||||
qu'ecrits par apple_vision_pose._parse_face_observation() :
|
||||
contour : 0..16 (17 pts, faceContour)
|
||||
left_eye : 17..24 (8 pts)
|
||||
right_eye : 25..32 (8 pts)
|
||||
left_brow : 33..38 (6 pts, leftEyebrow)
|
||||
right_brow : 39..44 (6 pts, rightEyebrow)
|
||||
outer_lips : 45..58 (14 pts, outerLips)
|
||||
inner_lips : 59..68 (10 pts, innerLips)
|
||||
nose : 69..74 (6 pts, nose)
|
||||
median : 75..80 (6 pts, medianLine — optionnel)
|
||||
pupils : 81..82 (2 pts, leftPupil rightPupil)
|
||||
|
||||
Le total exact varie selon macOS ; on cale 76 indices visibles
|
||||
generalement, le reste est ignore. La triangulation dynamique
|
||||
Delaunay s'adapte automatiquement.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence
|
||||
|
||||
FACE_OFFSETS: dict[str, tuple[int, int]] = {
|
||||
"contour": (0, 17),
|
||||
"left_eye": (17, 25),
|
||||
"right_eye": (25, 33),
|
||||
"left_brow": (33, 39),
|
||||
"right_brow": (39, 45),
|
||||
"outer_lips": (45, 59),
|
||||
"inner_lips": (59, 69),
|
||||
"nose": (69, 75),
|
||||
"median": (75, 81),
|
||||
"pupils": (81, 83),
|
||||
}
|
||||
FACE_MAX_LANDMARKS = 83
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HAND_TRIANGLES — 21 landmarks (standard MediaPipe / Apple Vision)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Indices : 0=wrist, 1..4=thumb, 5..8=index, 9..12=middle, 13..16=ring,
|
||||
# 17..20=little. Chaque doigt : MCP, PIP, DIP, TIP.
|
||||
HAND_TRIANGLES: list[tuple[int, int, int]] = [
|
||||
# Paume : fan depuis le poignet vers les bases des doigts
|
||||
(0, 1, 5),
|
||||
(0, 5, 9),
|
||||
(0, 9, 13),
|
||||
(0, 13, 17),
|
||||
# Pouce — strip (segments 1-2-3-4)
|
||||
(1, 2, 5), # base pouce -> index
|
||||
(2, 3, 5),
|
||||
# Index : segments 5->6->7->8 (strip avec voisin middle pour epaisseur)
|
||||
(5, 6, 9),
|
||||
(6, 7, 9),
|
||||
(7, 8, 9),
|
||||
# Middle : 9->10->11->12 (strip avec ring)
|
||||
(9, 10, 13),
|
||||
(10, 11, 13),
|
||||
(11, 12, 13),
|
||||
# Ring : 13->14->15->16 (strip avec little)
|
||||
(13, 14, 17),
|
||||
(14, 15, 17),
|
||||
(15, 16, 17),
|
||||
# Little : 17->18->19->20 — degenere en triangle avec le poignet
|
||||
(17, 18, 0),
|
||||
(18, 19, 17),
|
||||
(19, 20, 17),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BODY_TRIANGLES — 33 landmarks MediaPipe POSE_LANDMARKS
|
||||
# ---------------------------------------------------------------------------
|
||||
# Indices cles (MediaPipe) :
|
||||
# 0 nose
|
||||
# 7 left_ear 8 right_ear
|
||||
# 11 left_shoulder 12 right_shoulder
|
||||
# 13 left_elbow 14 right_elbow
|
||||
# 15 left_wrist 16 right_wrist
|
||||
# 23 left_hip 24 right_hip
|
||||
# 25 left_knee 26 right_knee
|
||||
# 27 left_ankle 28 right_ankle
|
||||
BODY_TRIANGLES: list[tuple[int, int, int]] = [
|
||||
# Cou + tete : nez + epaules
|
||||
(0, 11, 12),
|
||||
# Tronc QUAD divise en 4 triangles (mesh plus dense)
|
||||
(11, 12, 24),
|
||||
(11, 24, 23),
|
||||
(11, 12, 23),
|
||||
(12, 23, 24),
|
||||
# Bras gauche : triangles avant + face inverse (double face = visible cote-cote)
|
||||
(11, 13, 15),
|
||||
(11, 15, 13),
|
||||
# Bras droit
|
||||
(12, 14, 16),
|
||||
(12, 16, 14),
|
||||
# Jambe gauche : hip-knee-ankle + inverse
|
||||
(23, 25, 27),
|
||||
(23, 27, 25),
|
||||
# Jambe droite
|
||||
(24, 26, 28),
|
||||
(24, 28, 26),
|
||||
# Mailler le tronc avec les bras/jambes pour relier
|
||||
(11, 23, 13), # epaule-hanche-coude G
|
||||
(12, 24, 14), # epaule-hanche-coude D
|
||||
(23, 13, 25), # cuisse-haut au coude (croise)
|
||||
(24, 14, 26),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FACE_TRIANGLES — triangulation Delaunay dynamique cachee
|
||||
# ---------------------------------------------------------------------------
|
||||
# On ne hardcode pas car le nombre de landmarks Apple Vision face varie
|
||||
# entre versions macOS. Au premier frame, on calcule la triangulation 2D
|
||||
# Delaunay sur la liste plate des landmarks valides, puis on cache la
|
||||
# liste d'indices tant que la cardinalite ne change pas.
|
||||
|
||||
_FACE_TRI_CACHE: dict[int, list[tuple[int, int, int]]] = {}
|
||||
|
||||
|
||||
def build_face_triangles_dynamic(
|
||||
points_xy: Sequence[tuple[float, float]],
|
||||
) -> list[tuple[int, int, int]]:
|
||||
"""Triangulation Delaunay 2D des landmarks face. Cachee par cardinalite.
|
||||
|
||||
points_xy : liste plate de (x, y) normalises (longueur = N landmarks).
|
||||
Retourne : list[(i, j, k)] indices dans la liste d'entree.
|
||||
|
||||
Si scipy indisponible ou triangulation echoue, retourne [].
|
||||
"""
|
||||
n = len(points_xy)
|
||||
if n < 4:
|
||||
return []
|
||||
if n in _FACE_TRI_CACHE:
|
||||
return _FACE_TRI_CACHE[n]
|
||||
try:
|
||||
import numpy as np
|
||||
from scipy.spatial import Delaunay
|
||||
pts = np.asarray(points_xy, dtype=np.float32)
|
||||
# Filtre les points invalides (0,0) si presents
|
||||
valid = (pts[:, 0] > 0.0) | (pts[:, 1] > 0.0)
|
||||
if valid.sum() < 4:
|
||||
return []
|
||||
# On triangule sur tous les points (l'indice reste valide) mais on
|
||||
# filtre les triangles qui touchent un point invalide en aval.
|
||||
tri = Delaunay(pts).simplices
|
||||
triangles = [tuple(int(v) for v in t) for t in tri]
|
||||
except Exception:
|
||||
triangles = []
|
||||
_FACE_TRI_CACHE[n] = triangles
|
||||
return triangles
|
||||
|
||||
|
||||
# Triangulation par defaut : conservee comme fallback si Delaunay echoue.
|
||||
# Quelques triangles symboliques sur le visage minimum (contour + nez +
|
||||
# bouche) qui couvrent les regions critiques.
|
||||
FACE_TRIANGLES: list[tuple[int, int, int]] = [
|
||||
# Fan partiel sur le contour (8 triangles : 0..16 -> centre approx = 71 nose)
|
||||
(0, 1, 71), (1, 2, 71), (2, 3, 71), (3, 4, 71),
|
||||
(4, 5, 71), (5, 6, 71), (6, 7, 71), (7, 8, 71),
|
||||
(8, 9, 71), (9, 10, 71), (10, 11, 71), (11, 12, 71),
|
||||
(12, 13, 71), (13, 14, 71), (14, 15, 71), (15, 16, 71),
|
||||
# outerLips fan (45..58 -> centre 60)
|
||||
(45, 46, 60), (46, 47, 60), (47, 48, 60), (48, 49, 60),
|
||||
(49, 50, 60), (50, 51, 60), (51, 52, 60), (52, 53, 60),
|
||||
(53, 54, 60), (54, 55, 60), (55, 56, 60), (56, 57, 60),
|
||||
(57, 58, 60),
|
||||
# innerLips fan (59..68 -> centre 64)
|
||||
(59, 60, 64), (60, 61, 64), (61, 62, 64), (62, 63, 64),
|
||||
(65, 66, 64), (66, 67, 64), (67, 68, 64),
|
||||
]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FACE_OFFSETS",
|
||||
"FACE_MAX_LANDMARKS",
|
||||
"FACE_TRIANGLES",
|
||||
"HAND_TRIANGLES",
|
||||
"BODY_TRIANGLES",
|
||||
"build_face_triangles_dynamic",
|
||||
]
|
||||
@@ -0,0 +1,508 @@
|
||||
"""Multi-personne : Pose+Face+Hand Landmarkers MediaPipe en parallele.
|
||||
|
||||
HolisticLandmarker est MONO-personne (par design). Pour multi-personnes
|
||||
on utilise les 3 landmarkers spécialisés qui supportent `num_X=N` :
|
||||
- PoseLandmarker(num_poses=4)
|
||||
- FaceLandmarker(num_faces=4)
|
||||
- HandLandmarker(num_hands=8) (jusqu'a 4 personnes × 2 mains)
|
||||
|
||||
Chaque inference tourne sur la MEME frame webcam. Les resultats sont
|
||||
stockes independamment dans state.persons_body / persons_face /
|
||||
persons_hands. Le renderer dessine TOUS les segments de toutes les
|
||||
personnes, sans matching inter-modeles (acceptable visuellement).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from .action_head_pub import ActionHeadPublisher
|
||||
from .euro_filter import SkeletonFilter
|
||||
from .pose_bridge import PoseSoundBridge
|
||||
from .pose_filter import PoseFilterChain
|
||||
from .pose_filter import _is_finite # noqa: PLC2701 (intentional internal use)
|
||||
from .state import Kp3D, PoseKp, State
|
||||
from .tracker import IoUTracker
|
||||
|
||||
LOG = logging.getLogger("multi")
|
||||
|
||||
MODELS = {
|
||||
"pose": (
|
||||
"https://storage.googleapis.com/mediapipe-models/pose_landmarker/"
|
||||
"pose_landmarker_lite/float16/latest/pose_landmarker_lite.task"
|
||||
),
|
||||
"face": (
|
||||
"https://storage.googleapis.com/mediapipe-models/face_landmarker/"
|
||||
"face_landmarker/float16/latest/face_landmarker.task"
|
||||
),
|
||||
"hand": (
|
||||
"https://storage.googleapis.com/mediapipe-models/hand_landmarker/"
|
||||
"hand_landmarker/float16/latest/hand_landmarker.task"
|
||||
),
|
||||
}
|
||||
CACHE_DIR = Path.home() / ".cache" / "av-live-mediapipe"
|
||||
|
||||
|
||||
def _smooth_kps(skf: SkeletonFilter, pid: int, kps: list, t: float) -> list:
|
||||
"""Applique le One Euro filter sur chaque keypoint d'une personne."""
|
||||
if pid < 0:
|
||||
return kps # detection orpheline (sans track), pas de lissage
|
||||
out = []
|
||||
for k, kp in enumerate(kps):
|
||||
sx, sy, sz = skf.smooth(pid, k, kp.x, kp.y, kp.z, t)
|
||||
out.append(PoseKp(x=sx, y=sy, z=sz, c=kp.c))
|
||||
return out
|
||||
|
||||
|
||||
def _ensure_model(name: str) -> Path:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
path = CACHE_DIR / f"{name}_landmarker.task"
|
||||
if path.exists() and path.stat().st_size > 100_000:
|
||||
return path
|
||||
LOG.info("downloading %s model ...", name)
|
||||
urllib.request.urlretrieve(MODELS[name], path)
|
||||
LOG.info("%s OK (%d bytes)", name, path.stat().st_size)
|
||||
return path
|
||||
|
||||
|
||||
class MultiWorker:
|
||||
"""Worker multi-personne (pose + face + hands landmarkers paralleles)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state: State,
|
||||
camera_index: int = 0,
|
||||
target_fps: float = 18.0,
|
||||
num_persons: int = 4,
|
||||
min_conf: float = 0.4,
|
||||
) -> None:
|
||||
self.state = state
|
||||
self.camera_index = camera_index
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self.num_persons = num_persons
|
||||
self.min_conf = min_conf
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
# Lissage + tracking pour stabiliser les keypoints frame a frame
|
||||
# et garder des IDs de couleur persistants entre frames.
|
||||
self._tracker_body = IoUTracker(iou_threshold=0.20, max_miss=10)
|
||||
self._tracker_face = IoUTracker(iou_threshold=0.15, max_miss=10)
|
||||
self._tracker_hand = IoUTracker(iou_threshold=0.10, max_miss=6)
|
||||
self._smooth_body = SkeletonFilter(min_cutoff=1.2, beta=0.06)
|
||||
self._smooth_face = SkeletonFilter(min_cutoff=1.8, beta=0.04)
|
||||
self._smooth_hand = SkeletonFilter(min_cutoff=2.0, beta=0.10)
|
||||
# Pont OSC pose -> sclang
|
||||
self._sound_bridge = PoseSoundBridge(throttle_hz=30.0)
|
||||
self._action_pub = ActionHeadPublisher(state=self.state, bridge=self._sound_bridge)
|
||||
self._action_pub.start()
|
||||
# 3D pose filter chain : median, Kalman CV, lookahead, IK clamps.
|
||||
self._filter_chain = PoseFilterChain(state=self.state)
|
||||
# Discrimination state : per-pid frame counters for hysteresis.
|
||||
# _pid_lifetime : frames since pid created (visible).
|
||||
# _pid_last_bbox : last bbox seen for active pid (for re-association).
|
||||
# _pid_missing : frames since pid disappeared (None when active).
|
||||
self._pid_lifetime: dict[int, int] = {}
|
||||
self._pid_missing: dict[int, int] = {}
|
||||
self._pid_last_bbox: dict[int, tuple[float, float, float, float]] = {}
|
||||
# Discrimination thresholds — tunable via env.
|
||||
import os as _os
|
||||
self._ghost_min_visible = int(_os.environ.get("POSE_GHOST_MIN_VISIBLE", "10"))
|
||||
self._ghost_min_conf = float(_os.environ.get("POSE_GHOST_MIN_CONF", "0.5"))
|
||||
self._hand_min_visible = int(_os.environ.get("POSE_HAND_MIN_VISIBLE", "15"))
|
||||
self._face_min_visible = int(_os.environ.get("POSE_FACE_MIN_VISIBLE", "50"))
|
||||
self._nms_iou = float(_os.environ.get("POSE_NMS_IOU", "0.7"))
|
||||
# Counters exposed for debug.
|
||||
self._n_ghost_dropped = 0
|
||||
self._n_hand_dropped = 0
|
||||
self._n_face_dropped = 0
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Discrimination helpers — body ghost rejection, NMS, pid hysteresis,
|
||||
# face/hand visibility gates. All return filtered (kps, ids) lists.
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def _bbox_from_kps(kps: list) -> tuple[float, float, float, float]:
|
||||
if not kps:
|
||||
return (0.0, 0.0, 0.0, 0.0)
|
||||
xs = [kp.x for kp in kps]
|
||||
ys = [kp.y for kp in kps]
|
||||
return (min(xs), min(ys), max(xs), max(ys))
|
||||
|
||||
@staticmethod
|
||||
def _iou(a: tuple[float, float, float, float],
|
||||
b: tuple[float, float, float, float]) -> float:
|
||||
ix1 = max(a[0], b[0]); iy1 = max(a[1], b[1])
|
||||
ix2 = min(a[2], b[2]); iy2 = min(a[3], b[3])
|
||||
iw = max(0.0, ix2 - ix1); ih = max(0.0, iy2 - iy1)
|
||||
inter = iw * ih
|
||||
aw = max(0.0, a[2] - a[0]) * max(0.0, a[3] - a[1])
|
||||
bw = max(0.0, b[2] - b[0]) * max(0.0, b[3] - b[1])
|
||||
u = aw + bw - inter
|
||||
return inter / u if u > 1e-9 else 0.0
|
||||
|
||||
def _reject_ghosts_and_nms(
|
||||
self,
|
||||
bodies: list[list],
|
||||
bodies3d: list[list[Kp3D]],
|
||||
ids_body: list[int],
|
||||
) -> tuple[list[list], list[list[Kp3D]], list[int]]:
|
||||
"""Drop body detections with <N high-confidence joints, then NMS."""
|
||||
if not bodies:
|
||||
return bodies, bodies3d, ids_body
|
||||
# Score each body by mean confidence ; track visibility count.
|
||||
keep_mask = [True] * len(bodies)
|
||||
scores: list[float] = []
|
||||
for i, kps in enumerate(bodies):
|
||||
n_visible = sum(
|
||||
1 for kp in kps
|
||||
if kp.c >= self._ghost_min_conf
|
||||
and _is_finite(kp.x) and _is_finite(kp.y))
|
||||
if n_visible < self._ghost_min_visible:
|
||||
keep_mask[i] = False
|
||||
self._n_ghost_dropped += 1
|
||||
scores.append(
|
||||
sum(kp.c for kp in kps) / len(kps) if kps else 0.0)
|
||||
# NMS on remaining bboxes.
|
||||
bboxes = [self._bbox_from_kps(kps) for kps in bodies]
|
||||
order = sorted(
|
||||
[i for i in range(len(bodies)) if keep_mask[i]],
|
||||
key=lambda i: -scores[i])
|
||||
kept_order: list[int] = []
|
||||
for i in order:
|
||||
drop = False
|
||||
for j in kept_order:
|
||||
if self._iou(bboxes[i], bboxes[j]) > self._nms_iou:
|
||||
drop = True
|
||||
break
|
||||
if drop:
|
||||
keep_mask[i] = False
|
||||
else:
|
||||
kept_order.append(i)
|
||||
new_bodies = [bodies[i] for i in range(len(bodies)) if keep_mask[i]]
|
||||
new_ids = [ids_body[i] for i in range(len(bodies))
|
||||
if i < len(ids_body) and keep_mask[i]]
|
||||
# bodies3d aligned 1:1 with bodies.
|
||||
new_b3d: list[list[Kp3D]] = []
|
||||
if bodies3d:
|
||||
for i in range(min(len(bodies), len(bodies3d))):
|
||||
if keep_mask[i]:
|
||||
new_b3d.append(bodies3d[i])
|
||||
return new_bodies, new_b3d, new_ids
|
||||
|
||||
def _apply_pid_hysteresis(
|
||||
self,
|
||||
bodies: list[list],
|
||||
ids_body: list[int],
|
||||
) -> list[int]:
|
||||
"""Reuse a recently-disappeared pid when a young pid lands near
|
||||
its last bbox. Mutates self._pid_lifetime / _pid_missing /
|
||||
_pid_last_bbox in place. Returns possibly-remapped ids.
|
||||
"""
|
||||
# Tick all known pids missing counter ; will reset for visible ones.
|
||||
for pid in list(self._pid_missing.keys()):
|
||||
self._pid_missing[pid] += 1
|
||||
if self._pid_missing[pid] > 60: # forget after 2 s @30 fps
|
||||
self._pid_missing.pop(pid, None)
|
||||
self._pid_last_bbox.pop(pid, None)
|
||||
self._pid_lifetime.pop(pid, None)
|
||||
new_ids = list(ids_body)
|
||||
for i, pid in enumerate(ids_body):
|
||||
if pid < 0 or i >= len(bodies):
|
||||
continue
|
||||
bbox_i = self._bbox_from_kps(bodies[i])
|
||||
# If this pid is brand new (<10 frames) and we have an absent
|
||||
# older pid (>=30 frames lifetime, <30 frames missing) with a
|
||||
# close bbox, remap.
|
||||
age = self._pid_lifetime.get(pid, 0)
|
||||
if age < 10:
|
||||
best_old: int | None = None
|
||||
best_iou = 0.0
|
||||
for old_pid, miss in self._pid_missing.items():
|
||||
if old_pid == pid:
|
||||
continue
|
||||
if self._pid_lifetime.get(old_pid, 0) < 30:
|
||||
continue
|
||||
if miss > 30:
|
||||
continue
|
||||
old_bbox = self._pid_last_bbox.get(old_pid)
|
||||
if old_bbox is None:
|
||||
continue
|
||||
iou = self._iou(bbox_i, old_bbox)
|
||||
if iou > 0.3 and iou > best_iou:
|
||||
best_iou = iou
|
||||
best_old = old_pid
|
||||
if best_old is not None:
|
||||
new_ids[i] = best_old
|
||||
pid = best_old
|
||||
# Bookkeeping for visible pid.
|
||||
self._pid_lifetime[pid] = self._pid_lifetime.get(pid, 0) + 1
|
||||
self._pid_missing.pop(pid, None)
|
||||
self._pid_last_bbox[pid] = bbox_i
|
||||
# Pids previously visible but absent this frame -> mark missing.
|
||||
visible = set(new_ids)
|
||||
for pid in list(self._pid_lifetime.keys()):
|
||||
if pid not in visible and pid not in self._pid_missing:
|
||||
self._pid_missing[pid] = 1
|
||||
return new_ids
|
||||
|
||||
def _drop_low_visibility(
|
||||
self,
|
||||
kps_list: list[list],
|
||||
ids: list[int],
|
||||
min_visible: int,
|
||||
which: str,
|
||||
) -> tuple[list[list], list[int]]:
|
||||
out_kps: list[list] = []
|
||||
out_ids: list[int] = []
|
||||
for i, kps in enumerate(kps_list):
|
||||
n_ok = sum(
|
||||
1 for kp in kps
|
||||
if _is_finite(kp.x) and _is_finite(kp.y)
|
||||
and (kp.x != 0.0 or kp.y != 0.0))
|
||||
if n_ok < min_visible:
|
||||
if which == "face":
|
||||
self._n_face_dropped += 1
|
||||
else:
|
||||
self._n_hand_dropped += 1
|
||||
continue
|
||||
out_kps.append(kps)
|
||||
out_ids.append(ids[i] if i < len(ids) else -1)
|
||||
return out_kps, out_ids
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="multi", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
from mediapipe.tasks.python import BaseOptions
|
||||
from mediapipe.tasks.python.vision import (
|
||||
PoseLandmarker, PoseLandmarkerOptions,
|
||||
FaceLandmarker, FaceLandmarkerOptions,
|
||||
HandLandmarker, HandLandmarkerOptions,
|
||||
RunningMode,
|
||||
)
|
||||
except ModuleNotFoundError as e:
|
||||
LOG.error("deps manquantes : %s — uv sync --extra pose", e)
|
||||
return
|
||||
|
||||
try:
|
||||
pose_p = _ensure_model("pose")
|
||||
face_p = _ensure_model("face")
|
||||
hand_p = _ensure_model("hand")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("download models failed: %s", e)
|
||||
return
|
||||
|
||||
# GPU delegate (Metal sur macOS) : libere le CPU pour OSC, state,
|
||||
# mesh_rigger. Multi-HMR remote macm1 + MediaPipe GPU M5 =
|
||||
# workload distribue. Toggle via MEDIAPIPE_DELEGATE=cpu si plante.
|
||||
import os as _os
|
||||
_deleg_name = _os.environ.get("MEDIAPIPE_DELEGATE", "gpu").lower()
|
||||
_deleg = (BaseOptions.Delegate.GPU if _deleg_name == "gpu"
|
||||
else BaseOptions.Delegate.CPU)
|
||||
LOG.info("MediaPipe delegate = %s (env MEDIAPIPE_DELEGATE)",
|
||||
_deleg.name)
|
||||
pose = PoseLandmarker.create_from_options(PoseLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(pose_p),
|
||||
delegate=_deleg),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_poses=self.num_persons,
|
||||
min_pose_detection_confidence=self.min_conf,
|
||||
min_pose_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
face = FaceLandmarker.create_from_options(FaceLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(face_p),
|
||||
delegate=_deleg),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_faces=self.num_persons,
|
||||
min_face_detection_confidence=self.min_conf,
|
||||
min_face_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
hand = HandLandmarker.create_from_options(HandLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(hand_p),
|
||||
delegate=_deleg),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_hands=self.num_persons * 2,
|
||||
min_hand_detection_confidence=self.min_conf,
|
||||
min_hand_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
LOG.info("3 landmarkers prets (num=%d, delegate=%s)",
|
||||
self.num_persons, _deleg.name)
|
||||
|
||||
cap = cv2.VideoCapture(self.camera_index)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index %d indisponible (TCC ?)", self.camera_index)
|
||||
return
|
||||
LOG.info("camera ouverte (index %d)", self.camera_index)
|
||||
|
||||
t0_ms = int(time.monotonic() * 1000)
|
||||
while not self._stop.is_set():
|
||||
tA = time.monotonic()
|
||||
ok, frame_bgr = cap.read()
|
||||
if not ok or frame_bgr is None:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
h, w = frame_bgr.shape[:2]
|
||||
# MediaPipe GPU delegate on macOS uploads via CVPixelBuffer
|
||||
# which only accepts 4-channel formats. SRGB (3ch) crashes
|
||||
# in gpu_buffer_storage_cv_pixel_buffer.cc with
|
||||
# "unsupported ImageFrame format: 1". Use SRGBA when on GPU.
|
||||
if _deleg == BaseOptions.Delegate.GPU:
|
||||
frame_rgba = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGBA)
|
||||
mp_img = mp.Image(image_format=mp.ImageFormat.SRGBA,
|
||||
data=frame_rgba)
|
||||
else:
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
mp_img = mp.Image(image_format=mp.ImageFormat.SRGB,
|
||||
data=frame_rgb)
|
||||
ts = int(time.monotonic() * 1000) - t0_ms
|
||||
try:
|
||||
pose_res = pose.detect_for_video(mp_img, ts)
|
||||
face_res = face.detect_for_video(mp_img, ts)
|
||||
hand_res = hand.detect_for_video(mp_img, ts)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("inference: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# Encode webcam JPEG pour overlay
|
||||
ok2, jpg = cv2.imencode(".jpg", frame_bgr,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), 70])
|
||||
jpg_bytes = bytes(jpg) if ok2 else None
|
||||
|
||||
# Bodies : x/y normalises (image) + z (relative depth, NormalizedLandmark
|
||||
# fournit aussi z, plus precis que rien). pose_world_landmarks
|
||||
# donnerait des metres mais on garde un repere coherent avec face/hands.
|
||||
bodies = []
|
||||
pose_list = pose_res.pose_landmarks or []
|
||||
for landmarks_list in pose_list:
|
||||
kp_list = []
|
||||
for lm in landmarks_list[:33]:
|
||||
v = lm.visibility if lm.visibility is not None else 1.0
|
||||
z = float(lm.z) if lm.z is not None else 0.0
|
||||
kp_list.append(PoseKp(
|
||||
x=float(lm.x), y=float(lm.y), z=z, c=float(v)))
|
||||
bodies.append(kp_list)
|
||||
|
||||
# pose_world_landmarks : xyz metric, relative to hip-center.
|
||||
# Aligned 1:1 with pose_landmarks order. Empty fallback if
|
||||
# the MediaPipe build doesn't populate it.
|
||||
bodies3d: list[list[Kp3D]] = []
|
||||
world_list = getattr(pose_res, "pose_world_landmarks", None) or []
|
||||
for landmarks_list in world_list:
|
||||
kp3_list: list[Kp3D] = []
|
||||
for lm in landmarks_list[:33]:
|
||||
v = lm.visibility if lm.visibility is not None else 1.0
|
||||
kp3_list.append(Kp3D(
|
||||
x=float(lm.x), y=float(lm.y),
|
||||
z=float(lm.z if lm.z is not None else 0.0),
|
||||
c=float(v)))
|
||||
bodies3d.append(kp3_list)
|
||||
|
||||
faces = []
|
||||
for landmarks_list in (face_res.face_landmarks or []):
|
||||
kp_list = []
|
||||
for lm in landmarks_list[:478]:
|
||||
z = float(lm.z) if lm.z is not None else 0.0
|
||||
kp_list.append(PoseKp(
|
||||
x=float(lm.x), y=float(lm.y), z=z, c=1.0))
|
||||
faces.append(kp_list)
|
||||
|
||||
hands = []
|
||||
for landmarks_list in (hand_res.hand_landmarks or []):
|
||||
kp_list = []
|
||||
for lm in landmarks_list[:21]:
|
||||
z = float(lm.z) if lm.z is not None else 0.0
|
||||
kp_list.append(PoseKp(
|
||||
x=float(lm.x), y=float(lm.y), z=z, c=1.0))
|
||||
hands.append(kp_list)
|
||||
|
||||
# --- Tracking IDs persistants entre frames -----------------
|
||||
ids_body = self._tracker_body.update(bodies)
|
||||
ids_face = self._tracker_face.update(faces)
|
||||
ids_hand = self._tracker_hand.update(hands)
|
||||
# --- Discrimination : ghost reject + NMS + pid hysteresis --
|
||||
bodies, bodies3d, ids_body = self._reject_ghosts_and_nms(
|
||||
bodies, bodies3d, ids_body)
|
||||
ids_body = self._apply_pid_hysteresis(bodies, ids_body)
|
||||
faces, ids_face = self._drop_low_visibility(
|
||||
faces, ids_face, self._face_min_visible, "face")
|
||||
hands, ids_hand = self._drop_low_visibility(
|
||||
hands, ids_hand, self._hand_min_visible, "hand")
|
||||
# --- Lissage One Euro par keypoint -------------------------
|
||||
t_now = time.monotonic()
|
||||
bodies = [_smooth_kps(self._smooth_body, ids_body[i], kps, t_now)
|
||||
for i, kps in enumerate(bodies)]
|
||||
faces = [_smooth_kps(self._smooth_face, ids_face[i], kps, t_now)
|
||||
for i, kps in enumerate(faces)]
|
||||
hands = [_smooth_kps(self._smooth_hand, ids_hand[i], kps, t_now)
|
||||
for i, kps in enumerate(hands)]
|
||||
# --- Filter chain face + hands (median + Kalman 2D + lookahead)
|
||||
faces = self._filter_chain.apply_face(faces, ids_face, t_now)
|
||||
hands = self._filter_chain.apply_hand(hands, ids_hand, None, t_now)
|
||||
|
||||
# Pont sonore : envoi OSC /pose/* a sclang (body + face + hands)
|
||||
# 3D world landmarks share ids with bodies (same MediaPipe
|
||||
# detection, just a different coordinate space).
|
||||
ids_body3d = ids_body[:len(bodies3d)] if bodies3d else []
|
||||
if bodies3d:
|
||||
bodies3d = self._filter_chain.apply(bodies3d, ids_body3d, t_now)
|
||||
# Debug : log body3d count once / 5 s so we know MediaPipe
|
||||
# actually populates pose_world_landmarks.
|
||||
if not hasattr(self, "_dbg_b3d_t") or t_now - self._dbg_b3d_t > 5.0:
|
||||
LOG.info("body3d: n=%d (pose_world_landmarks)", len(bodies3d))
|
||||
self._dbg_b3d_t = t_now
|
||||
self._sound_bridge.send(
|
||||
bodies, ids_body, t_now,
|
||||
persons_face=faces, persons_face_ids=ids_face,
|
||||
persons_hands=hands, persons_hands_ids=ids_hand,
|
||||
persons_body3d=bodies3d, persons_body3d_ids=ids_body3d)
|
||||
|
||||
with self.state.lock():
|
||||
self.state.persons_body = bodies
|
||||
self.state.persons_face = faces
|
||||
self.state.persons_hands = hands
|
||||
self.state.persons_body_ids = ids_body
|
||||
self.state.persons_body3d = bodies3d
|
||||
self.state.persons_face_ids = ids_face
|
||||
self.state.persons_hands_ids = ids_hand
|
||||
# Compat single-person (1ere personne)
|
||||
if bodies:
|
||||
self.state.body_present = True
|
||||
for k in range(33):
|
||||
self.state.body_kp[k] = bodies[0][k] if k < len(bodies[0]) else PoseKp()
|
||||
else:
|
||||
self.state.body_present = False
|
||||
if faces:
|
||||
self.state.face_present = True
|
||||
for k in range(478):
|
||||
self.state.face_kp[k] = faces[0][k] if k < len(faces[0]) else PoseKp()
|
||||
else:
|
||||
self.state.face_present = False
|
||||
self.state.hands_present = bool(hands)
|
||||
self.state.pose_count = len(bodies)
|
||||
self.state.pose_last_t = time.monotonic()
|
||||
if jpg_bytes:
|
||||
self.state.last_webcam_jpeg = jpg_bytes
|
||||
|
||||
dt = time.monotonic() - tA
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
cap.release()
|
||||
pose.close(); face.close(); hand.close()
|
||||
LOG.info("multi worker stopped")
|
||||
@@ -0,0 +1,231 @@
|
||||
# Multi-HMR + RealityKit — pipeline temps réel multi-personne
|
||||
|
||||
> **IMPLÉMENTÉ 2026-05-13** — voir `MULTIHMR_README.md` pour l'utilisation.
|
||||
> Le pipeline complet (Multi-HMR worker + SMPL-X decoder + TCP sender +
|
||||
> Swift RealityKit app) est en place. Ce document garde l'analyse initiale.
|
||||
|
||||
**Pipeline cible** (post-Apple Vision, post-SMPLer-X) :
|
||||
|
||||
```
|
||||
AVFoundation (AVAssetReader / AVCaptureSession)
|
||||
↓ CVPixelBuffer (zéro copie, hardware decode)
|
||||
Multi-HMR CoreML (Naver 2024) :
|
||||
forward pass unique → SMPL-X (β, θ, expr) × N personnes + position 3D
|
||||
↓
|
||||
OneEuroFilter sur β/θ/expr (lissage non-négociable)
|
||||
↓
|
||||
smplx.SMPLXLayer(β, θ, expr) → vertices 10475 × N
|
||||
↓
|
||||
RealityKit : USDZ mesh skinné, push vertices/frame, render natif
|
||||
```
|
||||
|
||||
## Multi-HMR (CVPR 2024, Naver)
|
||||
|
||||
- Repo : <https://github.com/naver/multi-hmr>
|
||||
- Paper : [arXiv:2402.14654](https://arxiv.org/abs/2402.14654)
|
||||
- Output : SMPL-X complet (corps + mains + visage) + position 3D monde
|
||||
- **Multi-personne natif** : pas besoin de détecteur séparé, le modèle
|
||||
fait detection + pose en un seul forward
|
||||
- Backbone : DINOv2 ViT-B (~80M params)
|
||||
- Latence cible : 50 ms GPU NVIDIA. Sur M5 ANE ~80-120 ms (15-12 fps)
|
||||
|
||||
Variantes :
|
||||
- `multi-hmr-base` : ViT-B, 88 MB
|
||||
- `multi-hmr-large` : ViT-L, 304 MB
|
||||
|
||||
## Procédure d'installation Multi-HMR
|
||||
|
||||
```bash
|
||||
# 1. Clone
|
||||
git clone https://github.com/naver/multi-hmr ~/.cache/av-live-multihmr
|
||||
cd ~/.cache/av-live-multihmr
|
||||
|
||||
# 2. Dépendances Python
|
||||
uv pip install --python /path/to/.venv/bin/python \
|
||||
torch torchvision \
|
||||
smplx einops fairscale \
|
||||
iopath opencv-python \
|
||||
"numpy<2"
|
||||
|
||||
# 3. SMPL-X model (academic license)
|
||||
# Register at https://smpl-x.is.tue.mpg.de/
|
||||
# Download SMPLX_NEUTRAL.npz → models/smplx/
|
||||
|
||||
# 4. Checkpoint Multi-HMR
|
||||
wget https://download.europe.naverlabs.com/ComputerVision/multiHMR/multiHMR_896_L_synth_real_occ.pt \
|
||||
-O checkpoints/multiHMR_896_L_synth_real_occ.pt
|
||||
|
||||
# 5. CoreML conversion (one-shot)
|
||||
python convert_to_coreml.py \
|
||||
--checkpoint checkpoints/multiHMR_896_L_synth_real_occ.pt \
|
||||
--output ~/.cache/av-live-coreml/multi_hmr.mlpackage
|
||||
```
|
||||
|
||||
**Note critique** : la conversion CoreML est probablement bloquée par
|
||||
le même bug `BlobWriter not loaded` sur macOS-arm64 Python 3.14 (cf
|
||||
notre tentative YOLO11n-pose). Solutions :
|
||||
|
||||
1. **Faire la conversion sur une machine Linux ou macOS Python 3.12 isolé**
|
||||
→ copier le `.mlpackage` ensuite dans `~/.cache/av-live-coreml/`
|
||||
2. **Skip CoreML** : utiliser PyTorch MPS direct (un peu plus lent que
|
||||
ANE mais évite le pain de conversion)
|
||||
|
||||
## Worker à créer
|
||||
|
||||
`data_only_viz/multi_hmr_worker.py` (~250 lignes) :
|
||||
|
||||
```python
|
||||
import threading, time, logging
|
||||
from pathlib import Path
|
||||
import torch
|
||||
from .euro_filter import OneEuroFilter, SkeletonFilter
|
||||
from .state import State
|
||||
|
||||
LOG = logging.getLogger("multi_hmr")
|
||||
|
||||
class MultiHMRWorker:
|
||||
def __init__(self, state: State, ckpt_path: Path, smpl_path: Path,
|
||||
num_persons: int = 4, target_fps: float = 15.0,
|
||||
device: str = "mps"):
|
||||
...
|
||||
# OneEuroFilters par parameter SMPL-X (beta:10, theta:165, expr:10)
|
||||
self._smooth_beta = [OneEuroFilter(0.8, 0.05) for _ in range(10)]
|
||||
self._smooth_theta = [OneEuroFilter(1.2, 0.10) for _ in range(165)]
|
||||
self._smooth_expr = [OneEuroFilter(1.0, 0.08) for _ in range(10)]
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
try:
|
||||
import smplx, torch
|
||||
return Path("~/.cache/av-live-multihmr/checkpoints/...").expanduser().exists()
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
def start(self): ...
|
||||
def stop(self): ...
|
||||
|
||||
def _run(self):
|
||||
# 1. Load Multi-HMR model
|
||||
from multi_hmr.models import get_model # sys.path local
|
||||
model = get_model(self._ckpt_path).to(self._device).eval()
|
||||
|
||||
# 2. Load SMPL-X body model
|
||||
from smplx import SMPLXLayer
|
||||
smplx_layer = SMPLXLayer(self._smpl_path, gender='neutral').to(self._device)
|
||||
|
||||
# 3. Capture loop
|
||||
import cv2
|
||||
cap = cv2.VideoCapture(0)
|
||||
while not self._stop.is_set():
|
||||
ok, frame = cap.read()
|
||||
if not ok: continue
|
||||
|
||||
# 4. Multi-HMR forward (1 pass = N personnes)
|
||||
with torch.no_grad():
|
||||
tensor = preprocess(frame).to(self._device)
|
||||
outputs = model(tensor)
|
||||
# outputs.smplx_params : [N, 185] (betas + thetas + expr)
|
||||
# outputs.translation : [N, 3] (position 3D monde)
|
||||
|
||||
# 5. Lissage One Euro
|
||||
t = time.monotonic()
|
||||
smoothed = []
|
||||
for i, params in enumerate(outputs.smplx_params):
|
||||
smoothed.append(self._smooth_person(i, params, t))
|
||||
|
||||
# 6. Décodage SMPL-X → vertices
|
||||
persons = []
|
||||
for i, params in enumerate(smoothed):
|
||||
betas, thetas, exprs = split_params(params)
|
||||
out = smplx_layer(betas=betas, body_pose=thetas[:66],
|
||||
left_hand_pose=thetas[66:111],
|
||||
right_hand_pose=thetas[111:156],
|
||||
jaw_pose=thetas[156:159],
|
||||
expression=exprs, ...)
|
||||
verts = out.vertices.cpu().numpy() # (10475, 3)
|
||||
joints = out.joints.cpu().numpy() # (127, 3)
|
||||
persons.append({
|
||||
"vertices_3d": verts,
|
||||
"joints_3d": joints,
|
||||
"translation": outputs.translation[i].cpu().numpy(),
|
||||
"pid": self._tracker_update(...)
|
||||
})
|
||||
|
||||
# 7. Écrit dans State
|
||||
with self.state.lock():
|
||||
self.state.persons_smplx = persons
|
||||
self.state.persons_smplx_t = time.monotonic()
|
||||
```
|
||||
|
||||
## State extension
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class SMPLXPerson:
|
||||
pid: int
|
||||
vertices_3d: list[tuple[float, float, float]] # 10475 verts
|
||||
joints_3d: list[tuple[float, float, float]] # 127 joints
|
||||
translation: tuple[float, float, float] # 3D monde
|
||||
rotation: tuple[float, float, float, float] # quaternion
|
||||
|
||||
persons_smplx: list[SMPLXPerson] = field(default_factory=list)
|
||||
smplx_faces: list[tuple[int, int, int]] = field(default_factory=list) # 20908 statique
|
||||
```
|
||||
|
||||
## RealityKit bridge
|
||||
|
||||
**Option A : RKView via pyobjc** (complexe)
|
||||
|
||||
```python
|
||||
from RealityKit import ARView, Entity, ModelComponent, MeshResource
|
||||
# Charge USDZ template
|
||||
template = Entity.loadModelAsync("smplx_template.usdz")
|
||||
# Per frame: update vertex buffer
|
||||
for person in state.persons_smplx:
|
||||
entity.components[ModelComponent].mesh = MeshResource.generate(
|
||||
from_descriptors=[
|
||||
MeshDescriptor(
|
||||
positions=person.vertices_3d,
|
||||
indices=state.smplx_faces.flatten(),
|
||||
)
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
**Option B : application Swift native** (réaliste)
|
||||
|
||||
Création d'une app **AV-Live-Body** séparée en Swift :
|
||||
1. SwiftUI window avec ARView
|
||||
2. Reçoit les vertices via OSC depuis le worker Python Multi-HMR
|
||||
3. Render RealityKit natif sans bridge pyobjc
|
||||
|
||||
Format OSC :
|
||||
```
|
||||
/smplx/person <pid> <tx> <ty> <tz>
|
||||
/smplx/verts <pid> <10475 × 3 floats binaires>
|
||||
```
|
||||
|
||||
UDP packet trop gros (~125KB) → utiliser TCP ou shared memory.
|
||||
|
||||
## Recommandation pragmatique
|
||||
|
||||
**Court terme (cette session)** :
|
||||
- Garder Apple Vision body pose (marche, simple)
|
||||
- Body mesh : 8 triangles (tronc + bras + jambes) — déjà en place
|
||||
- Accepter que face/hands mesh est bloqué par pyobjc PyObjCPointer
|
||||
|
||||
**Moyen terme (1 semaine de travail)** :
|
||||
- Installer Multi-HMR sur un Python 3.12 séparé (éviter coremltools issues)
|
||||
- Worker dédié qui tourne 8-12 fps
|
||||
- Rendu mesh dans Metal pipeline existant (triangles remplis)
|
||||
|
||||
**Long terme (2+ semaines)** :
|
||||
- App Swift native avec RealityKit pour rendu mesh skinné
|
||||
- Python worker envoie params SMPL-X via OSC TCP
|
||||
- L'app Swift décode SMPL-X et rend
|
||||
|
||||
## Décision
|
||||
|
||||
Cette session : **rester sur Apple Vision body actuel** qui fonctionne.
|
||||
Tout le reste (Multi-HMR / RealityKit / SMPL-X) demande un setup
|
||||
substantiel hors scope d'une session live.
|
||||
@@ -0,0 +1,714 @@
|
||||
"""Worker Multi-HMR : capture webcam Mac, inference forward unique
|
||||
SMPL-X (multi-personne natif), extraction vertices v3d, ecriture State.
|
||||
|
||||
Le repo Multi-HMR n'est pas pip-installable — on injecte le clone dans
|
||||
sys.path au runtime. Chaque humain renvoye contient deja les vertices
|
||||
SMPL-X decodes (cle `v3d`, shape (10475, 3)) ; pas besoin du decoder
|
||||
SMPL-X separe en hot path (il reste utile pour les tests).
|
||||
|
||||
Cadence cible : 8-12 fps sur M5 (ViT-S). Lissage One Euro sur les
|
||||
shapes/expression pour limiter le jitter trame-a-trame.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .euro_filter import OneEuroFilter
|
||||
from .state import PoseKp, SMPLXPerson, State
|
||||
from .tracker import IoUTracker
|
||||
|
||||
LOG = logging.getLogger("multi_hmr")
|
||||
|
||||
CACHE = Path.home() / ".cache" / "av-live-multihmr"
|
||||
CKPT = CACHE / "checkpoints" / "multiHMR_672_S.pt"
|
||||
SMPLX_PATH = CACHE / "models" / "smplx" / "SMPLX_NEUTRAL.npz"
|
||||
MULTIHMR_REPO = CACHE / "multi-hmr"
|
||||
COREML_MLPACKAGE = Path(
|
||||
os.environ.get("COREML_MLPACKAGE")
|
||||
or str(CACHE / "multihmr_full_672_s.mlpackage"))
|
||||
|
||||
IMG_SIZE = 672
|
||||
N_VERTS = 10475
|
||||
|
||||
|
||||
class MultiHMRWorker:
|
||||
def __init__(self, state: State, num_persons: int = 4,
|
||||
target_fps: float = 10.0, device: str = "mps",
|
||||
det_thresh: float = 0.3,
|
||||
nms_kernel_size: int = 5,
|
||||
motion_gate: float = 5.0,
|
||||
camera_index: int = -1,
|
||||
backend: str | None = None) -> None:
|
||||
self.state = state
|
||||
self.num_persons = num_persons
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self.device = device
|
||||
self.det_thresh = det_thresh
|
||||
self.nms_kernel_size = nms_kernel_size
|
||||
# Motion gate : si la diff moyenne par pixel (sur frame 672x672
|
||||
# downsamplee a 112x112 pour speed) est < motion_gate, on skip
|
||||
# l'inference et on reutilise les v3d precedents. Seuil en
|
||||
# unites 0-255. Mettre <=0 pour desactiver.
|
||||
self.motion_gate = motion_gate
|
||||
# -1 = auto-select Mac BuiltInWideAngleCamera (cf _camera_select)
|
||||
self.camera_index = camera_index
|
||||
# backend: 'pytorch' (default) or 'coreml'. CoreML uses the
|
||||
# .mlpackage at COREML_MLPACKAGE, bypasses MPS torch, and runs
|
||||
# on ANE/GPU/CPU via CoreML.framework natively (3-4x faster).
|
||||
self.backend = (backend
|
||||
or os.environ.get("MULTIHMR_BACKEND", "pytorch")
|
||||
).strip().lower()
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._smooth_shape = [
|
||||
[OneEuroFilter(0.8, 0.05) for _ in range(10)]
|
||||
for _ in range(num_persons)
|
||||
]
|
||||
self._smooth_expr = [
|
||||
[OneEuroFilter(1.0, 0.08) for _ in range(10)]
|
||||
for _ in range(num_persons)
|
||||
]
|
||||
# iou_threshold bas + max_miss eleve + prediction velocity
|
||||
# (cf tracker.py) pour resister aux occlusions et au mouvement
|
||||
# rapide. Multi-HMR a 3 fps -> 30 frames = 10s de survie.
|
||||
self._tracker = IoUTracker(iou_threshold=0.15, max_miss=30)
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
backend = os.environ.get("MULTIHMR_BACKEND", "pytorch").strip().lower()
|
||||
if backend == "coreml":
|
||||
return COREML_MLPACKAGE.exists()
|
||||
if backend == "remote":
|
||||
try:
|
||||
from .multihmr_remote import MultiHMRRemoteBackend
|
||||
return MultiHMRRemoteBackend.is_available()
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
return CKPT.exists() and SMPLX_PATH.exists() and MULTIHMR_REPO.exists()
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="multi_hmr", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def _run(self) -> None:
|
||||
if self.backend == "coreml":
|
||||
self._run_coreml(remote=False)
|
||||
return
|
||||
if self.backend == "remote":
|
||||
self._run_coreml(remote=True)
|
||||
return
|
||||
self._run_pytorch()
|
||||
|
||||
def _run_pytorch(self) -> None:
|
||||
if str(MULTIHMR_REPO) not in sys.path:
|
||||
sys.path.insert(0, str(MULTIHMR_REPO))
|
||||
# Multi-HMR demo.py tire pyrender / pyvista (OpenGL offscreen) et
|
||||
# multi_hmr_anny (anny package non public). Aucun n'est necessaire
|
||||
# pour l'inference brute : on stubbe.
|
||||
import types as _t
|
||||
for mod in ("pyrender", "pyvista", "anny"):
|
||||
if mod not in sys.modules:
|
||||
sys.modules[mod] = _t.ModuleType(mod)
|
||||
try:
|
||||
import torch
|
||||
import cv2
|
||||
# Import direct du Model (sans passer par demo.load_model qui
|
||||
# depend de multi_hmr_anny).
|
||||
from model import Model # type: ignore
|
||||
except ImportError as e:
|
||||
LOG.error("deps manquantes : %s — uv sync --extra multihmr "
|
||||
"et bash scripts/setup_multihmr.sh", e)
|
||||
return
|
||||
|
||||
if self.device == "mps" and not torch.backends.mps.is_available():
|
||||
LOG.warning("MPS unavailable, falling back to cpu")
|
||||
device = "cpu"
|
||||
else:
|
||||
device = self.device
|
||||
|
||||
ckpt_name = CKPT.stem
|
||||
# SMPLX_DIR='models' et MEAN_PARAMS='models/smpl_mean_params.npz'
|
||||
# sont relatifs au cwd. On bascule dans le repo Multi-HMR pour la
|
||||
# construction du modele puis on revient.
|
||||
prev_cwd = os.getcwd()
|
||||
try:
|
||||
os.chdir(MULTIHMR_REPO)
|
||||
torch_device = torch.device(device)
|
||||
ckpt = torch.load(str(CKPT), map_location=torch_device,
|
||||
weights_only=False)
|
||||
kwargs = {k: v for k, v in vars(ckpt["args"]).items()}
|
||||
kwargs["type"] = ckpt["args"].train_return_type
|
||||
kwargs["img_size"] = ckpt["args"].img_size[0]
|
||||
model = Model(**kwargs).to(torch_device)
|
||||
model.load_state_dict(ckpt["model_state_dict"], strict=False)
|
||||
model.eval()
|
||||
# MPS mixed precision via torch.autocast : ~1.3-1.7x sur
|
||||
# ViT-S backbone, casts auto vers float16 pour les matmuls
|
||||
# gardant l'accumulator en float32 (necessaire MPS sinon
|
||||
# "Destination NDArray and Accumulator NDArray cannot have
|
||||
# different datatype" sur MPSNDArrayMatrixMultiplication).
|
||||
# Disable via env MULTIHMR_AUTOCAST=0.
|
||||
# autocast MPS teste 2026-05-13 : plus lent (400ms vs 270ms
|
||||
# baseline) car overhead de cast dans le forward. Defaut OFF.
|
||||
# Opt-in via MULTIHMR_AUTOCAST=1.
|
||||
self._use_autocast = (
|
||||
device == "mps"
|
||||
and os.environ.get("MULTIHMR_AUTOCAST", "0") == "1")
|
||||
if self._use_autocast:
|
||||
LOG.info("Multi-HMR PyTorch : MPS autocast (fp16) enabled")
|
||||
# torch.compile teste 2026-05-13 : plante en runtime avec
|
||||
# `TypeError: torch.Size() takes an iterable of 'int' (item
|
||||
# is 'FakeTensor')`. Multi-HMR a du shape-arithmetic non
|
||||
# traceable, on garde le eager.
|
||||
except Exception as e:
|
||||
LOG.error("Multi-HMR load failed: %s", e)
|
||||
os.chdir(prev_cwd)
|
||||
return
|
||||
finally:
|
||||
os.chdir(prev_cwd)
|
||||
LOG.info("Multi-HMR loaded (%s) on %s", ckpt_name, device)
|
||||
|
||||
# Camera intrinsics (focale = img_size par defaut). batch dim 1.
|
||||
focal = float(IMG_SIZE)
|
||||
K = torch.tensor([[[focal, 0.0, IMG_SIZE / 2.0],
|
||||
[0.0, focal, IMG_SIZE / 2.0],
|
||||
[0.0, 0.0, 1.0]]], device=device)
|
||||
|
||||
# Capture AVFoundation native — selection par device-type, pas
|
||||
# par index cv2 (qui ne suit pas l'ordre AVFoundation et finit
|
||||
# parfois sur l'iPhone Continuity).
|
||||
from ._av_capture import AVCapture, find_builtin_device, enumerate_devices
|
||||
if self.camera_index >= 0:
|
||||
devs = enumerate_devices()
|
||||
if self.camera_index >= len(devs):
|
||||
LOG.error("camera_index %d hors de %d devices",
|
||||
self.camera_index, len(devs))
|
||||
return
|
||||
info = devs[self.camera_index]
|
||||
else:
|
||||
info = find_builtin_device()
|
||||
if info is None:
|
||||
LOG.error("aucune BuiltInWideAngleCamera trouvee")
|
||||
return
|
||||
cap = AVCapture(info)
|
||||
if not cap.start():
|
||||
LOG.error("AVCapture start failed pour %s", info["name"])
|
||||
return
|
||||
LOG.info("camera ouverte %s (%s)", info["name"], info["type"])
|
||||
frame_count = 0
|
||||
persons_count = 0
|
||||
skipped_static = 0
|
||||
next_heartbeat = time.monotonic() + 5.0
|
||||
# Frame thumbnail precedent pour motion gate (112x112 gray).
|
||||
prev_thumb: np.ndarray | None = None
|
||||
|
||||
while not self._stop.is_set():
|
||||
t_cap_start = time.monotonic()
|
||||
ok, frame_bgr = cap.read(timeout_s=0.5)
|
||||
if not ok or frame_bgr is None:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
t_pre_start = time.monotonic()
|
||||
# Crop/resize au carre 896 pour matcher Multi-HMR
|
||||
h, w = frame_bgr.shape[:2]
|
||||
if (h, w) != (IMG_SIZE, IMG_SIZE):
|
||||
# Center-crop + resize
|
||||
side = min(h, w)
|
||||
y0 = (h - side) // 2
|
||||
x0 = (w - side) // 2
|
||||
frame_bgr = frame_bgr[y0:y0 + side, x0:x0 + side]
|
||||
frame_bgr = cv2.resize(frame_bgr, (IMG_SIZE, IMG_SIZE))
|
||||
|
||||
# Motion gate : downsample en 112x112 gris, diff vs frame
|
||||
# precedente. Si bouge peu, skip l'inference (re-utilise
|
||||
# les v3d deja en state).
|
||||
if self.motion_gate > 0:
|
||||
thumb = cv2.cvtColor(
|
||||
cv2.resize(frame_bgr, (112, 112)),
|
||||
cv2.COLOR_BGR2GRAY)
|
||||
if prev_thumb is not None:
|
||||
diff_mean = float(np.mean(
|
||||
cv2.absdiff(thumb, prev_thumb)))
|
||||
if diff_mean < self.motion_gate:
|
||||
prev_thumb = thumb
|
||||
skipped_static += 1
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
prev_thumb = thumb
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
# Publish to state for DINOv2 reid in MeshRigger.
|
||||
with self.state.lock():
|
||||
self.state.last_frame_rgb = frame_rgb
|
||||
self.state.last_frame_rgb_t = time.monotonic()
|
||||
tensor = torch.from_numpy(frame_rgb).permute(2, 0, 1).float()
|
||||
tensor = (tensor / 255.0).unsqueeze(0).to(device)
|
||||
|
||||
t_inf_start = time.monotonic()
|
||||
try:
|
||||
with torch.no_grad():
|
||||
if getattr(self, "_use_autocast", False):
|
||||
with torch.autocast(device_type="mps",
|
||||
dtype=torch.float16):
|
||||
humans = model(
|
||||
tensor,
|
||||
is_training=False,
|
||||
nms_kernel_size=self.nms_kernel_size,
|
||||
det_thresh=self.det_thresh,
|
||||
K=K,
|
||||
)
|
||||
else:
|
||||
humans = model(
|
||||
tensor,
|
||||
is_training=False,
|
||||
nms_kernel_size=self.nms_kernel_size,
|
||||
det_thresh=self.det_thresh,
|
||||
K=K,
|
||||
)
|
||||
except Exception as e:
|
||||
LOG.warning("inference failed: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
t_post_start = time.monotonic()
|
||||
t_now = time.monotonic()
|
||||
# Count frame + heartbeat regardless of detection — keeps the
|
||||
# FPS metric meaningful when nobody is in the camera view.
|
||||
frame_count += 1
|
||||
persons_count += len(humans) if humans else 0
|
||||
if t_now >= next_heartbeat:
|
||||
fps = frame_count / 5.0
|
||||
avg = persons_count / max(1, frame_count)
|
||||
LOG.info(
|
||||
"hb: %.1f fps, %.2f persons/frame, %d skipped (static)",
|
||||
fps, avg, skipped_static)
|
||||
frame_count = 0
|
||||
persons_count = 0
|
||||
skipped_static = 0
|
||||
next_heartbeat = t_now + 5.0
|
||||
if not humans:
|
||||
with self.state.lock():
|
||||
self.state.persons_smplx = []
|
||||
inf_ms = (t_post_start - t_inf_start) * 1e3
|
||||
LOG.debug("frame (no detect): inf=%.1fms", inf_ms)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# Dedup intra-frame : Multi-HMR peut retourner plusieurs
|
||||
# detections pour la meme personne. On combine bbox 2D IoU
|
||||
# ET distance pelvis 3D : drop ssi IoU > 0.4 ET dist < 30 cm.
|
||||
# Comme ca deux personnes qui se chevauchent en 2D (une
|
||||
# devant l'autre) restent distinctes grace au z.
|
||||
cand: list[tuple[
|
||||
float, float, float, float, float,
|
||||
np.ndarray, int]] = []
|
||||
for i, h in enumerate(humans):
|
||||
v = h["v3d"].detach().cpu().numpy()
|
||||
xmin = float(v[:, 0].min())
|
||||
ymin = float(v[:, 1].min())
|
||||
xmax = float(v[:, 0].max())
|
||||
ymax = float(v[:, 1].max())
|
||||
sc_raw = h.get("scores", 1.0)
|
||||
score = float(sc_raw.item()) if hasattr(
|
||||
sc_raw, "item") else float(sc_raw)
|
||||
transl = h.get("transl_pelvis", h.get("transl"))
|
||||
pelv = transl.detach().cpu().numpy().flatten()[:3]
|
||||
cand.append((score, xmin, ymin, xmax, ymax, pelv, i))
|
||||
cand.sort(key=lambda c: -c[0])
|
||||
keep_idx: list[int] = []
|
||||
kept: list[tuple[
|
||||
float, float, float, float, np.ndarray]] = []
|
||||
for sc, x0, y0, x1, y1, pelv, src_i in cand:
|
||||
a_area = max(0.0, x1 - x0) * max(0.0, y1 - y0)
|
||||
drop = False
|
||||
for (kx0, ky0, kx1, ky1, kpelv) in kept:
|
||||
ix0 = max(x0, kx0); iy0 = max(y0, ky0)
|
||||
ix1 = min(x1, kx1); iy1 = min(y1, ky1)
|
||||
iw = max(0.0, ix1 - ix0); ih = max(0.0, iy1 - iy0)
|
||||
inter = iw * ih
|
||||
if a_area <= 0 or inter <= 0:
|
||||
continue
|
||||
k_area = (kx1 - kx0) * (ky1 - ky0)
|
||||
iou = inter / (a_area + k_area - inter + 1e-9)
|
||||
pelv_d = float(np.linalg.norm(pelv - kpelv))
|
||||
# Drop seulement si TRES proches en 3D ET grand
|
||||
# overlap 2D. Seuils volontairement conservateurs
|
||||
# pour ne pas fusionner deux personnes serrees.
|
||||
if iou > 0.55 and pelv_d < 0.20:
|
||||
drop = True
|
||||
break
|
||||
if not drop:
|
||||
keep_idx.append(src_i)
|
||||
kept.append((x0, y0, x1, y1, pelv))
|
||||
if len(keep_idx) >= self.num_persons:
|
||||
break
|
||||
n_raw = len(humans)
|
||||
humans = [humans[i] for i in keep_idx]
|
||||
n_keep = len(humans)
|
||||
if n_raw != n_keep:
|
||||
LOG.debug("dedup: %d -> %d (raw det_thresh=%.2f)",
|
||||
n_raw, n_keep, self.det_thresh)
|
||||
|
||||
# Tracking via bbox approximee depuis verts projetes (xy)
|
||||
bboxes = []
|
||||
for h in humans:
|
||||
v = h["v3d"].detach().cpu().numpy() # (10475, 3)
|
||||
xmin, ymin = float(v[:, 0].min()), float(v[:, 1].min())
|
||||
xmax, ymax = float(v[:, 0].max()), float(v[:, 1].max())
|
||||
bboxes.append([PoseKp(x=xmin, y=ymin, c=1.0),
|
||||
PoseKp(x=xmax, y=ymax, c=1.0)])
|
||||
ids = self._tracker.update(bboxes)
|
||||
|
||||
persons: list[SMPLXPerson] = []
|
||||
for i, hh in enumerate(humans[:n_keep]):
|
||||
pid = ids[i] if i < len(ids) else i
|
||||
if pid < 0:
|
||||
continue
|
||||
|
||||
v3d = hh["v3d"].detach().cpu().numpy()
|
||||
transl = hh.get("transl_pelvis", hh.get("transl"))
|
||||
transl_np = transl.detach().cpu().numpy().flatten()
|
||||
|
||||
shape_raw = hh["shape"].detach().cpu().numpy().flatten()
|
||||
expr_raw = hh["expression"].detach().cpu().numpy().flatten()
|
||||
|
||||
# Skip persons with NaN/Inf vertices or transl : MPS can
|
||||
# occasionally emit garbage that propagates to AVLiveBody
|
||||
# as spikes / holes. We drop the frame for that pid and
|
||||
# let the receiver's retain window keep the last good mesh.
|
||||
if (not np.isfinite(v3d).all()
|
||||
or not np.isfinite(transl_np).all()):
|
||||
LOG.warning("Multi-HMR NaN/Inf at pid=%d, skipping", pid)
|
||||
continue
|
||||
# Sanity clamp on extreme vertex magnitudes (humans are
|
||||
# ~2 m ; vertices outside [-5, 5] m are model glitches).
|
||||
if float(np.abs(v3d).max()) > 5.0:
|
||||
LOG.warning(
|
||||
"Multi-HMR v3d extreme |max|=%.1f at pid=%d, skipping",
|
||||
float(np.abs(v3d).max()), pid,
|
||||
)
|
||||
continue
|
||||
|
||||
pid_c = pid % self.num_persons
|
||||
shape_n = min(10, len(shape_raw))
|
||||
expr_n = min(10, len(expr_raw))
|
||||
shape_smooth = np.zeros(10, dtype=np.float32)
|
||||
expr_smooth = np.zeros(10, dtype=np.float32)
|
||||
for k in range(shape_n):
|
||||
shape_smooth[k] = self._smooth_shape[pid_c][k](
|
||||
float(shape_raw[k]), t_now)
|
||||
for k in range(expr_n):
|
||||
expr_smooth[k] = self._smooth_expr[pid_c][k](
|
||||
float(expr_raw[k]), t_now)
|
||||
|
||||
persons.append(SMPLXPerson(
|
||||
pid=int(pid),
|
||||
vertices_3d=np.ascontiguousarray(v3d, dtype=np.float32),
|
||||
translation=np.ascontiguousarray(transl_np[:3], dtype=np.float32),
|
||||
confidence=float(hh.get("scores", 1.0)) if not hasattr(
|
||||
hh.get("scores", None), "item") else float(
|
||||
hh["scores"].item()),
|
||||
betas=np.ascontiguousarray(shape_smooth, dtype=np.float32),
|
||||
expression=np.ascontiguousarray(expr_smooth, dtype=np.float32),
|
||||
))
|
||||
|
||||
with self.state.lock():
|
||||
self.state.persons_smplx = persons
|
||||
self.state.smplx_last_t = t_now
|
||||
|
||||
t_end = time.monotonic()
|
||||
dt_total = (t_end - t_cap_start) * 1e3
|
||||
if LOG.isEnabledFor(logging.DEBUG) or dt_total > 100.0:
|
||||
LOG.log(
|
||||
logging.DEBUG if dt_total <= 100.0 else logging.WARNING,
|
||||
"frame: cap=%.1f pre=%.1f inf=%.1f post=%.1fms total=%.1fms",
|
||||
(t_pre_start - t_cap_start) * 1e3,
|
||||
(t_inf_start - t_pre_start) * 1e3,
|
||||
(t_post_start - t_inf_start) * 1e3,
|
||||
(t_end - t_post_start) * 1e3,
|
||||
dt_total,
|
||||
)
|
||||
|
||||
dt = time.monotonic() - t_cap_start
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
|
||||
cap.stop()
|
||||
LOG.info("multi_hmr worker stopped")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CoreML backend
|
||||
# ------------------------------------------------------------------
|
||||
def _run_coreml(self, remote: bool = False) -> None:
|
||||
"""CoreML inference path (ANE+GPU+CPU via Apple's framework).
|
||||
|
||||
Mirrors _run_pytorch but loads the .mlpackage via pyobjc + the
|
||||
CoreML.framework, bypassing torch/MPS entirely. ~3-4x faster
|
||||
on M5 (28.8ms median vs ~100ms with MPS).
|
||||
|
||||
If ``remote=True``, the local CoreML backend is swapped for a
|
||||
TCP client (``MultiHMRRemoteBackend``) that talks to a server
|
||||
running the same mlpackage on a faster Mac (macm1, M1 Max).
|
||||
"""
|
||||
try:
|
||||
import cv2
|
||||
except ImportError as e:
|
||||
LOG.error("opencv-python missing: %s", e)
|
||||
return
|
||||
try:
|
||||
if remote:
|
||||
from .multihmr_remote import MultiHMRRemoteBackend
|
||||
host = os.environ.get(
|
||||
"MULTIHMR_REMOTE_HOST", "192.168.0.175")
|
||||
port = int(os.environ.get(
|
||||
"MULTIHMR_REMOTE_PORT", "57140"))
|
||||
backend = MultiHMRRemoteBackend(host=host, port=port)
|
||||
LOG.info("Multi-HMR remote backend (%s:%d)", host, port)
|
||||
else:
|
||||
from .multihmr_coreml import MultiHMRCoreMLBackend
|
||||
backend = MultiHMRCoreMLBackend(COREML_MLPACKAGE)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("CoreML backend init failed: %s", e)
|
||||
return
|
||||
|
||||
focal = float(IMG_SIZE)
|
||||
K_np = np.array([[focal, 0.0, IMG_SIZE / 2.0],
|
||||
[0.0, focal, IMG_SIZE / 2.0],
|
||||
[0.0, 0.0, 1.0]], dtype=np.float32)
|
||||
|
||||
from ._av_capture import (
|
||||
AVCapture, find_builtin_device, enumerate_devices)
|
||||
if self.camera_index >= 0:
|
||||
devs = enumerate_devices()
|
||||
if self.camera_index >= len(devs):
|
||||
LOG.error("camera_index %d hors de %d devices",
|
||||
self.camera_index, len(devs))
|
||||
return
|
||||
info = devs[self.camera_index]
|
||||
else:
|
||||
info = find_builtin_device()
|
||||
if info is None:
|
||||
LOG.error("aucune BuiltInWideAngleCamera trouvee")
|
||||
return
|
||||
cap = AVCapture(info)
|
||||
if not cap.start():
|
||||
LOG.error("AVCapture start failed pour %s", info["name"])
|
||||
return
|
||||
LOG.info("camera ouverte %s (%s) [%s backend]",
|
||||
info["name"], info["type"],
|
||||
"remote" if remote else "coreml")
|
||||
|
||||
frame_count = 0
|
||||
persons_count = 0
|
||||
skipped_static = 0
|
||||
fresh_count = 0
|
||||
next_heartbeat = time.monotonic() + 5.0
|
||||
prev_thumb: np.ndarray | None = None
|
||||
|
||||
while not self._stop.is_set():
|
||||
t_cap_start = time.monotonic()
|
||||
ok, frame_bgr = cap.read(timeout_s=0.5)
|
||||
if not ok or frame_bgr is None:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
t_pre_start = time.monotonic()
|
||||
h, w = frame_bgr.shape[:2]
|
||||
if (h, w) != (IMG_SIZE, IMG_SIZE):
|
||||
side = min(h, w)
|
||||
y0 = (h - side) // 2
|
||||
x0 = (w - side) // 2
|
||||
frame_bgr = frame_bgr[y0:y0 + side, x0:x0 + side]
|
||||
frame_bgr = cv2.resize(frame_bgr, (IMG_SIZE, IMG_SIZE))
|
||||
|
||||
if self.motion_gate > 0:
|
||||
thumb = cv2.cvtColor(
|
||||
cv2.resize(frame_bgr, (112, 112)),
|
||||
cv2.COLOR_BGR2GRAY)
|
||||
if prev_thumb is not None:
|
||||
diff_mean = float(np.mean(
|
||||
cv2.absdiff(thumb, prev_thumb)))
|
||||
if diff_mean < self.motion_gate:
|
||||
prev_thumb = thumb
|
||||
skipped_static += 1
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
prev_thumb = thumb
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
with self.state.lock():
|
||||
self.state.last_frame_rgb = frame_rgb
|
||||
self.state.last_frame_rgb_t = time.monotonic()
|
||||
img = frame_rgb.transpose(2, 0, 1).astype(np.float32) / 255.0
|
||||
|
||||
t_inf_start = time.monotonic()
|
||||
try:
|
||||
humans = backend.infer(img, K_np, det_thresh=self.det_thresh)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("coreml inference failed: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# Async remote backend may return None when no fresh result
|
||||
# is ready yet — reuse the previous frame's humans so the
|
||||
# visualiser keeps drawing instead of clearing.
|
||||
if humans is None:
|
||||
humans = getattr(self, "_last_humans", []) or []
|
||||
reused_humans = True
|
||||
else:
|
||||
self._last_humans = humans
|
||||
reused_humans = False
|
||||
fresh_count += 1
|
||||
|
||||
t_post_start = time.monotonic()
|
||||
t_now = time.monotonic()
|
||||
frame_count += 1
|
||||
persons_count += len(humans) if humans else 0
|
||||
if reused_humans:
|
||||
LOG.debug("hb[remote]: reusing %d cached humans "
|
||||
"(no fresh result)", len(humans))
|
||||
if t_now >= next_heartbeat:
|
||||
fps = frame_count / 5.0
|
||||
fresh_fps = fresh_count / 5.0
|
||||
avg = persons_count / max(1, frame_count)
|
||||
LOG.info(
|
||||
"hb[coreml]: %.1f fps (fresh=%.1f), %.2f persons/frame, "
|
||||
"%d skipped", fps, fresh_fps, avg, skipped_static)
|
||||
frame_count = 0
|
||||
persons_count = 0
|
||||
fresh_count = 0
|
||||
skipped_static = 0
|
||||
next_heartbeat = t_now + 5.0
|
||||
|
||||
if not humans:
|
||||
with self.state.lock():
|
||||
self.state.persons_smplx = []
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# If async backend reused last humans, keep state untouched and
|
||||
# spin to the next frame without re-running dedup/tracker/
|
||||
# smoothing (saves ~3-5 ms CPU per loop iteration and avoids
|
||||
# walking the One-Euro filter forward on stale data).
|
||||
if reused_humans:
|
||||
dt = time.monotonic() - t_cap_start
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
continue
|
||||
|
||||
# Dedup intra-frame (same logic as pytorch path).
|
||||
cand: list[tuple[
|
||||
float, float, float, float, float,
|
||||
np.ndarray, int]] = []
|
||||
for i, hh in enumerate(humans):
|
||||
v = hh["v3d"].detach().cpu().numpy()
|
||||
xmin = float(v[:, 0].min()); ymin = float(v[:, 1].min())
|
||||
xmax = float(v[:, 0].max()); ymax = float(v[:, 1].max())
|
||||
score = float(hh["scores"].item())
|
||||
pelv = hh["transl_pelvis"].detach().cpu().numpy(
|
||||
).flatten()[:3]
|
||||
cand.append((score, xmin, ymin, xmax, ymax, pelv, i))
|
||||
cand.sort(key=lambda c: -c[0])
|
||||
keep_idx: list[int] = []
|
||||
kept: list[tuple[float, float, float, float, np.ndarray]] = []
|
||||
for sc, x0, y0, x1, y1, pelv, src_i in cand:
|
||||
a_area = max(0.0, x1 - x0) * max(0.0, y1 - y0)
|
||||
drop = False
|
||||
for (kx0, ky0, kx1, ky1, kpelv) in kept:
|
||||
ix0 = max(x0, kx0); iy0 = max(y0, ky0)
|
||||
ix1 = min(x1, kx1); iy1 = min(y1, ky1)
|
||||
iw = max(0.0, ix1 - ix0); ih = max(0.0, iy1 - iy0)
|
||||
inter = iw * ih
|
||||
if a_area <= 0 or inter <= 0:
|
||||
continue
|
||||
k_area = (kx1 - kx0) * (ky1 - ky0)
|
||||
iou = inter / (a_area + k_area - inter + 1e-9)
|
||||
pelv_d = float(np.linalg.norm(pelv - kpelv))
|
||||
if iou > 0.55 and pelv_d < 0.20:
|
||||
drop = True
|
||||
break
|
||||
if not drop:
|
||||
keep_idx.append(src_i)
|
||||
kept.append((x0, y0, x1, y1, pelv))
|
||||
if len(keep_idx) >= self.num_persons:
|
||||
break
|
||||
humans = [humans[i] for i in keep_idx]
|
||||
n_keep = len(humans)
|
||||
|
||||
bboxes = []
|
||||
for hh in humans:
|
||||
v = hh["v3d"].detach().cpu().numpy()
|
||||
xmin, ymin = float(v[:, 0].min()), float(v[:, 1].min())
|
||||
xmax, ymax = float(v[:, 0].max()), float(v[:, 1].max())
|
||||
bboxes.append([PoseKp(x=xmin, y=ymin, c=1.0),
|
||||
PoseKp(x=xmax, y=ymax, c=1.0)])
|
||||
ids = self._tracker.update(bboxes)
|
||||
|
||||
persons: list[SMPLXPerson] = []
|
||||
for i, hh in enumerate(humans[:n_keep]):
|
||||
pid = ids[i] if i < len(ids) else i
|
||||
if pid < 0:
|
||||
continue
|
||||
v3d = hh["v3d"].detach().cpu().numpy()
|
||||
transl_np = hh["transl_pelvis"].detach().cpu().numpy().flatten()
|
||||
shape_raw = hh["shape"].detach().cpu().numpy().flatten()
|
||||
expr_raw = hh["expression"].detach().cpu().numpy().flatten()
|
||||
|
||||
pid_c = pid % self.num_persons
|
||||
shape_n = min(10, len(shape_raw))
|
||||
expr_n = min(10, len(expr_raw))
|
||||
shape_smooth = np.zeros(10, dtype=np.float32)
|
||||
expr_smooth = np.zeros(10, dtype=np.float32)
|
||||
for k in range(shape_n):
|
||||
shape_smooth[k] = self._smooth_shape[pid_c][k](
|
||||
float(shape_raw[k]), t_now)
|
||||
for k in range(expr_n):
|
||||
expr_smooth[k] = self._smooth_expr[pid_c][k](
|
||||
float(expr_raw[k]), t_now)
|
||||
|
||||
persons.append(SMPLXPerson(
|
||||
pid=int(pid),
|
||||
vertices_3d=np.ascontiguousarray(v3d, dtype=np.float32),
|
||||
translation=np.ascontiguousarray(
|
||||
transl_np[:3], dtype=np.float32),
|
||||
confidence=float(hh["scores"].item()),
|
||||
betas=np.ascontiguousarray(shape_smooth, dtype=np.float32),
|
||||
expression=np.ascontiguousarray(expr_smooth, dtype=np.float32),
|
||||
))
|
||||
|
||||
with self.state.lock():
|
||||
self.state.persons_smplx = persons
|
||||
self.state.smplx_last_t = t_now
|
||||
|
||||
t_end = time.monotonic()
|
||||
dt_total = (t_end - t_cap_start) * 1e3
|
||||
if LOG.isEnabledFor(logging.DEBUG) or dt_total > 100.0:
|
||||
LOG.log(
|
||||
logging.DEBUG if dt_total <= 100.0 else logging.WARNING,
|
||||
"frame[coreml]: cap=%.1f pre=%.1f inf=%.1f "
|
||||
"post=%.1fms total=%.1fms",
|
||||
(t_pre_start - t_cap_start) * 1e3,
|
||||
(t_inf_start - t_pre_start) * 1e3,
|
||||
(t_post_start - t_inf_start) * 1e3,
|
||||
(t_end - t_post_start) * 1e3,
|
||||
dt_total,
|
||||
)
|
||||
|
||||
dt = time.monotonic() - t_cap_start
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
|
||||
cap.stop()
|
||||
LOG.info("multi_hmr coreml worker stopped")
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Multi-HMR CoreML backend (ANE/GPU/CPU via Apple's CoreML framework).
|
||||
|
||||
Python 3.14 cannot use `coremltools.MLModel` because `libcoremlpython`
|
||||
and `libmilstoragepython` native extensions are not distributed for
|
||||
3.14. We load CoreML.framework directly via `objc.loadBundle()` —
|
||||
same pattern as `coreml_pose.py`.
|
||||
|
||||
Unlike `coreml_pose.py`, this backend does NOT use Vision: Vision is
|
||||
limited to image inputs and cannot feed a second MLMultiArray (cam_K).
|
||||
We invoke `MLModel.predictionFromFeatures:error:` directly with a
|
||||
`MLDictionaryFeatureProvider` wrapping two `MLMultiArray`s.
|
||||
|
||||
Public API:
|
||||
backend = MultiHMRCoreMLBackend(mlpackage_path)
|
||||
humans = backend.infer(image_chw_f32, K_33_f32, det_thresh=0.3)
|
||||
# humans is a list[dict] with the same keys as the PyTorch model
|
||||
# output. Values are CoreMLArray instances that quack like torch
|
||||
# tensors (.detach().cpu().numpy() / .item()).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import objc
|
||||
from Foundation import NSURL
|
||||
|
||||
LOG = logging.getLogger("multihmr_coreml")
|
||||
|
||||
DEFAULT_MLPACKAGE = (
|
||||
Path.home() / ".cache" / "av-live-multihmr"
|
||||
/ "multihmr_full_672_s.mlpackage"
|
||||
)
|
||||
|
||||
# Multi-HMR exported with apply_topk(K=4): outputs are fixed shape.
|
||||
N_PERSONS_FIXED = 4
|
||||
N_VERTS = 10475
|
||||
|
||||
# CoreML output names from the exported .mlpackage.
|
||||
OUT_V3D = "var_2420" # (4, 10475, 3)
|
||||
OUT_TRANSL = "var_2423" # (4, 1, 3)
|
||||
OUT_SCORES = "var_2436" # (4,)
|
||||
OUT_BETAS = "var_2439" # (4, 10)
|
||||
OUT_EXPR = "var_2442" # (4, 10)
|
||||
OUT_JOINTS = "var_2445" # (4, 127, 3) SMPL-X joints incl. fingers
|
||||
|
||||
# MLMultiArrayDataType raw values (from CoreML headers).
|
||||
ML_DTYPE_FLOAT32 = 65568
|
||||
ML_DTYPE_FLOAT16 = 65552
|
||||
ML_DTYPE_DOUBLE = 65600
|
||||
ML_DTYPE_INT32 = 131104
|
||||
|
||||
|
||||
_NS: dict[str, Any] = {}
|
||||
_FRAMEWORKS_LOADED = False
|
||||
|
||||
|
||||
def _load_frameworks() -> dict[str, Any]:
|
||||
global _FRAMEWORKS_LOADED
|
||||
if _FRAMEWORKS_LOADED:
|
||||
return _NS
|
||||
objc.loadBundle("CoreML", _NS,
|
||||
"/System/Library/Frameworks/CoreML.framework")
|
||||
_FRAMEWORKS_LOADED = True
|
||||
return _NS
|
||||
|
||||
|
||||
class CoreMLArray:
|
||||
"""Tiny tensor-like adapter so the existing worker hot path can
|
||||
treat CoreML outputs the same way it treats torch tensors.
|
||||
|
||||
Supports `.detach().cpu().numpy()` and `.item()`. The wrapper is
|
||||
a no-op around a numpy array; we keep the chain so callers don't
|
||||
need any conditional branch."""
|
||||
|
||||
__slots__ = ("_arr",)
|
||||
|
||||
def __init__(self, arr: np.ndarray) -> None:
|
||||
self._arr = arr
|
||||
|
||||
def detach(self) -> "CoreMLArray":
|
||||
return self
|
||||
|
||||
def cpu(self) -> "CoreMLArray":
|
||||
return self
|
||||
|
||||
def numpy(self) -> np.ndarray:
|
||||
return self._arr
|
||||
|
||||
def item(self) -> float:
|
||||
return float(self._arr.reshape(-1)[0])
|
||||
|
||||
@property
|
||||
def shape(self) -> tuple[int, ...]:
|
||||
return tuple(self._arr.shape)
|
||||
|
||||
|
||||
def _np_to_mlarray(arr: np.ndarray):
|
||||
"""Create a contiguous float32 MLMultiArray from a numpy array.
|
||||
|
||||
We always feed FLOAT32 — even though outputs are FLOAT16, CoreML
|
||||
will auto-cast on the input side."""
|
||||
ns = _load_frameworks()
|
||||
MLMultiArray = ns["MLMultiArray"]
|
||||
arr = np.ascontiguousarray(arr, dtype=np.float32)
|
||||
shape = [int(s) for s in arr.shape]
|
||||
ml = MLMultiArray.alloc().initWithShape_dataType_error_(
|
||||
shape, ML_DTYPE_FLOAT32, None)
|
||||
if ml is None:
|
||||
raise RuntimeError("MLMultiArray alloc failed")
|
||||
# Copy bytes through dataPointer (raw void*). pyobjc exposes it as
|
||||
# a memoryview-like opaque; we use ctypes to memcpy.
|
||||
import ctypes
|
||||
ptr = ml.dataPointer()
|
||||
n_bytes = arr.nbytes
|
||||
# pyobjc returns either an objc.varlist or a Python int pointer.
|
||||
addr = int(ptr) if isinstance(ptr, int) else ctypes.cast(
|
||||
ptr, ctypes.c_void_p).value
|
||||
if addr is None:
|
||||
raise RuntimeError("MLMultiArray dataPointer null")
|
||||
ctypes.memmove(addr, arr.ctypes.data, n_bytes)
|
||||
return ml
|
||||
|
||||
|
||||
def _mlarray_to_np(ml) -> np.ndarray:
|
||||
"""Copy an MLMultiArray (FLOAT16 or FLOAT32) into a numpy float32."""
|
||||
import ctypes
|
||||
shape = tuple(int(s) for s in ml.shape())
|
||||
dtype_id = int(ml.dataType())
|
||||
count = 1
|
||||
for s in shape:
|
||||
count *= s
|
||||
ptr = ml.dataPointer()
|
||||
addr = int(ptr) if isinstance(ptr, int) else ctypes.cast(
|
||||
ptr, ctypes.c_void_p).value
|
||||
if addr is None:
|
||||
raise RuntimeError("MLMultiArray dataPointer null")
|
||||
if dtype_id == ML_DTYPE_FLOAT16:
|
||||
raw = (ctypes.c_uint16 * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).view(np.float16).astype(np.float32)
|
||||
elif dtype_id == ML_DTYPE_FLOAT32:
|
||||
raw = (ctypes.c_float * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).copy()
|
||||
elif dtype_id == ML_DTYPE_DOUBLE:
|
||||
raw = (ctypes.c_double * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).astype(np.float32)
|
||||
else:
|
||||
raise RuntimeError(f"unsupported MLMultiArray dtype {dtype_id}")
|
||||
return arr.reshape(shape)
|
||||
|
||||
|
||||
class MultiHMRCoreMLBackend:
|
||||
"""CoreML inference wrapper for Multi-HMR (full_672_s)."""
|
||||
|
||||
def __init__(self, mlpackage_path: Path | None = None) -> None:
|
||||
self.path = Path(mlpackage_path) if mlpackage_path else DEFAULT_MLPACKAGE
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"mlpackage missing: {self.path}")
|
||||
ns = _load_frameworks()
|
||||
MLModel = ns["MLModel"]
|
||||
MLModelConfiguration = ns["MLModelConfiguration"]
|
||||
cfg = MLModelConfiguration.alloc().init()
|
||||
# MLComputeUnits: 0=CPUOnly, 1=CPUAndGPU, 2=All (ANE+GPU+CPU),
|
||||
# 3=CPUAndNeuralEngine. Bench M5 2026-05-14 (under live-worker
|
||||
# contention, 30 iter median, full Multi-HMR predict+copy):
|
||||
# CPU_AND_GPU = 252 ms (baseline)
|
||||
# ALL = 246 ms (within noise, ANE doesn't help)
|
||||
# CPU_AND_NE = 1301 ms (ANE solo catastrophic)
|
||||
# CPU_ONLY = 1152 ms
|
||||
# Standalone (no contention) FP32 = 139 ms = 7.2 fps. Default
|
||||
# stays CPU+GPU. Override with COREML_COMPUTE_UNITS env var
|
||||
# (`all`, `cpu_and_gpu`, `cpu_and_ne`, `cpu_only`) for A/B testing.
|
||||
cu_env = os.environ.get("COREML_COMPUTE_UNITS", "").strip().lower()
|
||||
cu_map = {"cpu_only": 0, "cpu_and_gpu": 1, "all": 2,
|
||||
"cpu_and_ne": 3}
|
||||
cu = cu_map.get(cu_env, 1)
|
||||
try:
|
||||
cfg.setComputeUnits_(cu)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
url = NSURL.fileURLWithPath_(str(self.path))
|
||||
# .mlpackage must be compiled to .mlmodelc before MLModel can
|
||||
# load it. compileModelAtURL_error_ returns an NSURL to a
|
||||
# temp .mlmodelc bundle.
|
||||
compiled_url = MLModel.compileModelAtURL_error_(url, None)
|
||||
if compiled_url is None:
|
||||
raise RuntimeError(f"compileModelAtURL failed for {self.path}")
|
||||
model = MLModel.modelWithContentsOfURL_configuration_error_(
|
||||
compiled_url, cfg, None)
|
||||
if model is None:
|
||||
raise RuntimeError(f"MLModel load failed for {compiled_url}")
|
||||
self._model = model
|
||||
self._ns = ns
|
||||
cu_name = {0: "CPU_ONLY", 1: "CPU+GPU", 2: "ALL", 3: "CPU+NE"}.get(
|
||||
cu, str(cu))
|
||||
LOG.info("Multi-HMR CoreML model loaded (%s, computeUnits=%s)",
|
||||
self.path.name, cu_name)
|
||||
|
||||
@staticmethod
|
||||
def is_available(mlpackage_path: Path | None = None) -> bool:
|
||||
p = Path(mlpackage_path) if mlpackage_path else DEFAULT_MLPACKAGE
|
||||
if not p.exists():
|
||||
return False
|
||||
try:
|
||||
_load_frameworks()
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
def _predict(self, image_4d: np.ndarray, K_33: np.ndarray) -> dict:
|
||||
ns = self._ns
|
||||
MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"]
|
||||
MLFeatureValue = ns["MLFeatureValue"]
|
||||
img_ml = _np_to_mlarray(image_4d)
|
||||
k_ml = _np_to_mlarray(K_33)
|
||||
feats = {
|
||||
"image": MLFeatureValue.featureValueWithMultiArray_(img_ml),
|
||||
"cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml),
|
||||
}
|
||||
provider = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
if provider is None:
|
||||
raise RuntimeError("MLDictionaryFeatureProvider alloc failed")
|
||||
out = self._model.predictionFromFeatures_error_(provider, None)
|
||||
if out is None:
|
||||
raise RuntimeError("MLModel predict failed")
|
||||
names = [str(n) for n in out.featureNames()]
|
||||
result = {}
|
||||
for n in names:
|
||||
fv = out.featureValueForName_(n)
|
||||
ml = fv.multiArrayValue()
|
||||
if ml is None:
|
||||
continue
|
||||
result[n] = _mlarray_to_np(ml)
|
||||
return result
|
||||
|
||||
def infer(
|
||||
self,
|
||||
image_chw_float32: np.ndarray,
|
||||
K_33: np.ndarray,
|
||||
det_thresh: float = 0.3,
|
||||
) -> list[dict]:
|
||||
"""Run a forward pass and return list of humans dicts.
|
||||
|
||||
Args:
|
||||
image_chw_float32: (3, 672, 672) or (1, 3, 672, 672) in [0,1].
|
||||
K_33: (3, 3) or (1, 3, 3) camera intrinsics.
|
||||
det_thresh: scores threshold; CoreML forwards K=4 always.
|
||||
|
||||
Returns:
|
||||
list[dict] with keys v3d, transl_pelvis, scores, shape,
|
||||
expression. Values are CoreMLArray wrappers.
|
||||
"""
|
||||
img = np.asarray(image_chw_float32, dtype=np.float32)
|
||||
if img.ndim == 3:
|
||||
img = img[np.newaxis, ...]
|
||||
if img.shape != (1, 3, 672, 672):
|
||||
raise ValueError(f"image shape {img.shape}, expected (1,3,672,672)")
|
||||
K = np.asarray(K_33, dtype=np.float32)
|
||||
if K.ndim == 2:
|
||||
K = K[np.newaxis, ...]
|
||||
if K.shape != (1, 3, 3):
|
||||
raise ValueError(f"K shape {K.shape}, expected (1,3,3)")
|
||||
|
||||
raw = self._predict(img, K)
|
||||
v3d = raw.get(OUT_V3D)
|
||||
transl = raw.get(OUT_TRANSL)
|
||||
scores = raw.get(OUT_SCORES)
|
||||
betas = raw.get(OUT_BETAS)
|
||||
expr = raw.get(OUT_EXPR)
|
||||
if any(x is None for x in (v3d, transl, scores, betas, expr)):
|
||||
raise RuntimeError(
|
||||
"missing outputs; got keys=" + ",".join(raw.keys()))
|
||||
|
||||
humans: list[dict] = []
|
||||
for k in range(N_PERSONS_FIXED):
|
||||
sc = float(scores[k])
|
||||
if sc < det_thresh:
|
||||
continue
|
||||
humans.append({
|
||||
"v3d": CoreMLArray(v3d[k]), # (10475, 3)
|
||||
"transl_pelvis": CoreMLArray(transl[k]), # (1, 3)
|
||||
"scores": CoreMLArray(np.array([sc], dtype=np.float32)),
|
||||
"shape": CoreMLArray(betas[k]), # (10,)
|
||||
"expression": CoreMLArray(expr[k]), # (10,)
|
||||
})
|
||||
return humans
|
||||
@@ -0,0 +1,460 @@
|
||||
"""Multi-HMR remote backend: drop-in replacement of MultiHMRCoreMLBackend
|
||||
that delegates inference to a remote TCP server (see
|
||||
``scripts/multihmr_server.py``).
|
||||
|
||||
Protocol (little-endian, persistent connection):
|
||||
|
||||
Request:
|
||||
[4B uint32 payload_len]
|
||||
[4B magic "REQ\x01"]
|
||||
[1B uint8 format_id] # 1 = raw RGB uint8 HWC, 2 = JPEG (variable length)
|
||||
[3B padding]
|
||||
[variable image bytes] # IMG_BYTES if format=1, else JPEG bytes
|
||||
[9 float32 K = 36 bytes]
|
||||
|
||||
The K block is *always* the last 36 bytes of the payload, regardless of
|
||||
``format_id`` — the server slices it off before treating the rest as the
|
||||
image.
|
||||
|
||||
Response:
|
||||
[4B uint32 payload_len]
|
||||
[4B magic "RSP\x01"]
|
||||
[4B int32 status]
|
||||
[v3d : 4*10475*3 f32]
|
||||
[transl: 4*1*3 f32]
|
||||
[scores: 4 f32]
|
||||
[betas: 4*10 f32]
|
||||
[expr : 4*10 f32]
|
||||
|
||||
Two extra features over the bare RPC:
|
||||
|
||||
* JPEG compression (``MULTIHMR_REMOTE_JPEG=1``, default ON, quality 80).
|
||||
Cuts wire bytes from ~1.35 MB to ~50-150 KB.
|
||||
|
||||
* Asynchronous double-buffer (``MULTIHMR_REMOTE_ASYNC=1``, default ON).
|
||||
``infer()`` is decoupled from the I/O round-trip via a dedicated worker
|
||||
thread and two ``Queue(maxsize=1)`` slots. When the out-queue is empty
|
||||
``infer()`` returns ``None`` — the worker loop reuses its last humans
|
||||
list so the visualiser keeps drawing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import socket
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
LOG = logging.getLogger("multihmr_remote")
|
||||
|
||||
IMG_SIZE = 672
|
||||
N_PERSONS_FIXED = 4
|
||||
N_VERTS = 10475
|
||||
|
||||
MAGIC_REQ = b"REQ\x01"
|
||||
MAGIC_RSP = b"RSP\x01"
|
||||
|
||||
FORMAT_RAW = 1
|
||||
FORMAT_JPEG = 2
|
||||
|
||||
IMG_BYTES = IMG_SIZE * IMG_SIZE * 3
|
||||
K_BYTES = 9 * 4
|
||||
REQ_HEADER = 4 + 1 + 3 # magic + format_id + 3-byte pad
|
||||
|
||||
V3D_BYTES = N_PERSONS_FIXED * N_VERTS * 3 * 4
|
||||
TRANSL_BYTES = N_PERSONS_FIXED * 1 * 3 * 4
|
||||
SCORES_BYTES = N_PERSONS_FIXED * 4
|
||||
BETAS_BYTES = N_PERSONS_FIXED * 10 * 4
|
||||
EXPR_BYTES = N_PERSONS_FIXED * 10 * 4
|
||||
RSP_HEADER = 4 + 4
|
||||
RSP_PAYLOAD_LEN = (RSP_HEADER + V3D_BYTES + TRANSL_BYTES
|
||||
+ SCORES_BYTES + BETAS_BYTES + EXPR_BYTES)
|
||||
|
||||
|
||||
def _env_flag(name: str, default: bool) -> bool:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None:
|
||||
return default
|
||||
return raw.strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _recv_exact(sock: socket.socket, n: int) -> bytes:
|
||||
buf = bytearray(n)
|
||||
view = memoryview(buf)
|
||||
pos = 0
|
||||
while pos < n:
|
||||
got = sock.recv_into(view[pos:])
|
||||
if got == 0:
|
||||
raise ConnectionError("peer closed mid-stream")
|
||||
pos += got
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def encode_request_raw(image_uint8_hwc: np.ndarray,
|
||||
K_33: np.ndarray) -> bytes:
|
||||
"""Raw uint8 HWC request (format_id=1, fixed payload length)."""
|
||||
if image_uint8_hwc.shape != (IMG_SIZE, IMG_SIZE, 3):
|
||||
raise ValueError(
|
||||
f"image shape {image_uint8_hwc.shape} != "
|
||||
f"({IMG_SIZE},{IMG_SIZE},3)")
|
||||
if image_uint8_hwc.dtype != np.uint8:
|
||||
raise ValueError(f"image dtype {image_uint8_hwc.dtype} != uint8")
|
||||
K = np.ascontiguousarray(K_33, dtype="<f4").reshape(9)
|
||||
img = np.ascontiguousarray(image_uint8_hwc, dtype=np.uint8)
|
||||
img_bytes = img.tobytes()
|
||||
header_after_magic = bytes([FORMAT_RAW, 0, 0, 0])
|
||||
payload_len = REQ_HEADER + len(img_bytes) + K_BYTES
|
||||
return b"".join([
|
||||
struct.pack("<I", payload_len),
|
||||
MAGIC_REQ,
|
||||
header_after_magic,
|
||||
img_bytes,
|
||||
K.tobytes(),
|
||||
])
|
||||
|
||||
|
||||
def encode_request_jpeg(jpeg_bytes: bytes, K_33: np.ndarray) -> bytes:
|
||||
"""JPEG request (format_id=2, variable payload length)."""
|
||||
K = np.ascontiguousarray(K_33, dtype="<f4").reshape(9)
|
||||
header_after_magic = bytes([FORMAT_JPEG, 0, 0, 0])
|
||||
payload_len = REQ_HEADER + len(jpeg_bytes) + K_BYTES
|
||||
return b"".join([
|
||||
struct.pack("<I", payload_len),
|
||||
MAGIC_REQ,
|
||||
header_after_magic,
|
||||
jpeg_bytes,
|
||||
K.tobytes(),
|
||||
])
|
||||
|
||||
|
||||
def decode_response(payload: bytes) -> tuple[
|
||||
np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, int]:
|
||||
if len(payload) != RSP_PAYLOAD_LEN:
|
||||
raise ValueError(
|
||||
f"rsp payload {len(payload)} != {RSP_PAYLOAD_LEN}")
|
||||
if payload[:4] != MAGIC_RSP:
|
||||
raise ValueError(f"bad rsp magic {payload[:4]!r}")
|
||||
status = struct.unpack("<i", payload[4:8])[0]
|
||||
off = 8
|
||||
v3d = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED * N_VERTS * 3,
|
||||
offset=off).reshape(N_PERSONS_FIXED, N_VERTS, 3)
|
||||
off += V3D_BYTES
|
||||
transl = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED * 1 * 3,
|
||||
offset=off).reshape(N_PERSONS_FIXED, 1, 3)
|
||||
off += TRANSL_BYTES
|
||||
scores = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED,
|
||||
offset=off).reshape(N_PERSONS_FIXED)
|
||||
off += SCORES_BYTES
|
||||
betas = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED * 10,
|
||||
offset=off).reshape(N_PERSONS_FIXED, 10)
|
||||
off += BETAS_BYTES
|
||||
expr = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED * 10,
|
||||
offset=off).reshape(N_PERSONS_FIXED, 10)
|
||||
return (v3d.copy(), transl.copy(), scores.copy(),
|
||||
betas.copy(), expr.copy(), int(status))
|
||||
|
||||
|
||||
# Back-compat shim — old call sites used encode_request(img, K) for raw.
|
||||
def encode_request(image_uint8_hwc: np.ndarray, K_33: np.ndarray) -> bytes:
|
||||
return encode_request_raw(image_uint8_hwc, K_33)
|
||||
|
||||
|
||||
class _Tensorlike:
|
||||
"""Mimics CoreMLArray to avoid a hard import on multihmr_coreml."""
|
||||
__slots__ = ("_arr",)
|
||||
|
||||
def __init__(self, arr: np.ndarray) -> None:
|
||||
self._arr = arr
|
||||
|
||||
def detach(self) -> "_Tensorlike":
|
||||
return self
|
||||
|
||||
def cpu(self) -> "_Tensorlike":
|
||||
return self
|
||||
|
||||
def numpy(self) -> np.ndarray:
|
||||
return self._arr
|
||||
|
||||
def item(self) -> float:
|
||||
return float(self._arr.reshape(-1)[0])
|
||||
|
||||
@property
|
||||
def shape(self) -> tuple[int, ...]:
|
||||
return tuple(self._arr.shape)
|
||||
|
||||
|
||||
def _humans_from_arrays(v3d: np.ndarray, transl: np.ndarray,
|
||||
scores: np.ndarray, betas: np.ndarray,
|
||||
expr: np.ndarray, det_thresh: float
|
||||
) -> list[dict[str, Any]]:
|
||||
humans: list[dict[str, Any]] = []
|
||||
for k in range(N_PERSONS_FIXED):
|
||||
sc = float(scores[k])
|
||||
if sc < det_thresh:
|
||||
continue
|
||||
humans.append({
|
||||
"v3d": _Tensorlike(v3d[k]),
|
||||
"transl_pelvis": _Tensorlike(transl[k]),
|
||||
"scores": _Tensorlike(np.array([sc], dtype=np.float32)),
|
||||
"shape": _Tensorlike(betas[k]),
|
||||
"expression": _Tensorlike(expr[k]),
|
||||
})
|
||||
return humans
|
||||
|
||||
|
||||
class MultiHMRRemoteBackend:
|
||||
"""TCP client backend mirroring ``MultiHMRCoreMLBackend.infer`` API.
|
||||
|
||||
JPEG compression and async double-buffering are toggleable via env
|
||||
(``MULTIHMR_REMOTE_JPEG``, ``MULTIHMR_REMOTE_ASYNC``).
|
||||
"""
|
||||
|
||||
def __init__(self, host: str = "192.168.0.175", port: int = 57140,
|
||||
connect_timeout: float = 3.0,
|
||||
io_timeout: float = 5.0) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.connect_timeout = connect_timeout
|
||||
self.io_timeout = io_timeout
|
||||
self._sock: socket.socket | None = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
self.use_jpeg = _env_flag("MULTIHMR_REMOTE_JPEG", True)
|
||||
self.jpeg_quality = int(os.environ.get(
|
||||
"MULTIHMR_REMOTE_JPEG_QUALITY", "80"))
|
||||
self.use_async = _env_flag("MULTIHMR_REMOTE_ASYNC", True)
|
||||
|
||||
# Async pipeline state.
|
||||
# Multi-buffer queues (2 in / 3 out) absorb jitter without
|
||||
# stalling capture. Drop-oldest semantics on overflow.
|
||||
self._in_q: queue.Queue[tuple[bytes, float, float]] = queue.Queue(
|
||||
maxsize=2)
|
||||
self._out_q: queue.Queue[
|
||||
tuple[list[dict[str, Any]], dict[str, float]]
|
||||
] = queue.Queue(maxsize=3)
|
||||
self._stop = threading.Event()
|
||||
self._async_det_thresh = 0.3
|
||||
self._worker_thread: threading.Thread | None = None
|
||||
self._last_stats: dict[str, float] = {}
|
||||
|
||||
if self.use_jpeg:
|
||||
try:
|
||||
import cv2 # noqa: F401
|
||||
except ImportError:
|
||||
LOG.warning("cv2 unavailable client-side, disabling JPEG")
|
||||
self.use_jpeg = False
|
||||
|
||||
if self.use_async:
|
||||
self._start_worker()
|
||||
LOG.info(
|
||||
"MultiHMRRemoteBackend %s:%d (jpeg=%s q=%d, async=%s)",
|
||||
host, port, self.use_jpeg, self.jpeg_quality, self.use_async)
|
||||
|
||||
# -- connection management -------------------------------------------
|
||||
|
||||
def _connect(self) -> socket.socket:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(self.connect_timeout)
|
||||
sock.connect((self.host, self.port))
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
sock.settimeout(self.io_timeout)
|
||||
LOG.info("connected to %s:%d", self.host, self.port)
|
||||
return sock
|
||||
|
||||
def _ensure_sock(self) -> socket.socket:
|
||||
if self._sock is None:
|
||||
self._sock = self._connect()
|
||||
return self._sock
|
||||
|
||||
def _drop_sock(self) -> None:
|
||||
if self._sock is not None:
|
||||
try:
|
||||
self._sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
self._sock = None
|
||||
|
||||
@staticmethod
|
||||
def is_available(host: str | None = None, port: int | None = None
|
||||
) -> bool:
|
||||
host = host or os.environ.get(
|
||||
"MULTIHMR_REMOTE_HOST", "192.168.0.175")
|
||||
port = port or int(os.environ.get(
|
||||
"MULTIHMR_REMOTE_PORT", "57140"))
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(1.0)
|
||||
s.connect((host, port))
|
||||
s.close()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
# -- request encoding ------------------------------------------------
|
||||
|
||||
def _encode_request_from_chw(
|
||||
self, image_chw_float32: np.ndarray, K_33: np.ndarray
|
||||
) -> tuple[bytes, float]:
|
||||
"""Return (request bytes, encode_ms)."""
|
||||
img = np.asarray(image_chw_float32, dtype=np.float32)
|
||||
if img.ndim == 4 and img.shape[0] == 1:
|
||||
img = img[0]
|
||||
if img.shape != (3, IMG_SIZE, IMG_SIZE):
|
||||
raise ValueError(
|
||||
f"image shape {img.shape} != (3,{IMG_SIZE},{IMG_SIZE})")
|
||||
img_hwc = np.clip(img.transpose(1, 2, 0) * 255.0, 0.0, 255.0
|
||||
).astype(np.uint8)
|
||||
K = np.asarray(K_33, dtype=np.float32)
|
||||
if K.ndim == 3 and K.shape[0] == 1:
|
||||
K = K[0]
|
||||
if K.shape != (3, 3):
|
||||
raise ValueError(f"K shape {K.shape} != (3,3)")
|
||||
|
||||
t0 = time.monotonic()
|
||||
if self.use_jpeg:
|
||||
import cv2 # local import to keep optional
|
||||
# cv2.imencode wants BGR for nicest JPEG perceptually but the
|
||||
# server decodes back to RGB ; encode RGB->BGR once for parity.
|
||||
bgr = cv2.cvtColor(img_hwc, cv2.COLOR_RGB2BGR)
|
||||
ok, enc = cv2.imencode(
|
||||
".jpg", bgr,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), self.jpeg_quality])
|
||||
if not ok:
|
||||
raise RuntimeError("cv2.imencode failed")
|
||||
req = encode_request_jpeg(bytes(enc), K)
|
||||
else:
|
||||
req = encode_request_raw(img_hwc, K)
|
||||
enc_ms = (time.monotonic() - t0) * 1e3
|
||||
return req, enc_ms
|
||||
|
||||
# -- synchronous fallback -------------------------------------------
|
||||
|
||||
def _send_recv(self, req: bytes) -> bytes:
|
||||
attempts = 0
|
||||
last_err: Exception | None = None
|
||||
while attempts < 2:
|
||||
attempts += 1
|
||||
try:
|
||||
sock = self._ensure_sock()
|
||||
sock.sendall(req)
|
||||
len_buf = _recv_exact(sock, 4)
|
||||
payload_len = struct.unpack("<I", len_buf)[0]
|
||||
if payload_len != RSP_PAYLOAD_LEN:
|
||||
raise ValueError(
|
||||
f"unexpected rsp len {payload_len}")
|
||||
return _recv_exact(sock, payload_len)
|
||||
except (ConnectionError, BrokenPipeError, OSError,
|
||||
socket.timeout) as e:
|
||||
LOG.warning("rpc failed (try %d): %s", attempts, e)
|
||||
self._drop_sock()
|
||||
last_err = e
|
||||
raise RuntimeError(f"remote inference failed: {last_err}")
|
||||
|
||||
# -- async worker ---------------------------------------------------
|
||||
|
||||
def _start_worker(self) -> None:
|
||||
self._worker_thread = threading.Thread(
|
||||
target=self._async_loop, name="multihmr-remote",
|
||||
daemon=True)
|
||||
self._worker_thread.start()
|
||||
|
||||
def _async_loop(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
req, t_submit, det_thresh = self._in_q.get(timeout=0.5)
|
||||
except queue.Empty:
|
||||
continue
|
||||
t_send = time.monotonic()
|
||||
try:
|
||||
with self._lock:
|
||||
payload = self._send_recv(req)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("async send_recv failed: %s", e)
|
||||
continue
|
||||
t_recv = time.monotonic()
|
||||
try:
|
||||
v3d, transl, scores, betas, expr, status = decode_response(
|
||||
payload)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("decode_response failed: %s", e)
|
||||
continue
|
||||
if status != 0:
|
||||
humans: list[dict[str, Any]] = []
|
||||
else:
|
||||
humans = _humans_from_arrays(
|
||||
v3d, transl, scores, betas, expr, det_thresh)
|
||||
stats = {
|
||||
"queue_wait_ms": (t_send - t_submit) * 1e3,
|
||||
"rpc_ms": (t_recv - t_send) * 1e3,
|
||||
}
|
||||
# Drop any pending stale output before pushing.
|
||||
try:
|
||||
self._out_q.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
try:
|
||||
self._out_q.put_nowait((humans, stats))
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
# -- public API -----------------------------------------------------
|
||||
|
||||
def infer(
|
||||
self,
|
||||
image_chw_float32: np.ndarray,
|
||||
K_33: np.ndarray,
|
||||
det_thresh: float = 0.3,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""In sync mode returns the humans list (possibly empty).
|
||||
|
||||
In async mode, submits the new frame (non-blocking, drop-newest
|
||||
if previous frame still in flight) and returns whatever output
|
||||
is ready in the out-queue. Returns ``None`` if nothing is ready
|
||||
yet — caller must reuse its last humans list.
|
||||
"""
|
||||
req, _enc_ms = self._encode_request_from_chw(
|
||||
image_chw_float32, K_33)
|
||||
|
||||
if not self.use_async:
|
||||
with self._lock:
|
||||
payload = self._send_recv(req)
|
||||
v3d, transl, scores, betas, expr, status = decode_response(
|
||||
payload)
|
||||
if status != 0:
|
||||
return []
|
||||
return _humans_from_arrays(
|
||||
v3d, transl, scores, betas, expr, det_thresh)
|
||||
|
||||
# Async path.
|
||||
self._async_det_thresh = det_thresh
|
||||
# drop-newest semantics: keep the freshest pending frame
|
||||
try:
|
||||
self._in_q.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
try:
|
||||
self._in_q.put_nowait((req, time.monotonic(), det_thresh))
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
try:
|
||||
humans, stats = self._out_q.get_nowait()
|
||||
except queue.Empty:
|
||||
return None
|
||||
self._last_stats = stats
|
||||
return humans
|
||||
|
||||
def close(self) -> None:
|
||||
self._stop.set()
|
||||
with self._lock:
|
||||
self._drop_sock()
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Worker NLF : capture webcam Mac, inference TorchScript multi-personne,
|
||||
extraction vertices 3D nonparametriques SMPL (6890 verts), ecriture State.
|
||||
|
||||
NLF (Sarandi, NeurIPS 2024) fournit des vertices directement via le path
|
||||
nonparametrique — pas besoin de modele SMPL-X externe. Le checkpoint
|
||||
TorchScript est auto-contenu : detecteur + estimateur multi-personne.
|
||||
|
||||
Cadence cible : 8-12 fps sur M5 (NLF-L). NLF-S pour > 15 fps.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .state import NLFPerson, State
|
||||
|
||||
LOG = logging.getLogger("nlf")
|
||||
|
||||
CACHE = Path.home() / ".cache" / "av-live-nlf"
|
||||
CKPT_L = CACHE / "nlf_l_multi.torchscript"
|
||||
CKPT_S = CACHE / "nlf_s_multi.torchscript"
|
||||
|
||||
N_VERTS = 6890
|
||||
N_JOINTS = 24
|
||||
|
||||
FAIL_THRESHOLD = 30 # ~1 s at 30 fps before giving up
|
||||
|
||||
|
||||
class NLFWorker:
|
||||
def __init__(self, state: State, num_persons: int = 4,
|
||||
target_fps: float = 10.0, device: str = "mps",
|
||||
use_small: bool = False) -> None:
|
||||
self.state = state
|
||||
self.num_persons = num_persons
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self.device = device
|
||||
self.ckpt_path = CKPT_S if use_small else CKPT_L
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._smooth_pos: list[list] = []
|
||||
self.failure_count = 0
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
return CKPT_L.exists() or CKPT_S.exists()
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="nlf", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def _record_success(self) -> None:
|
||||
self.failure_count = 0
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
import torch
|
||||
import cv2
|
||||
except ImportError as e:
|
||||
LOG.error("deps manquantes : %s — uv sync --extra nlf", e)
|
||||
return
|
||||
|
||||
if self.device == "mps" and not torch.backends.mps.is_available():
|
||||
LOG.warning("MPS unavailable, falling back to cpu")
|
||||
device = "cpu"
|
||||
else:
|
||||
device = self.device
|
||||
|
||||
if not self.ckpt_path.exists():
|
||||
if CKPT_L.exists():
|
||||
self.ckpt_path = CKPT_L
|
||||
elif CKPT_S.exists():
|
||||
self.ckpt_path = CKPT_S
|
||||
else:
|
||||
LOG.error("No NLF checkpoint found in %s", CACHE)
|
||||
return
|
||||
|
||||
try:
|
||||
model = torch.jit.load(
|
||||
str(self.ckpt_path), map_location=device).eval()
|
||||
except Exception as e:
|
||||
LOG.error("NLF load failed: %s", e)
|
||||
return
|
||||
ckpt_name = self.ckpt_path.stem
|
||||
LOG.info("NLF loaded (%s) on %s", ckpt_name, device)
|
||||
|
||||
from .euro_filter import OneEuroFilter
|
||||
from .tracker import IoUTracker
|
||||
from .state import PoseKp
|
||||
self._smooth_pos = [
|
||||
[OneEuroFilter(0.8, 0.05) for _ in range(3)]
|
||||
for _ in range(self.num_persons)
|
||||
]
|
||||
tracker = IoUTracker(iou_threshold=0.20, max_miss=8)
|
||||
|
||||
cap = cv2.VideoCapture(0)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index 0 indisponible")
|
||||
return
|
||||
LOG.info("camera ouverte")
|
||||
|
||||
while not self._stop.is_set():
|
||||
t0 = time.monotonic()
|
||||
ok, frame_bgr = cap.read()
|
||||
if not ok:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
tensor = torch.from_numpy(frame_rgb).permute(2, 0, 1)
|
||||
frame_batch = tensor.unsqueeze(0).to(device)
|
||||
|
||||
try:
|
||||
with torch.inference_mode():
|
||||
pred = model.detect_smpl_batched(frame_batch)
|
||||
except NotImplementedError as e:
|
||||
self.failure_count += 1
|
||||
if self.failure_count >= FAIL_THRESHOLD:
|
||||
LOG.error(
|
||||
"NLF inference unsupported on device=%s after %d frames: %s. "
|
||||
"TorchScript checkpoint is CUDA-only; install CUDA or switch backend.",
|
||||
device, self.failure_count, e,
|
||||
)
|
||||
return
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
except Exception as e:
|
||||
self.failure_count += 1
|
||||
if self.failure_count >= FAIL_THRESHOLD:
|
||||
LOG.error("NLF inference failed %d frames in a row, stopping: %s",
|
||||
self.failure_count, e)
|
||||
return
|
||||
LOG.warning("inference failed: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
self._record_success()
|
||||
|
||||
verts_all = pred.get("vertices3d_nonparam")
|
||||
joints_all = pred.get("joints3d_nonparam")
|
||||
trans_all = pred.get("trans")
|
||||
|
||||
if verts_all is None or len(verts_all) == 0:
|
||||
with self.state.lock():
|
||||
self.state.persons_nlf = []
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
verts_batch = verts_all[0]
|
||||
joints_batch = joints_all[0] if joints_all is not None else None
|
||||
trans_batch = trans_all[0] if trans_all is not None else None
|
||||
|
||||
n_detected = min(verts_batch.shape[0], self.num_persons)
|
||||
t_now = time.monotonic()
|
||||
|
||||
bboxes = []
|
||||
for i in range(n_detected):
|
||||
v = verts_batch[i].cpu().numpy()
|
||||
xmin, ymin = v[:, 0].min(), v[:, 1].min()
|
||||
xmax, ymax = v[:, 0].max(), v[:, 1].max()
|
||||
bboxes.append([PoseKp(x=float(xmin), y=float(ymin), c=1.0),
|
||||
PoseKp(x=float(xmax), y=float(ymax), c=1.0)])
|
||||
|
||||
ids = tracker.update(bboxes)
|
||||
|
||||
persons = []
|
||||
for i in range(n_detected):
|
||||
pid = ids[i] if i < len(ids) else i
|
||||
if pid < 0:
|
||||
continue
|
||||
|
||||
v_np = verts_batch[i].cpu().numpy()
|
||||
j_np = (joints_batch[i].cpu().numpy()
|
||||
if joints_batch is not None
|
||||
else np.zeros((N_JOINTS, 3), dtype=np.float32))
|
||||
t_np = (trans_batch[i].cpu().numpy()
|
||||
if trans_batch is not None
|
||||
else np.zeros(3, dtype=np.float32))
|
||||
|
||||
pid_c = pid % self.num_persons
|
||||
t_smooth = np.array([
|
||||
self._smooth_pos[pid_c][k](float(t_np[k]), t_now)
|
||||
for k in range(3)
|
||||
], dtype=np.float32)
|
||||
|
||||
persons.append(NLFPerson(
|
||||
pid=int(pid),
|
||||
vertices_3d=tuple(map(tuple, v_np)),
|
||||
joints_3d=tuple(map(tuple, j_np)),
|
||||
translation=tuple(t_smooth.tolist()),
|
||||
confidence=1.0,
|
||||
))
|
||||
|
||||
with self.state.lock():
|
||||
self.state.persons_nlf = persons
|
||||
self.state.nlf_last_t = t_now
|
||||
|
||||
dt = time.monotonic() - t0
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
|
||||
cap.release()
|
||||
LOG.info("nlf worker stopped")
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Thread OSC : ecoute UDP :57123 et alimente l'objet State.
|
||||
|
||||
Le pont data_feeds (Python) et SC poussent les messages /data/* /sync/*
|
||||
en parallele. On les agrege dans le State partage avec le renderer
|
||||
Metal.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from pythonosc import dispatcher, osc_server
|
||||
|
||||
from .state import State
|
||||
|
||||
LOG = logging.getLogger("osc")
|
||||
|
||||
|
||||
def _build(state: State) -> dispatcher.Dispatcher:
|
||||
d = dispatcher.Dispatcher()
|
||||
|
||||
# ---- /sync/* (SC -> visualizer) -----------------------------------
|
||||
def _bpm(addr, *args):
|
||||
with state.lock(): state.bpm = float(args[0]) if args else state.bpm
|
||||
def _beat(addr, *args):
|
||||
with state.lock(): state.beat = int(args[0]) if args else state.beat
|
||||
def _rms(addr, *args):
|
||||
with state.lock(): state.rms = float(args[0]) if args else state.rms
|
||||
def _amp(addr, *args):
|
||||
if len(args) >= 2:
|
||||
with state.lock(): state.amps[str(args[0])] = float(args[1])
|
||||
def _album(addr, *args):
|
||||
with state.lock(): state.album = str(args[0]) if args else state.album
|
||||
|
||||
d.map("/sync/bpm", _bpm)
|
||||
d.map("/sync/beat", _beat)
|
||||
d.map("/sync/rms", _rms)
|
||||
d.map("/sync/amp", _amp)
|
||||
d.map("/sync/album", _album)
|
||||
|
||||
# ---- /data/* (data_feeds bridge) ----------------------------------
|
||||
def _hb(addr, *_):
|
||||
with state.lock():
|
||||
state.bridge_alive = True
|
||||
state.last_heartbeat = time.monotonic()
|
||||
d.map("/data/heartbeat", _hb)
|
||||
|
||||
def _kp(addr, *args):
|
||||
if args:
|
||||
with state.lock(): state.swpc_kp = float(args[0])
|
||||
d.map("/data/swpc/kp", _kp)
|
||||
|
||||
def _flare(addr, *args):
|
||||
if len(args) >= 3:
|
||||
with state.lock(): state.swpc_flare_norm = float(args[2])
|
||||
d.map("/data/swpc/xray", _flare)
|
||||
|
||||
def _wind(addr, *args):
|
||||
if args:
|
||||
with state.lock(): state.swpc_wind_speed = float(args[0])
|
||||
d.map("/data/swpc/wind", _wind)
|
||||
|
||||
def _bz(addr, *args):
|
||||
if args:
|
||||
with state.lock(): state.swpc_bz = float(args[0])
|
||||
d.map("/data/swpc/bz", _bz)
|
||||
|
||||
def _netz(addr, *args):
|
||||
if args:
|
||||
with state.lock(): state.netz_dev = float(args[0])
|
||||
d.map("/data/netzfrequenz/dev", _netz)
|
||||
|
||||
def _strike(addr, *args):
|
||||
if len(args) >= 3:
|
||||
with state.lock():
|
||||
state.last_lightning = (float(args[0]), float(args[1]), float(args[2]))
|
||||
state.last_lightning_t = time.monotonic()
|
||||
d.map("/data/blitzortung/strike", _strike)
|
||||
|
||||
def _lrate(addr, *args):
|
||||
if args:
|
||||
with state.lock(): state.lightning_rate_min = float(args[0])
|
||||
d.map("/data/blitzortung/rate", _lrate)
|
||||
|
||||
def _quake(addr, *args):
|
||||
if args:
|
||||
with state.lock():
|
||||
state.usgs_last_mag = float(args[0])
|
||||
state.usgs_last_mag_t = time.monotonic()
|
||||
d.map("/data/usgs/event", _quake)
|
||||
|
||||
def _av_count(addr, *args):
|
||||
if args:
|
||||
with state.lock(): state.aviation_count = int(args[0])
|
||||
d.map("/data/opensky/count", _av_count)
|
||||
|
||||
def _social(addr, *args):
|
||||
if args:
|
||||
with state.lock(): state.social_rate = float(args[0])
|
||||
d.map("/data/bluesky/rate", _social)
|
||||
|
||||
# NOTE: sound_algo/control/data_feeds.scd also listens to /data/pose/{count,skel}
|
||||
# for sonification. Both consumers are intentional. Do NOT consolidate.
|
||||
def _pose_count(addr, *args):
|
||||
if args:
|
||||
with state.lock():
|
||||
state.pose_count = int(args[0])
|
||||
state.pose_last_t = time.monotonic()
|
||||
d.map("/data/pose/count", _pose_count)
|
||||
|
||||
# ---- Preset open-data (envoyé par launcher) -----------------
|
||||
def _preset(addr, *args):
|
||||
if args:
|
||||
with state.lock(): state.active_preset = str(args[0])
|
||||
LOG.info("preset -> %s", state.active_preset)
|
||||
d.map("/control/preset", _preset)
|
||||
|
||||
# ---- Mode visuel (changement live) ---------------------------
|
||||
def _viz_mode(addr, *args):
|
||||
if not args:
|
||||
return
|
||||
a = args[0]
|
||||
if isinstance(a, str):
|
||||
try:
|
||||
idx = state.viz_mode_names.index(a)
|
||||
except ValueError:
|
||||
LOG.warning("viz mode inconnu : %r", a)
|
||||
return
|
||||
else:
|
||||
idx = int(a)
|
||||
idx = max(0, min(7, idx))
|
||||
with state.lock(): state.viz_mode = idx
|
||||
LOG.info("viz mode -> %d (%s)", idx, state.viz_mode_names[idx])
|
||||
d.map("/control/vizMode", _viz_mode)
|
||||
|
||||
def _pose_skel(addr, *args):
|
||||
# idx, conf_avg, x0 y0 c0 ... x16 y16 c16
|
||||
if len(args) < 2 + 17 * 3:
|
||||
return
|
||||
idx = int(args[0])
|
||||
if idx != 0:
|
||||
return # on ne suit que le sujet 0 pour le rendu
|
||||
with state.lock():
|
||||
for k in range(17):
|
||||
off = 2 + k * 3
|
||||
kp = state.pose_kp[k]
|
||||
kp.x = float(args[off])
|
||||
kp.y = float(args[off + 1])
|
||||
kp.c = float(args[off + 2])
|
||||
state.pose_last_t = time.monotonic()
|
||||
d.map("/data/pose/skel", _pose_skel)
|
||||
|
||||
return d
|
||||
|
||||
|
||||
class OscListener:
|
||||
def __init__(self, state: State, host: str = "127.0.0.1", port: int = 57123):
|
||||
self.state = state
|
||||
self.host = host
|
||||
self.port = port
|
||||
self._server: osc_server.BlockingOSCUDPServer | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
d = _build(self.state)
|
||||
self._server = osc_server.ThreadingOSCUDPServer((self.host, self.port), d)
|
||||
self._thread = threading.Thread(
|
||||
target=self._server.serve_forever, name="osc", daemon=True)
|
||||
self._thread.start()
|
||||
LOG.info("listening on %s:%d", self.host, self.port)
|
||||
|
||||
def stop(self) -> None:
|
||||
if self._server:
|
||||
self._server.shutdown()
|
||||
self._server = None
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Webcam + YOLOv8-pose integre au visualizer Metal.
|
||||
|
||||
Pourquoi ici plutot que dans data_feeds/feeds/pose.py :
|
||||
- Un seul process = une seule webcam (macOS interdit la double ouverture)
|
||||
- GPU partage : inference MPS et rendu Metal sur le meme device
|
||||
- TCC : si data_only_viz est lance par le launcher bundle, le subprocess
|
||||
Python herite (au moins une fois) du contexte camera autorise.
|
||||
|
||||
Met a jour directement state.pose_kp[17] sous lock. Le shader Metal lit
|
||||
ces valeurs a chaque frame via renderer._update_skeleton.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .state import PoseKp, State
|
||||
|
||||
LOG = logging.getLogger("pose")
|
||||
|
||||
|
||||
class PoseWorker:
|
||||
"""Thread daemon de capture + inference."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
state: State,
|
||||
model_name: str = "yolov8n-pose.pt",
|
||||
device: str = "mps",
|
||||
camera_index: int = 0,
|
||||
target_fps: float = 20.0,
|
||||
conf_thresh: float = 0.35,
|
||||
max_persons: int = 4,
|
||||
) -> None:
|
||||
self.state = state
|
||||
self.model_name = model_name
|
||||
self.device = device
|
||||
self.camera_index = camera_index
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self.conf_thresh = conf_thresh
|
||||
self.max_persons = max_persons
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stop = threading.Event()
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="pose", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
import cv2
|
||||
import numpy as np
|
||||
from ultralytics import YOLO
|
||||
except ModuleNotFoundError as e:
|
||||
LOG.error("dependances manquantes : %s — uv sync --extra pose", e)
|
||||
return
|
||||
|
||||
LOG.info("loading %s on %s", self.model_name, self.device)
|
||||
try:
|
||||
model = YOLO(self.model_name)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("YOLO load failed: %s", e)
|
||||
return
|
||||
|
||||
cap = cv2.VideoCapture(self.camera_index)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index %d indisponible (TCC ?)", self.camera_index)
|
||||
return
|
||||
LOG.info("camera ouverte (index %d)", self.camera_index)
|
||||
|
||||
while not self._stop.is_set():
|
||||
t0 = time.monotonic()
|
||||
ok, frame = cap.read()
|
||||
if not ok or frame is None:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
h, w = frame.shape[:2]
|
||||
try:
|
||||
results = model.predict(
|
||||
frame, device=self.device, conf=self.conf_thresh,
|
||||
verbose=False, max_det=self.max_persons,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("inference failed: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
if not results:
|
||||
with self.state.lock():
|
||||
self.state.pose_count = 0
|
||||
self.state.pose_last_t = time.monotonic()
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
res = results[0]
|
||||
kp_xy = getattr(res.keypoints, "xy", None)
|
||||
kp_conf = getattr(res.keypoints, "conf", None)
|
||||
n = 0 if kp_xy is None else int(len(kp_xy))
|
||||
if n == 0:
|
||||
with self.state.lock():
|
||||
self.state.pose_count = 0
|
||||
self.state.pose_last_t = time.monotonic()
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# On suit le sujet 0 pour le squelette overlay (cf renderer).
|
||||
pts = kp_xy[0].cpu().numpy()
|
||||
cfs = (kp_conf[0].cpu().numpy() if kp_conf is not None
|
||||
else np.ones(len(pts), dtype=float))
|
||||
# Encode la frame en JPEG (qualite 70, ~30 ko) pour l'affichage
|
||||
# NSImageView. Cheaper que d'envoyer du raw a Metal et marche
|
||||
# peu importe le viz mode actif.
|
||||
ok2, jpg = cv2.imencode(".jpg", frame,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), 70])
|
||||
jpg_bytes = bytes(jpg) if ok2 else None
|
||||
with self.state.lock():
|
||||
self.state.pose_count = n
|
||||
for k in range(min(17, len(pts))):
|
||||
x, y = pts[k]
|
||||
self.state.pose_kp[k] = PoseKp(
|
||||
x=float(x) / max(1.0, w),
|
||||
y=float(y) / max(1.0, h),
|
||||
c=float(cfs[k]),
|
||||
)
|
||||
self.state.pose_last_t = time.monotonic()
|
||||
if jpg_bytes:
|
||||
self.state.last_webcam_jpeg = jpg_bytes
|
||||
|
||||
# cadence cible
|
||||
dt = time.monotonic() - t0
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
cap.release()
|
||||
LOG.info("pose worker stopped")
|
||||
@@ -0,0 +1,330 @@
|
||||
"""Pont sonore pose -> SC.
|
||||
|
||||
Envoie en OSC les coordonnees des keypoints saillants vers sclang
|
||||
(127.0.0.1:57121) pour qu'ils pilotent des synthdefs en temps reel.
|
||||
|
||||
Routes emises :
|
||||
/pose/count <n> nombre de personnes detectees
|
||||
/pose/center <pid> <cx> <cy> centre du corps (moyenne kp visibles)
|
||||
/pose/wrist <pid> <l|r> <x> <y> poignet gauche / droit (normalises)
|
||||
/pose/head <pid> <x> <y> <c> position du nez (visage)
|
||||
/pose/sho_span <pid> <dx> ecart epaules (estime distance camera)
|
||||
/pose/limb_span <pid> <span> envergure brassse (poignet a poignet)
|
||||
/face/count <n> nombre de visages detectes
|
||||
/face/kp <pid> <idx> <x> <y> <z> <c> 68-pt subset (dlib mapping)
|
||||
/hand/count <n_left> <n_right> nombre de mains gauche / droite
|
||||
/hand/kp <pid> <side[0=L|1=R]> <idx> <x> <y> <z> <c> 21 landmarks
|
||||
/pose3d/count <n> nombre de squelettes 3D
|
||||
/pose3d/kp <pid> <idx> <x> <y> <z> <c> 33 MediaPipe world landmarks (metres)
|
||||
|
||||
Mapping pose -> son est defini cote SC dans sound_algo/data_only/scenes.scd
|
||||
(scene `live_pose`). Face / hand keypoints are consumed by the Swift
|
||||
launcher (AVLiveBody) on 127.0.0.1:57126 for skeleton overlay rendering.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Iterable, Sequence
|
||||
|
||||
from pythonosc.udp_client import SimpleUDPClient
|
||||
|
||||
LOG = logging.getLogger("pose_bridge")
|
||||
|
||||
|
||||
# Indices MediaPipe POSE_LANDMARKS (cf JOINT_MAP dans apple_vision_pose.py)
|
||||
NOSE = 0
|
||||
LEFT_SHO = 11
|
||||
RIGHT_SHO = 12
|
||||
LEFT_WRIST = 15
|
||||
RIGHT_WRIST = 16
|
||||
LEFT_HIP = 23
|
||||
RIGHT_HIP = 24
|
||||
|
||||
|
||||
# MediaPipe FaceMesh (468 landmarks) -> 68-point dlib-style subset.
|
||||
# Mapping inspired by community references (Google MediaPipe ->
|
||||
# iBUG 68 facial landmarks). Order matches dlib 68-point convention :
|
||||
# [0..16] jaw contour (left to right)
|
||||
# [17..21] right brow
|
||||
# [22..26] left brow
|
||||
# [27..30] nose bridge (top to tip)
|
||||
# [31..35] nostril base (right to left)
|
||||
# [36..41] right eye (CCW from outer corner)
|
||||
# [42..47] left eye (CCW from inner corner)
|
||||
# [48..59] outer lip (CCW from right corner)
|
||||
# [60..67] inner lip (CCW from right corner)
|
||||
FACE_68_FROM_MP: tuple[int, ...] = (
|
||||
# Jaw (17)
|
||||
127, 234, 132, 172, 150, 176, 148, 152,
|
||||
377, 400, 365, 397, 361, 401, 366, 447, 356,
|
||||
# Right brow (5) — mediapipe perspective is mirrored vs subject
|
||||
70, 63, 105, 66, 107,
|
||||
# Left brow (5)
|
||||
336, 296, 334, 293, 300,
|
||||
# Nose bridge (4)
|
||||
168, 6, 197, 195,
|
||||
# Nostril base (5)
|
||||
98, 97, 2, 326, 327,
|
||||
# Right eye (6)
|
||||
33, 160, 158, 133, 153, 144,
|
||||
# Left eye (6)
|
||||
362, 385, 387, 263, 373, 380,
|
||||
# Outer lip (12)
|
||||
61, 39, 37, 0, 267, 269, 291, 405, 314, 17, 84, 181,
|
||||
# Inner lip (8)
|
||||
78, 81, 13, 311, 308, 402, 14, 178,
|
||||
)
|
||||
assert len(FACE_68_FROM_MP) == 68
|
||||
|
||||
|
||||
class PoseSoundBridge:
|
||||
"""Envoie les keypoints en OSC vers sclang. Throttle a 30 Hz max."""
|
||||
|
||||
def __init__(self, sclang_host: str = "127.0.0.1",
|
||||
sclang_port: int = 57121, throttle_hz: float = 30.0) -> None:
|
||||
self._client = SimpleUDPClient(sclang_host, sclang_port)
|
||||
# Broadcast secondaire vers AV-Live-Body (Swift) pour overlay
|
||||
# skeleton dans la fenetre RealityKit. Silent si pas connecte.
|
||||
import os as _os
|
||||
_avbody_host = _os.environ.get("AVBODY_HOST", "127.0.0.1")
|
||||
self._avbody = SimpleUDPClient(_avbody_host, 57126)
|
||||
self._period = 1.0 / max(1.0, throttle_hz)
|
||||
self._last_t = 0.0
|
||||
|
||||
def send(self, persons_body: list, persons_body_ids: list, t_now: float,
|
||||
*,
|
||||
persons_face: Sequence[Sequence[Any]] | None = None,
|
||||
persons_face_ids: Sequence[int] | None = None,
|
||||
persons_hands: Sequence[Sequence[Any]] | None = None,
|
||||
persons_hands_ids: Sequence[int] | None = None,
|
||||
persons_body3d: Sequence[Sequence[Any]] | None = None,
|
||||
persons_body3d_ids: Sequence[int] | None = None) -> None:
|
||||
"""Envoie les keypoints de toutes les personnes detectees.
|
||||
Throttle automatiquement. Face / hand sont optionnels et envoyes
|
||||
sur le meme socket :57126 vers AVLiveBody."""
|
||||
if t_now - self._last_t < self._period:
|
||||
return
|
||||
self._last_t = t_now
|
||||
|
||||
n = len(persons_body)
|
||||
try:
|
||||
self._client.send_message("/pose/count", [int(n)])
|
||||
try: self._avbody.send_message("/pose/count", [int(n)])
|
||||
except OSError: pass
|
||||
except OSError:
|
||||
return # SC pas la, on continue silencieusement
|
||||
|
||||
if n > 0:
|
||||
for i, body in enumerate(persons_body):
|
||||
pid = persons_body_ids[i] if i < len(persons_body_ids) else i
|
||||
self._emit_person(int(pid), body)
|
||||
|
||||
# Face / hand : independant de la presence de body kp (utile en
|
||||
# mode face-only ou hand-only).
|
||||
if persons_face is not None:
|
||||
self._send_face(persons_face, persons_face_ids or [])
|
||||
if persons_hands is not None:
|
||||
self._send_hand(persons_hands, persons_hands_ids or [])
|
||||
if persons_body3d is not None:
|
||||
self._send_body3d(persons_body3d, persons_body3d_ids or [])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _emit_person(self, pid: int, body: list) -> None:
|
||||
cli = self._client
|
||||
|
||||
# Centre = moyenne des kp visibles
|
||||
visible = [(kp.x, kp.y) for kp in body if kp.c > 0.3]
|
||||
if not visible:
|
||||
return
|
||||
cx = sum(p[0] for p in visible) / len(visible)
|
||||
cy = sum(p[1] for p in visible) / len(visible)
|
||||
cli.send_message("/pose/center", [pid, float(cx), float(cy)])
|
||||
try: self._avbody.send_message("/pose/center", [pid, float(cx), float(cy)])
|
||||
except OSError: pass
|
||||
|
||||
# Nez (visage) — important pour piloter une voix
|
||||
if len(body) > NOSE and body[NOSE].c > 0.3:
|
||||
cli.send_message("/pose/head", [
|
||||
pid, float(body[NOSE].x), float(body[NOSE].y),
|
||||
float(body[NOSE].c),
|
||||
])
|
||||
|
||||
# Poignets gauche/droit
|
||||
if len(body) > LEFT_WRIST and body[LEFT_WRIST].c > 0.3:
|
||||
cli.send_message("/pose/wrist", [
|
||||
pid, "l", float(body[LEFT_WRIST].x), float(body[LEFT_WRIST].y),
|
||||
])
|
||||
if len(body) > RIGHT_WRIST and body[RIGHT_WRIST].c > 0.3:
|
||||
cli.send_message("/pose/wrist", [
|
||||
pid, "r", float(body[RIGHT_WRIST].x), float(body[RIGHT_WRIST].y),
|
||||
])
|
||||
|
||||
# Ecart epaules (proxy distance camera : plus large = plus pres)
|
||||
if (len(body) > RIGHT_SHO
|
||||
and body[LEFT_SHO].c > 0.3 and body[RIGHT_SHO].c > 0.3):
|
||||
dx = abs(body[LEFT_SHO].x - body[RIGHT_SHO].x)
|
||||
cli.send_message("/pose/sho_span", [pid, float(dx)])
|
||||
try: self._avbody.send_message("/pose/sho_span", [pid, float(dx)])
|
||||
except OSError: pass
|
||||
|
||||
# Envergure poignets (mouvement expressif)
|
||||
if (len(body) > RIGHT_WRIST
|
||||
and body[LEFT_WRIST].c > 0.3 and body[RIGHT_WRIST].c > 0.3):
|
||||
span = ((body[LEFT_WRIST].x - body[RIGHT_WRIST].x) ** 2
|
||||
+ (body[LEFT_WRIST].y - body[RIGHT_WRIST].y) ** 2) ** 0.5
|
||||
cli.send_message("/pose/limb_span", [pid, float(span)])
|
||||
try: self._avbody.send_message("/pose/limb_span", [pid, float(span)])
|
||||
except OSError: pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def send_face(self, persons_face: Sequence[Sequence[Any]],
|
||||
persons_face_ids: Sequence[int], t_now: float,
|
||||
force: bool = False) -> None:
|
||||
"""Public throttled entry point for face keypoints.
|
||||
|
||||
Emits a 68-point dlib-style subset of the 468 MediaPipe FaceMesh
|
||||
landmarks per person on /face/count + /face/kp routes.
|
||||
"""
|
||||
if not force and (t_now - self._last_t) < self._period:
|
||||
return
|
||||
self._send_face(persons_face, persons_face_ids)
|
||||
|
||||
def send_hand(self, persons_hands: Sequence[Sequence[Any]],
|
||||
persons_hands_ids: Sequence[int], t_now: float,
|
||||
force: bool = False) -> None:
|
||||
"""Public throttled entry point for hand keypoints.
|
||||
|
||||
Emits the full 21-landmark hand skeleton per detected hand on
|
||||
/hand/count + /hand/kp routes. Side is inferred from id parity
|
||||
(MediaPipe Hand task does not flag left/right reliably) : we
|
||||
treat odd ids as right, even as left, which matches the
|
||||
convention used by the smoother / tracker upstream.
|
||||
"""
|
||||
if not force and (t_now - self._last_t) < self._period:
|
||||
return
|
||||
self._send_hand(persons_hands, persons_hands_ids)
|
||||
|
||||
def _send_face(self, persons_face: Sequence[Sequence[Any]],
|
||||
persons_face_ids: Sequence[int]) -> None:
|
||||
n = len(persons_face)
|
||||
try:
|
||||
self._avbody.send_message("/face/count", [int(n)])
|
||||
except OSError:
|
||||
return
|
||||
for i, face in enumerate(persons_face):
|
||||
if not face:
|
||||
continue
|
||||
pid = persons_face_ids[i] if i < len(persons_face_ids) else i
|
||||
n_lm = len(face)
|
||||
for slot, mp_idx in enumerate(FACE_68_FROM_MP):
|
||||
if mp_idx >= n_lm:
|
||||
continue
|
||||
kp = face[mp_idx]
|
||||
try:
|
||||
self._avbody.send_message("/face/kp", [
|
||||
int(pid), int(slot),
|
||||
float(kp.x), float(kp.y),
|
||||
float(getattr(kp, "z", 0.0)),
|
||||
float(getattr(kp, "c", 1.0)),
|
||||
])
|
||||
except OSError:
|
||||
return
|
||||
|
||||
def _send_hand(self, persons_hands: Sequence[Sequence[Any]],
|
||||
persons_hands_ids: Sequence[int]) -> None:
|
||||
n_left = 0
|
||||
n_right = 0
|
||||
for i in range(len(persons_hands)):
|
||||
pid = persons_hands_ids[i] if i < len(persons_hands_ids) else i
|
||||
if int(pid) % 2 == 0:
|
||||
n_left += 1
|
||||
else:
|
||||
n_right += 1
|
||||
try:
|
||||
self._avbody.send_message("/hand/count", [int(n_left), int(n_right)])
|
||||
except OSError:
|
||||
return
|
||||
for i, hand in enumerate(persons_hands):
|
||||
if not hand:
|
||||
continue
|
||||
pid = persons_hands_ids[i] if i < len(persons_hands_ids) else i
|
||||
side = 1 if int(pid) % 2 else 0
|
||||
for idx, kp in enumerate(hand[:21]):
|
||||
try:
|
||||
self._avbody.send_message("/hand/kp", [
|
||||
int(pid), int(side), int(idx),
|
||||
float(kp.x), float(kp.y),
|
||||
float(getattr(kp, "z", 0.0)),
|
||||
float(getattr(kp, "c", 1.0)),
|
||||
])
|
||||
except OSError:
|
||||
return
|
||||
|
||||
def send_body3d(self, persons_body3d: Sequence[Sequence[Any]],
|
||||
persons_body3d_ids: Sequence[int], t_now: float,
|
||||
force: bool = False) -> None:
|
||||
"""Public throttled entry point for 3D body world landmarks.
|
||||
|
||||
Emits 33 MediaPipe pose_world_landmarks per person on
|
||||
/pose3d/count + /pose3d/kp routes. Coordinates are in meters,
|
||||
relative to the hip-center (MediaPipe convention: x=right,
|
||||
y=down, z=forward from the camera).
|
||||
"""
|
||||
if not force and (t_now - self._last_t) < self._period:
|
||||
return
|
||||
self._send_body3d(persons_body3d, persons_body3d_ids)
|
||||
|
||||
def _send_body3d(self, persons_body3d: Sequence[Sequence[Any]],
|
||||
persons_body3d_ids: Sequence[int]) -> None:
|
||||
n = len(persons_body3d)
|
||||
try:
|
||||
self._avbody.send_message("/pose3d/count", [int(n)])
|
||||
except OSError:
|
||||
return
|
||||
for i, body in enumerate(persons_body3d):
|
||||
if not body:
|
||||
continue
|
||||
pid = persons_body3d_ids[i] if i < len(persons_body3d_ids) else i
|
||||
for idx, kp in enumerate(body[:33]):
|
||||
try:
|
||||
self._avbody.send_message("/pose3d/kp", [
|
||||
int(pid), int(idx),
|
||||
float(kp.x), float(kp.y),
|
||||
float(getattr(kp, "z", 0.0)),
|
||||
float(getattr(kp, "c", 1.0)),
|
||||
])
|
||||
except OSError:
|
||||
return
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def send_action(self, pid: int, label_idx: int,
|
||||
probs, t_now: float, force: bool = False) -> None:
|
||||
"""Send action classification result via /pose/action OSC route.
|
||||
|
||||
Sends: [pid (int), label_idx (int), prob_0 (float), prob_1 (float), prob_2 (float)]
|
||||
"""
|
||||
if not force and (t_now - self._last_t) < self._period:
|
||||
return
|
||||
p = [float(probs[0]), float(probs[1]), float(probs[2])]
|
||||
self._client.send_message("/pose/action", [int(pid), int(label_idx), *p])
|
||||
|
||||
def send_kin(self, pid: int, kin,
|
||||
t_now: float, force: bool = False) -> None:
|
||||
"""Send kinematic angles via /pose/kin OSC route.
|
||||
|
||||
Sends: [pid (int), kin_0 (float), kin_1 (float), kin_2 (float)]
|
||||
"""
|
||||
if not force and (t_now - self._last_t) < self._period:
|
||||
return
|
||||
self._client.send_message(
|
||||
"/pose/kin",
|
||||
[int(pid), float(kin[0]), float(kin[1]), float(kin[2])],
|
||||
)
|
||||
|
||||
def send_enter(self, pid: int) -> None:
|
||||
"""Send lifecycle event when person enters frame."""
|
||||
self._client.send_message("/pose/enter", [int(pid)])
|
||||
|
||||
def send_leave(self, pid: int) -> None:
|
||||
"""Send lifecycle event when person leaves frame."""
|
||||
self._client.send_message("/pose/leave", [int(pid)])
|
||||
@@ -0,0 +1,730 @@
|
||||
"""3D pose filtering chain : median spike removal, Kalman CV smoothing,
|
||||
spring-damper organic inertia, lookahead extrapolation, IK angular clamps.
|
||||
|
||||
Operates on lists of Kp3D (metric, hip-centered) keyed by track id.
|
||||
|
||||
Stages are toggleable via the POSE_FILTER env var :
|
||||
POSE_FILTER=median+kalman+lookahead+ik (default)
|
||||
POSE_FILTER=median
|
||||
POSE_FILTER=off
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Iterable
|
||||
|
||||
from .state import Kp3D, State
|
||||
|
||||
LOG = logging.getLogger("pose_filter")
|
||||
|
||||
NUM_JOINTS = 33
|
||||
DEFAULT_STAGES = ("median", "kalman", "lookahead", "ik")
|
||||
ALL_STAGES = ("median", "kalman", "spring", "lookahead", "ik")
|
||||
|
||||
# MediaPipe POSE_LANDMARKS indices used by IK constraints.
|
||||
L_SHOULDER, R_SHOULDER = 11, 12
|
||||
L_ELBOW, R_ELBOW = 13, 14
|
||||
L_WRIST, R_WRIST = 15, 16
|
||||
L_HIP, R_HIP = 23, 24
|
||||
L_KNEE, R_KNEE = 25, 26
|
||||
L_ANKLE, R_ANKLE = 27, 28
|
||||
L_FOOT, R_FOOT = 31, 32
|
||||
|
||||
# (parent_idx, joint_idx, child_idx, min_deg, max_deg)
|
||||
JOINT_LIMITS: tuple[tuple[int, int, int, float, float], ...] = (
|
||||
(L_SHOULDER, L_ELBOW, L_WRIST, 0.0, 175.0),
|
||||
(R_SHOULDER, R_ELBOW, R_WRIST, 0.0, 175.0),
|
||||
(L_HIP, L_KNEE, L_ANKLE, 0.0, 175.0),
|
||||
(R_HIP, R_KNEE, R_ANKLE, 0.0, 175.0),
|
||||
(L_KNEE, L_ANKLE, L_FOOT, 60.0, 135.0),
|
||||
(R_KNEE, R_ANKLE, R_FOOT, 60.0, 135.0),
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------- utilities --------------------------------
|
||||
|
||||
def _is_finite(v: float) -> bool:
|
||||
return v == v and v not in (float("inf"), float("-inf"))
|
||||
|
||||
|
||||
def _kp_finite(kp: Kp3D) -> bool:
|
||||
return _is_finite(kp.x) and _is_finite(kp.y) and _is_finite(kp.z)
|
||||
|
||||
|
||||
def _median(values: list[float]) -> float:
|
||||
s = sorted(values)
|
||||
n = len(s)
|
||||
if n == 0:
|
||||
return 0.0
|
||||
if n % 2 == 1:
|
||||
return s[n // 2]
|
||||
return 0.5 * (s[n // 2 - 1] + s[n // 2])
|
||||
|
||||
|
||||
def _std(values: list[float], mu: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
var = sum((v - mu) ** 2 for v in values) / len(values)
|
||||
return math.sqrt(var)
|
||||
|
||||
|
||||
# ----------------------------- median filter ----------------------------
|
||||
|
||||
class MedianFilter:
|
||||
"""Per (pid, joint) ring buffer ; replaces spikes outside 3σ by median."""
|
||||
|
||||
def __init__(self, window: int = 3) -> None:
|
||||
self.window = max(1, window)
|
||||
self._buf: dict[tuple[int, int], deque[tuple[float, float, float]]] = {}
|
||||
|
||||
def reset(self) -> None:
|
||||
self._buf.clear()
|
||||
|
||||
def apply(self, pid: int, joint_idx: int, x: float, y: float, z: float
|
||||
) -> tuple[float, float, float]:
|
||||
key = (pid, joint_idx)
|
||||
buf = self._buf.get(key)
|
||||
if buf is None:
|
||||
buf = deque(maxlen=self.window)
|
||||
self._buf[key] = buf
|
||||
|
||||
# Spike detection requires history.
|
||||
out = (x, y, z)
|
||||
if not (_is_finite(x) and _is_finite(y) and _is_finite(z)):
|
||||
if buf:
|
||||
med = (_median([v[0] for v in buf]),
|
||||
_median([v[1] for v in buf]),
|
||||
_median([v[2] for v in buf]))
|
||||
out = med
|
||||
else:
|
||||
out = (0.0, 0.0, 0.0)
|
||||
elif len(buf) >= self.window:
|
||||
for axis_idx, val in enumerate(out):
|
||||
col = [v[axis_idx] for v in buf]
|
||||
med = _median(col)
|
||||
sigma = _std(col, med)
|
||||
if sigma > 1e-6 and abs(val - med) > 3.0 * sigma:
|
||||
out = tuple(med if i == axis_idx else out[i]
|
||||
for i in range(3)) # type: ignore[assignment]
|
||||
buf.append(out)
|
||||
return out
|
||||
|
||||
|
||||
# ----------------------------- Kalman CV --------------------------------
|
||||
|
||||
@dataclass
|
||||
class _KalmanState:
|
||||
# State vector [x, y, z, vx, vy, vz]
|
||||
x: list[float] = field(default_factory=lambda: [0.0] * 6)
|
||||
# 6x6 covariance flattened
|
||||
P: list[list[float]] = field(default_factory=lambda: [[0.0] * 6 for _ in range(6)])
|
||||
initialised: bool = False
|
||||
last_t: float = 0.0
|
||||
|
||||
|
||||
def _mat_eye(n: int, s: float = 1.0) -> list[list[float]]:
|
||||
return [[s if i == j else 0.0 for j in range(n)] for i in range(n)]
|
||||
|
||||
|
||||
def _mat_mul(A: list[list[float]], B: list[list[float]]) -> list[list[float]]:
|
||||
ra, ca = len(A), len(A[0])
|
||||
cb = len(B[0])
|
||||
out = [[0.0] * cb for _ in range(ra)]
|
||||
for i in range(ra):
|
||||
Ai = A[i]
|
||||
for k in range(ca):
|
||||
aik = Ai[k]
|
||||
if aik == 0.0:
|
||||
continue
|
||||
Bk = B[k]
|
||||
for j in range(cb):
|
||||
out[i][j] += aik * Bk[j]
|
||||
return out
|
||||
|
||||
|
||||
def _mat_add(A: list[list[float]], B: list[list[float]]) -> list[list[float]]:
|
||||
return [[A[i][j] + B[i][j] for j in range(len(A[0]))] for i in range(len(A))]
|
||||
|
||||
|
||||
def _mat_sub(A: list[list[float]], B: list[list[float]]) -> list[list[float]]:
|
||||
return [[A[i][j] - B[i][j] for j in range(len(A[0]))] for i in range(len(A))]
|
||||
|
||||
|
||||
def _mat_T(A: list[list[float]]) -> list[list[float]]:
|
||||
return [[A[i][j] for i in range(len(A))] for j in range(len(A[0]))]
|
||||
|
||||
|
||||
def _mat_inv3(M: list[list[float]]) -> list[list[float]]:
|
||||
a, b, c = M[0]
|
||||
d, e, f = M[1]
|
||||
g, h, i = M[2]
|
||||
A = e * i - f * h
|
||||
B = -(d * i - f * g)
|
||||
C = d * h - e * g
|
||||
det = a * A + b * B + c * C
|
||||
if abs(det) < 1e-12:
|
||||
return _mat_eye(3, 1.0)
|
||||
inv_det = 1.0 / det
|
||||
return [
|
||||
[A * inv_det, -(b * i - c * h) * inv_det, (b * f - c * e) * inv_det],
|
||||
[B * inv_det, (a * i - c * g) * inv_det, -(a * f - c * d) * inv_det],
|
||||
[C * inv_det, -(a * h - b * g) * inv_det, (a * e - b * d) * inv_det],
|
||||
]
|
||||
|
||||
|
||||
class KalmanCV:
|
||||
"""Constant-velocity Kalman per (pid, joint_idx) on R^3."""
|
||||
|
||||
def __init__(self, q: float = 1e-3, r: float = 1e-2) -> None:
|
||||
self.q = q
|
||||
self.r = r
|
||||
self._states: dict[tuple[int, int], _KalmanState] = {}
|
||||
self._H = [
|
||||
[1.0, 0.0, 0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 1.0, 0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 0.0, 1.0, 0.0, 0.0, 0.0],
|
||||
]
|
||||
|
||||
def reset(self) -> None:
|
||||
self._states.clear()
|
||||
|
||||
def get_velocity(self, pid: int, joint_idx: int) -> tuple[float, float, float]:
|
||||
st = self._states.get((pid, joint_idx))
|
||||
if st is None or not st.initialised:
|
||||
return (0.0, 0.0, 0.0)
|
||||
return (st.x[3], st.x[4], st.x[5])
|
||||
|
||||
def step(self, pid: int, joint_idx: int, mx: float, my: float, mz: float,
|
||||
t_now: float) -> tuple[float, float, float]:
|
||||
key = (pid, joint_idx)
|
||||
st = self._states.get(key)
|
||||
if st is None:
|
||||
st = _KalmanState()
|
||||
self._states[key] = st
|
||||
|
||||
if not st.initialised:
|
||||
st.x = [mx, my, mz, 0.0, 0.0, 0.0]
|
||||
st.P = _mat_eye(6, 1.0)
|
||||
st.initialised = True
|
||||
st.last_t = t_now
|
||||
return (mx, my, mz)
|
||||
|
||||
dt = max(1e-3, min(0.2, t_now - st.last_t))
|
||||
st.last_t = t_now
|
||||
|
||||
# Predict
|
||||
F = _mat_eye(6, 1.0)
|
||||
F[0][3] = dt
|
||||
F[1][4] = dt
|
||||
F[2][5] = dt
|
||||
x_pred = [
|
||||
st.x[0] + dt * st.x[3],
|
||||
st.x[1] + dt * st.x[4],
|
||||
st.x[2] + dt * st.x[5],
|
||||
st.x[3], st.x[4], st.x[5],
|
||||
]
|
||||
Q = _mat_eye(6, self.q)
|
||||
P_pred = _mat_add(_mat_mul(_mat_mul(F, st.P), _mat_T(F)), Q)
|
||||
|
||||
# Update
|
||||
z = [mx, my, mz]
|
||||
# y = z - H x_pred
|
||||
Hx = [x_pred[0], x_pred[1], x_pred[2]]
|
||||
y = [z[i] - Hx[i] for i in range(3)]
|
||||
# S = H P H^T + R (3x3)
|
||||
HP = _mat_mul(self._H, P_pred)
|
||||
S = [[HP[i][j] for j in range(3)] for i in range(3)]
|
||||
# add HP*H^T rest cols (cols 3..5) -> 0 contribution since H rest zero
|
||||
for i in range(3):
|
||||
S[i][i] += self.r
|
||||
S_inv = _mat_inv3(S)
|
||||
# K = P H^T S^-1 (6x3)
|
||||
PHt = [[P_pred[i][j] for j in range(3)] for i in range(6)]
|
||||
K = _mat_mul(PHt, S_inv)
|
||||
# x = x_pred + K y
|
||||
x_new = [x_pred[i] + sum(K[i][j] * y[j] for j in range(3))
|
||||
for i in range(6)]
|
||||
# P = (I - K H) P_pred
|
||||
KH = [[K[i][0] if j == 0 else (K[i][1] if j == 1 else (K[i][2] if j == 2 else 0.0))
|
||||
for j in range(6)] for i in range(6)]
|
||||
I6 = _mat_eye(6, 1.0)
|
||||
st.P = _mat_mul(_mat_sub(I6, KH), P_pred)
|
||||
st.x = x_new
|
||||
return (x_new[0], x_new[1], x_new[2])
|
||||
|
||||
|
||||
# --------------------------- spring damper ------------------------------
|
||||
|
||||
class SpringDamper:
|
||||
"""Critically-tunable spring-damper per (pid, joint_idx) on R^3."""
|
||||
|
||||
def __init__(self, stiffness: float = 200.0, damping: float = 15.0,
|
||||
mass: float = 1.0, enabled: bool = True) -> None:
|
||||
self.k = stiffness
|
||||
self.c = damping
|
||||
self.m = max(1e-3, mass)
|
||||
self.enabled = enabled
|
||||
self._pos: dict[tuple[int, int], list[float]] = {}
|
||||
self._vel: dict[tuple[int, int], list[float]] = {}
|
||||
self._last_t: dict[tuple[int, int], float] = {}
|
||||
|
||||
def reset(self) -> None:
|
||||
self._pos.clear()
|
||||
self._vel.clear()
|
||||
self._last_t.clear()
|
||||
|
||||
def step(self, pid: int, joint_idx: int, tx: float, ty: float, tz: float,
|
||||
t_now: float) -> tuple[float, float, float]:
|
||||
if not self.enabled:
|
||||
return (tx, ty, tz)
|
||||
key = (pid, joint_idx)
|
||||
pos = self._pos.get(key)
|
||||
if pos is None:
|
||||
self._pos[key] = [tx, ty, tz]
|
||||
self._vel[key] = [0.0, 0.0, 0.0]
|
||||
self._last_t[key] = t_now
|
||||
return (tx, ty, tz)
|
||||
dt = max(1e-3, min(0.1, t_now - self._last_t[key]))
|
||||
self._last_t[key] = t_now
|
||||
vel = self._vel[key]
|
||||
target = (tx, ty, tz)
|
||||
for i in range(3):
|
||||
# F = k(target - pos) - c * vel
|
||||
f = self.k * (target[i] - pos[i]) - self.c * vel[i]
|
||||
a = f / self.m
|
||||
vel[i] += a * dt
|
||||
pos[i] += vel[i] * dt
|
||||
return (pos[0], pos[1], pos[2])
|
||||
|
||||
|
||||
# --------------------------- lookahead ----------------------------------
|
||||
|
||||
class LookaheadPredictor:
|
||||
"""Linear extrapolation using Kalman velocities, capped to avoid blow-ups."""
|
||||
|
||||
def __init__(self, lookahead_ms: float = 50.0, max_velocity: float = 5.0
|
||||
) -> None:
|
||||
self.lookahead_s = lookahead_ms / 1000.0
|
||||
self.max_v = max_velocity
|
||||
|
||||
def step(self, x: float, y: float, z: float,
|
||||
vx: float, vy: float, vz: float) -> tuple[float, float, float]:
|
||||
def clamp(v: float) -> float:
|
||||
if v > self.max_v:
|
||||
return self.max_v
|
||||
if v < -self.max_v:
|
||||
return -self.max_v
|
||||
return v
|
||||
dt = self.lookahead_s
|
||||
return (x + clamp(vx) * dt, y + clamp(vy) * dt, z + clamp(vz) * dt)
|
||||
|
||||
|
||||
# --------------------------- IK constraints -----------------------------
|
||||
|
||||
def _vec_sub(a: tuple[float, float, float], b: tuple[float, float, float]
|
||||
) -> tuple[float, float, float]:
|
||||
return (a[0] - b[0], a[1] - b[1], a[2] - b[2])
|
||||
|
||||
|
||||
def _vec_add(a: tuple[float, float, float], b: tuple[float, float, float]
|
||||
) -> tuple[float, float, float]:
|
||||
return (a[0] + b[0], a[1] + b[1], a[2] + b[2])
|
||||
|
||||
|
||||
def _vec_scale(a: tuple[float, float, float], s: float
|
||||
) -> tuple[float, float, float]:
|
||||
return (a[0] * s, a[1] * s, a[2] * s)
|
||||
|
||||
|
||||
def _vec_dot(a: tuple[float, float, float], b: tuple[float, float, float]
|
||||
) -> float:
|
||||
return a[0] * b[0] + a[1] * b[1] + a[2] * b[2]
|
||||
|
||||
|
||||
def _vec_norm(a: tuple[float, float, float]) -> float:
|
||||
return math.sqrt(_vec_dot(a, a))
|
||||
|
||||
|
||||
def _vec_normalize(a: tuple[float, float, float], eps: float = 1e-9
|
||||
) -> tuple[float, float, float]:
|
||||
n = _vec_norm(a)
|
||||
if n < eps:
|
||||
return (1.0, 0.0, 0.0)
|
||||
return (a[0] / n, a[1] / n, a[2] / n)
|
||||
|
||||
|
||||
def _slerp_dir(d_from: tuple[float, float, float],
|
||||
d_to: tuple[float, float, float],
|
||||
t: float) -> tuple[float, float, float]:
|
||||
"""Slerp between two unit-ish vectors."""
|
||||
a = _vec_normalize(d_from)
|
||||
b = _vec_normalize(d_to)
|
||||
cos_a = max(-1.0, min(1.0, _vec_dot(a, b)))
|
||||
ang = math.acos(cos_a)
|
||||
if ang < 1e-6:
|
||||
return a
|
||||
sa = math.sin(ang)
|
||||
if abs(sa) < 1e-6:
|
||||
# antiparallel : pick an arbitrary perpendicular, then rotate.
|
||||
ortho = (1.0, 0.0, 0.0) if abs(a[0]) < 0.9 else (0.0, 1.0, 0.0)
|
||||
# Gram-Schmidt
|
||||
d = _vec_dot(ortho, a)
|
||||
perp = (ortho[0] - d * a[0], ortho[1] - d * a[1], ortho[2] - d * a[2])
|
||||
perp = _vec_normalize(perp)
|
||||
# rotate a by t*pi around perp axis : Rodrigues for angle = t*pi
|
||||
theta = t * ang
|
||||
cs, sn = math.cos(theta), math.sin(theta)
|
||||
# cross(perp, a)
|
||||
cx = perp[1] * a[2] - perp[2] * a[1]
|
||||
cy = perp[2] * a[0] - perp[0] * a[2]
|
||||
cz = perp[0] * a[1] - perp[1] * a[0]
|
||||
dot_pa = _vec_dot(perp, a)
|
||||
return (a[0] * cs + cx * sn + perp[0] * dot_pa * (1 - cs),
|
||||
a[1] * cs + cy * sn + perp[1] * dot_pa * (1 - cs),
|
||||
a[2] * cs + cz * sn + perp[2] * dot_pa * (1 - cs))
|
||||
w1 = math.sin((1.0 - t) * ang) / sa
|
||||
w2 = math.sin(t * ang) / sa
|
||||
return (a[0] * w1 + b[0] * w2,
|
||||
a[1] * w1 + b[1] * w2,
|
||||
a[2] * w1 + b[2] * w2)
|
||||
|
||||
|
||||
class IKConstraints:
|
||||
"""Clamp interior joint angles for elbows, knees, ankles."""
|
||||
|
||||
def __init__(self, limits: Iterable[tuple[int, int, int, float, float]]
|
||||
= JOINT_LIMITS) -> None:
|
||||
self.limits = tuple(limits)
|
||||
|
||||
def apply(self, kps: list[Kp3D]) -> list[Kp3D]:
|
||||
if len(kps) < NUM_JOINTS:
|
||||
return kps
|
||||
out = list(kps)
|
||||
for parent_i, joint_i, child_i, min_deg, max_deg in self.limits:
|
||||
if max(parent_i, joint_i, child_i) >= len(out):
|
||||
continue
|
||||
p = (out[parent_i].x, out[parent_i].y, out[parent_i].z)
|
||||
j = (out[joint_i].x, out[joint_i].y, out[joint_i].z)
|
||||
c = (out[child_i].x, out[child_i].y, out[child_i].z)
|
||||
v_pj = _vec_sub(p, j) # from joint to parent
|
||||
v_cj = _vec_sub(c, j) # from joint to child
|
||||
n_pj = _vec_norm(v_pj)
|
||||
n_cj = _vec_norm(v_cj)
|
||||
if n_pj < 1e-6 or n_cj < 1e-6:
|
||||
continue
|
||||
cos_a = max(-1.0, min(1.0, _vec_dot(v_pj, v_cj) / (n_pj * n_cj)))
|
||||
ang_deg = math.degrees(math.acos(cos_a))
|
||||
min_r = math.radians(min_deg)
|
||||
max_r = math.radians(max_deg)
|
||||
target_r: float | None = None
|
||||
if ang_deg < min_deg:
|
||||
target_r = min_r
|
||||
elif ang_deg > max_deg:
|
||||
target_r = max_r
|
||||
if target_r is None:
|
||||
continue
|
||||
# Interpolate child direction toward parent direction (or away)
|
||||
# so the new angle matches target_r.
|
||||
cur_r = math.acos(cos_a)
|
||||
# t such that new_angle = (1-t)*cur + t*pi between dirs ; use slerp.
|
||||
# Find t in [0,1] s.t. slerp(d_cj, d_pj, t) makes angle = target_r
|
||||
# The angle between slerp result and d_pj is (1-t)*cur_r.
|
||||
# So target_r = (1 - t) * cur_r -> t = 1 - target_r / cur_r
|
||||
if cur_r < 1e-6:
|
||||
continue
|
||||
t = 1.0 - (target_r / cur_r)
|
||||
t = max(0.0, min(1.0, t))
|
||||
d_cj = _vec_normalize(v_cj)
|
||||
d_pj = _vec_normalize(v_pj)
|
||||
new_dir = _slerp_dir(d_cj, d_pj, t)
|
||||
new_child = _vec_add(j, _vec_scale(new_dir, n_cj))
|
||||
old = out[child_i]
|
||||
out[child_i] = Kp3D(x=new_child[0], y=new_child[1],
|
||||
z=new_child[2], c=old.c)
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------- chain wrapper ------------------------------
|
||||
|
||||
def _parse_env_stages() -> tuple[str, ...]:
|
||||
raw = os.environ.get("POSE_FILTER")
|
||||
if raw is None:
|
||||
return DEFAULT_STAGES
|
||||
raw = raw.strip().lower()
|
||||
if raw in ("off", "none", "0", "false"):
|
||||
return ()
|
||||
parts = tuple(p.strip() for p in raw.replace(",", "+").split("+") if p.strip())
|
||||
return tuple(p for p in parts if p in ALL_STAGES)
|
||||
|
||||
|
||||
class PoseFilterChain:
|
||||
"""Chain : median → kalman → spring → lookahead → ik."""
|
||||
|
||||
def __init__(self, state: State | None = None,
|
||||
enabled_stages: Iterable[str] | None = None) -> None:
|
||||
self.state = state
|
||||
if enabled_stages is None:
|
||||
stages = _parse_env_stages()
|
||||
else:
|
||||
stages = tuple(s for s in enabled_stages if s in ALL_STAGES)
|
||||
self.enabled = stages
|
||||
self.median = MedianFilter(window=3)
|
||||
self.kalman = KalmanCV()
|
||||
self.spring = SpringDamper(enabled="spring" in self.enabled)
|
||||
self.lookahead = LookaheadPredictor()
|
||||
self.ik = IKConstraints()
|
||||
self.last_apply_ms: float = 0.0
|
||||
LOG.info("PoseFilterChain stages=%s", self.enabled or ("off",))
|
||||
|
||||
def reset(self) -> None:
|
||||
self.median.reset()
|
||||
self.kalman.reset()
|
||||
self.spring.reset()
|
||||
|
||||
def apply(self, bodies3d: list[list[Kp3D]], ids: list[int],
|
||||
t_now: float) -> list[list[Kp3D]]:
|
||||
if not bodies3d or not self.enabled:
|
||||
self.last_apply_ms = 0.0
|
||||
return bodies3d
|
||||
t0 = time.perf_counter()
|
||||
out: list[list[Kp3D]] = []
|
||||
use_median = "median" in self.enabled
|
||||
use_kalman = "kalman" in self.enabled
|
||||
use_spring = "spring" in self.enabled
|
||||
use_lookahead = "lookahead" in self.enabled
|
||||
use_ik = "ik" in self.enabled
|
||||
|
||||
for body_i, kps in enumerate(bodies3d):
|
||||
pid = ids[body_i] if body_i < len(ids) else -1
|
||||
new_kps: list[Kp3D] = []
|
||||
for j_idx, kp in enumerate(kps):
|
||||
x, y, z, c = kp.x, kp.y, kp.z, kp.c
|
||||
if use_median:
|
||||
x, y, z = self.median.apply(pid, j_idx, x, y, z)
|
||||
if use_kalman:
|
||||
x, y, z = self.kalman.step(pid, j_idx, x, y, z, t_now)
|
||||
if use_spring:
|
||||
x, y, z = self.spring.step(pid, j_idx, x, y, z, t_now)
|
||||
if use_lookahead and use_kalman:
|
||||
vx, vy, vz = self.kalman.get_velocity(pid, j_idx)
|
||||
x, y, z = self.lookahead.step(x, y, z, vx, vy, vz)
|
||||
new_kps.append(Kp3D(x=x, y=y, z=z, c=c))
|
||||
if use_ik:
|
||||
new_kps = self.ik.apply(new_kps)
|
||||
out.append(new_kps)
|
||||
|
||||
self.last_apply_ms = (time.perf_counter() - t0) * 1000.0
|
||||
return out
|
||||
|
||||
# ---- Face / hand smoothing entry points ---------------------------
|
||||
def apply_face(self, faces: list[list], ids: list[int],
|
||||
t_now: float) -> list[list]:
|
||||
if not hasattr(self, "_face_chain"):
|
||||
self._face_chain = FaceFilterChain()
|
||||
return self._face_chain.apply(faces, ids, t_now)
|
||||
|
||||
def apply_hand(self, hands: list[list], ids: list[int],
|
||||
handedness: list[str] | None,
|
||||
t_now: float) -> list[list]:
|
||||
if not hasattr(self, "_hand_chain"):
|
||||
self._hand_chain = HandFilterChain()
|
||||
return self._hand_chain.apply(hands, ids, handedness, t_now)
|
||||
|
||||
|
||||
# ============================ face / hand =================================
|
||||
|
||||
# Face and hand filtering operate on PoseKp lists (normalized x,y in [0,1]
|
||||
# + z relative depth + confidence). We only apply temporal smoothing
|
||||
# (median + Kalman 2D + lookahead) — no IK, no spring.
|
||||
|
||||
def _parse_env_face_stages() -> tuple[str, ...]:
|
||||
raw = os.environ.get("POSE_FILTER_FACE")
|
||||
if raw is None:
|
||||
return ("median", "kalman", "lookahead")
|
||||
raw = raw.strip().lower()
|
||||
if raw in ("off", "none", "0", "false"):
|
||||
return ()
|
||||
parts = tuple(p.strip() for p in raw.replace(",", "+").split("+") if p.strip())
|
||||
return tuple(p for p in parts if p in ("median", "kalman", "lookahead"))
|
||||
|
||||
|
||||
def _parse_env_hand_stages() -> tuple[str, ...]:
|
||||
raw = os.environ.get("POSE_FILTER_HAND")
|
||||
if raw is None:
|
||||
return ("median", "kalman", "lookahead")
|
||||
raw = raw.strip().lower()
|
||||
if raw in ("off", "none", "0", "false"):
|
||||
return ()
|
||||
parts = tuple(p.strip() for p in raw.replace(",", "+").split("+") if p.strip())
|
||||
return tuple(p for p in parts if p in ("median", "kalman", "lookahead"))
|
||||
|
||||
|
||||
class AlphaBetaCV:
|
||||
"""Lightweight alpha-beta filter (scalar Kalman approximation).
|
||||
|
||||
Far cheaper than the 6x6 KalmanCV : O(1) per joint per axis with no
|
||||
matrix algebra. Suited to face/hand smoothing where the full CV
|
||||
Kalman is overkill.
|
||||
"""
|
||||
|
||||
def __init__(self, alpha: float = 0.55, beta: float = 0.15) -> None:
|
||||
self.alpha = alpha
|
||||
self.beta = beta
|
||||
# state[key] = [x, y, z, vx, vy, vz, last_t]
|
||||
self._st: dict[tuple[int, int], list[float]] = {}
|
||||
|
||||
def reset(self) -> None:
|
||||
self._st.clear()
|
||||
|
||||
def get_velocity(self, pid: int, joint_idx: int
|
||||
) -> tuple[float, float, float]:
|
||||
s = self._st.get((pid, joint_idx))
|
||||
if s is None:
|
||||
return (0.0, 0.0, 0.0)
|
||||
return (s[3], s[4], s[5])
|
||||
|
||||
def step(self, pid: int, joint_idx: int, mx: float, my: float,
|
||||
mz: float, t_now: float) -> tuple[float, float, float]:
|
||||
key = (pid, joint_idx)
|
||||
s = self._st.get(key)
|
||||
if s is None:
|
||||
self._st[key] = [mx, my, mz, 0.0, 0.0, 0.0, t_now]
|
||||
return (mx, my, mz)
|
||||
dt = max(1e-3, min(0.2, t_now - s[6]))
|
||||
s[6] = t_now
|
||||
# Predict
|
||||
x_pred = s[0] + s[3] * dt
|
||||
y_pred = s[1] + s[4] * dt
|
||||
z_pred = s[2] + s[5] * dt
|
||||
# Residual
|
||||
rx = mx - x_pred
|
||||
ry = my - y_pred
|
||||
rz = mz - z_pred
|
||||
# Update
|
||||
s[0] = x_pred + self.alpha * rx
|
||||
s[1] = y_pred + self.alpha * ry
|
||||
s[2] = z_pred + self.alpha * rz
|
||||
s[3] += (self.beta / dt) * rx
|
||||
s[4] += (self.beta / dt) * ry
|
||||
s[5] += (self.beta / dt) * rz
|
||||
return (s[0], s[1], s[2])
|
||||
|
||||
|
||||
class FaceFilterChain:
|
||||
"""Per-pid temporal smoothing for face landmarks (median + Kalman + lookahead).
|
||||
|
||||
Lookahead 30 ms ; max velocity in normalized units/s.
|
||||
"""
|
||||
|
||||
def __init__(self, lookahead_ms: float = 30.0,
|
||||
enabled_stages: Iterable[str] | None = None) -> None:
|
||||
if enabled_stages is None:
|
||||
stages = _parse_env_face_stages()
|
||||
else:
|
||||
stages = tuple(s for s in enabled_stages
|
||||
if s in ("median", "kalman", "lookahead"))
|
||||
self.enabled = stages
|
||||
self.median = MedianFilter(window=3)
|
||||
self.kalman = AlphaBetaCV(alpha=0.55, beta=0.15)
|
||||
self.lookahead = LookaheadPredictor(
|
||||
lookahead_ms=lookahead_ms, max_velocity=2.0)
|
||||
self.last_apply_ms: float = 0.0
|
||||
|
||||
def reset(self) -> None:
|
||||
self.median.reset()
|
||||
self.kalman.reset()
|
||||
|
||||
def apply(self, faces: list[list], ids: list[int],
|
||||
t_now: float) -> list[list]:
|
||||
if not faces or not self.enabled:
|
||||
self.last_apply_ms = 0.0
|
||||
return faces
|
||||
t0 = time.perf_counter()
|
||||
use_median = "median" in self.enabled
|
||||
use_kalman = "kalman" in self.enabled
|
||||
use_lookahead = "lookahead" in self.enabled
|
||||
out: list[list] = []
|
||||
for f_i, kps in enumerate(faces):
|
||||
pid = ids[f_i] if f_i < len(ids) else -1
|
||||
# Encode pid with a face-side namespace to avoid colliding with
|
||||
# body and hand kalman/median caches.
|
||||
key_pid = pid * 13 + 1 if pid >= 0 else pid
|
||||
new_kps = []
|
||||
for j_idx, kp in enumerate(kps):
|
||||
x, y, z, c = kp.x, kp.y, kp.z, kp.c
|
||||
if use_median:
|
||||
x, y, z = self.median.apply(key_pid, j_idx, x, y, z)
|
||||
if use_kalman:
|
||||
x, y, z = self.kalman.step(key_pid, j_idx, x, y, z, t_now)
|
||||
if use_lookahead and use_kalman:
|
||||
vx, vy, vz = self.kalman.get_velocity(key_pid, j_idx)
|
||||
x, y, z = self.lookahead.step(x, y, z, vx, vy, vz)
|
||||
new_kps.append(type(kp)(x=x, y=y, z=z, c=c))
|
||||
out.append(new_kps)
|
||||
self.last_apply_ms = (time.perf_counter() - t0) * 1000.0
|
||||
return out
|
||||
|
||||
|
||||
class HandFilterChain:
|
||||
"""Per-pid+side temporal smoothing for hand landmarks.
|
||||
|
||||
Left and right hands keep independent filter state via a namespaced
|
||||
pid (pid*2 for left, pid*2+1 for right). When handedness is not
|
||||
provided, hands fall back to a side-agnostic namespace.
|
||||
"""
|
||||
|
||||
def __init__(self, lookahead_ms: float = 30.0,
|
||||
enabled_stages: Iterable[str] | None = None) -> None:
|
||||
if enabled_stages is None:
|
||||
stages = _parse_env_hand_stages()
|
||||
else:
|
||||
stages = tuple(s for s in enabled_stages
|
||||
if s in ("median", "kalman", "lookahead"))
|
||||
self.enabled = stages
|
||||
self.median = MedianFilter(window=3)
|
||||
self.kalman = AlphaBetaCV(alpha=0.6, beta=0.2)
|
||||
self.lookahead = LookaheadPredictor(
|
||||
lookahead_ms=lookahead_ms, max_velocity=4.0)
|
||||
self.last_apply_ms: float = 0.0
|
||||
|
||||
def reset(self) -> None:
|
||||
self.median.reset()
|
||||
self.kalman.reset()
|
||||
|
||||
def apply(self, hands: list[list], ids: list[int],
|
||||
handedness: list[str] | None,
|
||||
t_now: float) -> list[list]:
|
||||
if not hands or not self.enabled:
|
||||
self.last_apply_ms = 0.0
|
||||
return hands
|
||||
t0 = time.perf_counter()
|
||||
use_median = "median" in self.enabled
|
||||
use_kalman = "kalman" in self.enabled
|
||||
use_lookahead = "lookahead" in self.enabled
|
||||
out: list[list] = []
|
||||
for h_i, kps in enumerate(hands):
|
||||
pid = ids[h_i] if h_i < len(ids) else -1
|
||||
side = (handedness[h_i] if handedness and h_i < len(handedness)
|
||||
else "u").lower()
|
||||
side_bit = 0 if side.startswith("l") else (1 if side.startswith("r") else 2)
|
||||
# Namespace : (pid << 2) | side_bit — keeps L/R independent.
|
||||
key_pid = (pid * 4 + side_bit + 7) if pid >= 0 else pid
|
||||
new_kps = []
|
||||
for j_idx, kp in enumerate(kps):
|
||||
x, y, z, c = kp.x, kp.y, kp.z, kp.c
|
||||
if use_median:
|
||||
x, y, z = self.median.apply(key_pid, j_idx, x, y, z)
|
||||
if use_kalman:
|
||||
x, y, z = self.kalman.step(key_pid, j_idx, x, y, z, t_now)
|
||||
if use_lookahead and use_kalman:
|
||||
vx, vy, vz = self.kalman.get_velocity(key_pid, j_idx)
|
||||
x, y, z = self.lookahead.step(x, y, z, vx, vy, vz)
|
||||
new_kps.append(type(kp)(x=x, y=y, z=z, c=c))
|
||||
out.append(new_kps)
|
||||
self.last_apply_ms = (time.perf_counter() - t0) * 1000.0
|
||||
return out
|
||||
@@ -0,0 +1,93 @@
|
||||
[project]
|
||||
name = "av-live-data-only-viz"
|
||||
version = "0.1.0"
|
||||
description = "Native Metal (pyobjc) visualizer for AV-Live data-only mode"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"pyobjc-core>=10.3",
|
||||
"pyobjc-framework-Cocoa>=10.3",
|
||||
"pyobjc-framework-Metal>=10.3",
|
||||
"pyobjc-framework-MetalKit>=10.3",
|
||||
"pyobjc-framework-Quartz>=10.3",
|
||||
"pyobjc-framework-AVFoundation>=10.3",
|
||||
"python-osc>=1.8.3",
|
||||
"numpy>=1.26,<2",
|
||||
"scipy>=1.13", # linear_sum_assignment pour tracker IoU (ByteTrack-like)
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
pose = [
|
||||
"coremltools>=9.0",
|
||||
"mediapipe>=0.10.35",
|
||||
"opencv-python>=4.10",
|
||||
"ultralytics>=8.3",
|
||||
]
|
||||
# DETRPose (transformer multi-personne, 2025). Le repo lui-meme n'est pas
|
||||
# pip-installable — voir docstring data_only_viz/detrpose.py pour la
|
||||
# procedure de clone manuel + telechargement du checkpoint.
|
||||
# Les deps listees ici sont uniquement les libs Python tirables via pip.
|
||||
detrpose = [
|
||||
"torch>=2.4",
|
||||
"torchvision>=0.19",
|
||||
"transformers>=4.40",
|
||||
"omegaconf>=2.3",
|
||||
"cloudpickle>=3.0",
|
||||
"pycocotools>=2.0",
|
||||
"xtcocotools>=1.14",
|
||||
"scipy>=1.13",
|
||||
"iopath>=0.1.10",
|
||||
"opencv-python>=4.10",
|
||||
]
|
||||
nlf = [
|
||||
"torch>=2.4",
|
||||
"torchvision>=0.19",
|
||||
"opencv-python>=4.10",
|
||||
"numpy>=1.26",
|
||||
]
|
||||
# Multi-HMR (Naver Labs, ECCV 2024). Pas pip-installable : on clone le repo
|
||||
# dans ~/.cache/av-live-multihmr/multi-hmr et on injecte sys.path au runtime.
|
||||
# SMPL-X NEUTRAL.npz requiert un compte academique sur smpl-x.is.tue.mpg.de.
|
||||
multihmr = [
|
||||
"torch>=2.4",
|
||||
"torchvision>=0.19",
|
||||
"smplx>=0.1.28",
|
||||
"einops>=0.8",
|
||||
"iopath>=0.1.10",
|
||||
"huggingface-hub>=0.24",
|
||||
"opencv-python>=4.10",
|
||||
"numpy>=1.26,<2",
|
||||
"scipy>=1.13",
|
||||
"torchgeometry>=0.1.2",
|
||||
# Multi-HMR utils (utils/humans.py, utils/render.py)
|
||||
"roma>=1.5",
|
||||
"trimesh>=4.4",
|
||||
"pillow>=10.0",
|
||||
"tqdm>=4.65",
|
||||
]
|
||||
# SMPLer-X (S-Lab, ECCV 2024, NON-COMMERCIAL). Code source vendu
|
||||
# via git submodule third_party/SMPLer-X (fork electron-rare).
|
||||
# mmcv-lite suffit pour Config (le repo vendorise sa propre mmpose).
|
||||
# Le detector mmdet est remplace par YOLO Ultralytics (extras pose).
|
||||
smplerx = [
|
||||
"torch>=2.4",
|
||||
"torchvision>=0.19",
|
||||
"smplx>=0.1.28",
|
||||
"mmcv-lite>=2.1",
|
||||
"ultralytics>=8.3",
|
||||
"opencv-python>=4.10",
|
||||
"numpy>=1.26,<2",
|
||||
"scipy>=1.13",
|
||||
"einops>=0.8",
|
||||
"pillow>=10.0",
|
||||
"tqdm>=4.65",
|
||||
"yacs>=0.1.8",
|
||||
"timm>=1.0",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
package = false
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9.0.3",
|
||||
]
|
||||
@@ -0,0 +1,521 @@
|
||||
"""Renderer Metal natif via pyobjc.
|
||||
|
||||
Architecture :
|
||||
- MTKView : NSView Metal-managee, callback drawInMTKView
|
||||
- MTLDevice : GPU par defaut
|
||||
- MTLCommandQueue : queue de submission
|
||||
- 2 pipelines :
|
||||
bg_pipeline : fullscreen tri + fbm fragment shader (1 draw call)
|
||||
skel_pipeline : lignes pour le squelette pose (max 16 segments)
|
||||
- Uniforms buffer 56 octets (14 floats) cf scene.metal::SceneUniforms
|
||||
|
||||
Le renderer ne possede PAS de state thread : il lit le State partage
|
||||
sous lock a chaque frame (60 fps).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import struct
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
import objc
|
||||
from Cocoa import NSObject, NSColor, NSMakeRect
|
||||
from Metal import (
|
||||
MTLCreateSystemDefaultDevice,
|
||||
MTLCompileOptions,
|
||||
MTLPrimitiveTypeTriangle,
|
||||
MTLPrimitiveTypeLine,
|
||||
MTLRenderPipelineDescriptor,
|
||||
MTLResourceStorageModeShared,
|
||||
MTLVertexAttributeDescriptor,
|
||||
MTLVertexBufferLayoutDescriptor,
|
||||
MTLVertexDescriptor,
|
||||
MTLVertexFormatFloat,
|
||||
MTLVertexFormatFloat2,
|
||||
MTLVertexFormatFloat3,
|
||||
MTLVertexStepFunctionPerVertex,
|
||||
)
|
||||
# MTKViewDelegate est un @protocol Obj-C ; pas besoin d'import Python.
|
||||
# pyobjc detecte automatiquement l'implementation par signature.
|
||||
from MetalKit import MTKView # noqa: F401 (utilise par d'autres modules)
|
||||
|
||||
from .mesh_topology import (
|
||||
BODY_TRIANGLES,
|
||||
FACE_TRIANGLES,
|
||||
HAND_TRIANGLES,
|
||||
build_face_triangles_dynamic,
|
||||
)
|
||||
from .state import State
|
||||
|
||||
LOG = logging.getLogger("renderer")
|
||||
|
||||
# Triangle primitive constant (Metal MTLPrimitiveType.triangle = 3)
|
||||
MTL_PRIMITIVE_TRIANGLE = 3
|
||||
|
||||
# 17 keypoints COCO bones (16 paires) — legacy YOLO fallback
|
||||
COCO_BONES: list[tuple[int, int]] = [
|
||||
(0, 1), (0, 2), (1, 3), (2, 4),
|
||||
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10),
|
||||
(5, 11), (6, 12), (11, 12),
|
||||
(11, 13), (13, 15), (12, 14), (14, 16),
|
||||
]
|
||||
|
||||
|
||||
def _mediapipe_bones():
|
||||
"""Charge les connexions MediaPipe (body+face+hands) au runtime.
|
||||
Retourne 4 listes : (body_bones, face_bones, lhand_bones, rhand_bones)
|
||||
avec chaque bone = (idx_a, idx_b).
|
||||
Retourne None si mediapipe absent (fallback COCO)."""
|
||||
try:
|
||||
from mediapipe.tasks.python.vision import (
|
||||
FaceLandmarksConnections as F,
|
||||
HandLandmarksConnections as H,
|
||||
PoseLandmarksConnections as P,
|
||||
)
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
def to_pairs(cs):
|
||||
return [(c.start, c.end) for c in cs]
|
||||
|
||||
# 35 body
|
||||
body = to_pairs(P.POSE_LANDMARKS)
|
||||
# Visage HAUTE DENSITE : tesselation complete (2556) + contours +
|
||||
# iris + nez. Donne ~2700 segments rien que pour le visage.
|
||||
face = (
|
||||
to_pairs(F.FACE_LANDMARKS_TESSELATION) # 2556 segs (mesh dense)
|
||||
+ to_pairs(F.FACE_LANDMARKS_FACE_OVAL)
|
||||
+ to_pairs(F.FACE_LANDMARKS_LIPS)
|
||||
+ to_pairs(F.FACE_LANDMARKS_LEFT_EYE)
|
||||
+ to_pairs(F.FACE_LANDMARKS_RIGHT_EYE)
|
||||
+ to_pairs(F.FACE_LANDMARKS_LEFT_EYEBROW)
|
||||
+ to_pairs(F.FACE_LANDMARKS_RIGHT_EYEBROW)
|
||||
+ to_pairs(F.FACE_LANDMARKS_LEFT_IRIS)
|
||||
+ to_pairs(F.FACE_LANDMARKS_RIGHT_IRIS)
|
||||
+ to_pairs(F.FACE_LANDMARKS_NOSE)
|
||||
)
|
||||
# 21 mains (chacune)
|
||||
hand = to_pairs(H.HAND_CONNECTIONS)
|
||||
return body, face, hand, hand # left+right partagent les connexions
|
||||
|
||||
|
||||
# Capacite max du vertex buffer skeleton.
|
||||
# 16384 segs * 2 verts * 5 floats (xyz + conf + pid) * 4 bytes = 640 KB
|
||||
# Couvre 4 personnes : 4×35 body + 4×2700 face + 8×21 hand = ~11000 segs.
|
||||
SKEL_MAX_SEGS = 16384
|
||||
SKEL_VERT_FLOATS = 5 # x, y, z, conf, person_id
|
||||
|
||||
# Mesh : ~8192 triangles × 3 verts × 5 floats × 4 = 480 KB.
|
||||
# Couvre 4 personnes : 4×~80 face + 4×~16 body + 8×~18 hand ≈ 600 triangles
|
||||
# pour le hardcode, ou ~4×~150 face Delaunay = ~600 triangles. Marge large.
|
||||
MESH_MAX_TRIS = 8192
|
||||
MESH_VERT_FLOATS = 5 # identique au skel
|
||||
MESH_MAX_VERTS = 10475 # SMPL-X is the larger family; SMPL (6890) fits inside
|
||||
|
||||
# struct SceneUniforms : 17 floats packs = 68 octets, padding a 80 (multiple
|
||||
# de 16, regle Metal). On y stocke (time, rms, kp_norm, netz_dev,
|
||||
# lightning_flash, flare, wind_norm, bz_norm, social_rate, pose_alive,
|
||||
# pose_count, width, height, viz_mode, _pad0, _pad1).
|
||||
UNIFORM_FLOATS = 20 # +4 floats : hand_l_x/y, hand_r_x/y
|
||||
UNIFORM_SIZE = UNIFORM_FLOATS * 4 # 80 octets, aligne 16
|
||||
|
||||
|
||||
class MetalRenderer(NSObject):
|
||||
"""Delegate de MTKView, drive l'integralite du rendu."""
|
||||
|
||||
def initWithState_(self, state: State): # noqa: N802
|
||||
self = objc.super(MetalRenderer, self).init()
|
||||
if self is None:
|
||||
return None
|
||||
self._state = state
|
||||
self._device = MTLCreateSystemDefaultDevice()
|
||||
if self._device is None:
|
||||
raise RuntimeError("Metal non disponible sur ce systeme")
|
||||
LOG.info("device: %s", self._device.name())
|
||||
self._queue = self._device.newCommandQueue()
|
||||
self._uniforms_buf = self._device.newBufferWithLength_options_(
|
||||
UNIFORM_SIZE, MTLResourceStorageModeShared)
|
||||
# Skeleton : N segs × 2 verts × 5 floats (xyz + conf + pid)
|
||||
self._skel_buf = self._device.newBufferWithLength_options_(
|
||||
SKEL_MAX_SEGS * 2 * SKEL_VERT_FLOATS * 4, MTLResourceStorageModeShared)
|
||||
# Mesh buffer : triangles face/hand/body
|
||||
self._mesh_buf = self._device.newBufferWithLength_options_(
|
||||
MESH_MAX_TRIS * 3 * MESH_VERT_FLOATS * 4, MTLResourceStorageModeShared)
|
||||
self._mp_bones = _mediapipe_bones() # None si pas dispo
|
||||
self._init_skel_cpu_buffer()
|
||||
self._init_mesh_cpu_buffer()
|
||||
self._build_pipelines()
|
||||
self._last_lightning_emit = 0.0
|
||||
return self
|
||||
|
||||
# ---- Pipelines -------------------------------------------------
|
||||
def _build_pipelines(self):
|
||||
src_path = Path(__file__).parent / "shaders" / "scene.metal"
|
||||
source = src_path.read_text()
|
||||
opts = MTLCompileOptions.alloc().init()
|
||||
lib, err = self._device.newLibraryWithSource_options_error_(
|
||||
source, opts, None)
|
||||
if lib is None:
|
||||
raise RuntimeError(f"shader compile failed: {err}")
|
||||
self._lib = lib
|
||||
|
||||
# --- Background pipeline (no vertex buffer) ---
|
||||
bg = MTLRenderPipelineDescriptor.alloc().init()
|
||||
bg.setVertexFunction_(lib.newFunctionWithName_("bg_vertex"))
|
||||
bg.setFragmentFunction_(lib.newFunctionWithName_("bg_fragment"))
|
||||
bg.colorAttachments().objectAtIndexedSubscript_(0).setPixelFormat_(80) # BGRA8Unorm
|
||||
p, err = self._device.newRenderPipelineStateWithDescriptor_error_(bg, None)
|
||||
if p is None:
|
||||
raise RuntimeError(f"bg pipeline failed: {err}")
|
||||
self._bg_pipe = p
|
||||
|
||||
# --- Skeleton pipeline (vertex buffer pos+conf) ---
|
||||
# Skeleton vertex layout : pos (float3) + conf (float) + pid (float)
|
||||
# Stride = 5 floats × 4 bytes = 20 bytes
|
||||
vd = MTLVertexDescriptor.vertexDescriptor()
|
||||
a0 = vd.attributes().objectAtIndexedSubscript_(0)
|
||||
a0.setFormat_(MTLVertexFormatFloat3); a0.setOffset_(0); a0.setBufferIndex_(0)
|
||||
a1 = vd.attributes().objectAtIndexedSubscript_(1)
|
||||
a1.setFormat_(MTLVertexFormatFloat); a1.setOffset_(12); a1.setBufferIndex_(0)
|
||||
a2 = vd.attributes().objectAtIndexedSubscript_(2)
|
||||
a2.setFormat_(MTLVertexFormatFloat); a2.setOffset_(16); a2.setBufferIndex_(0)
|
||||
ld = vd.layouts().objectAtIndexedSubscript_(0)
|
||||
ld.setStride_(20); ld.setStepFunction_(MTLVertexStepFunctionPerVertex)
|
||||
|
||||
sk = MTLRenderPipelineDescriptor.alloc().init()
|
||||
sk.setVertexFunction_(lib.newFunctionWithName_("skel_vertex"))
|
||||
sk.setFragmentFunction_(lib.newFunctionWithName_("skel_fragment"))
|
||||
sk.setVertexDescriptor_(vd)
|
||||
ca = sk.colorAttachments().objectAtIndexedSubscript_(0)
|
||||
ca.setPixelFormat_(80) # BGRA8Unorm
|
||||
ca.setBlendingEnabled_(True)
|
||||
# SrcAlpha, OneMinusSrcAlpha (= 4, 5 dans MTLBlendFactor)
|
||||
ca.setSourceRGBBlendFactor_(4)
|
||||
ca.setDestinationRGBBlendFactor_(5)
|
||||
ca.setSourceAlphaBlendFactor_(4)
|
||||
ca.setDestinationAlphaBlendFactor_(5)
|
||||
p, err = self._device.newRenderPipelineStateWithDescriptor_error_(sk, None)
|
||||
if p is None:
|
||||
raise RuntimeError(f"skel pipeline failed: {err}")
|
||||
self._skel_pipe = p
|
||||
|
||||
# --- Mesh pipeline (triangles face/hand/body) ---
|
||||
# Vertex layout strictement identique au skel (reutilise vd).
|
||||
vd_mesh = MTLVertexDescriptor.vertexDescriptor()
|
||||
a0 = vd_mesh.attributes().objectAtIndexedSubscript_(0)
|
||||
a0.setFormat_(MTLVertexFormatFloat3); a0.setOffset_(0); a0.setBufferIndex_(0)
|
||||
a1 = vd_mesh.attributes().objectAtIndexedSubscript_(1)
|
||||
a1.setFormat_(MTLVertexFormatFloat); a1.setOffset_(12); a1.setBufferIndex_(0)
|
||||
a2 = vd_mesh.attributes().objectAtIndexedSubscript_(2)
|
||||
a2.setFormat_(MTLVertexFormatFloat); a2.setOffset_(16); a2.setBufferIndex_(0)
|
||||
ld2 = vd_mesh.layouts().objectAtIndexedSubscript_(0)
|
||||
ld2.setStride_(20); ld2.setStepFunction_(MTLVertexStepFunctionPerVertex)
|
||||
|
||||
mp = MTLRenderPipelineDescriptor.alloc().init()
|
||||
mp.setVertexFunction_(lib.newFunctionWithName_("mesh_vertex"))
|
||||
mp.setFragmentFunction_(lib.newFunctionWithName_("mesh_fragment"))
|
||||
mp.setVertexDescriptor_(vd_mesh)
|
||||
cm = mp.colorAttachments().objectAtIndexedSubscript_(0)
|
||||
cm.setPixelFormat_(80)
|
||||
cm.setBlendingEnabled_(True)
|
||||
# SrcAlpha, OneMinusSrcAlpha — alpha classique pour voile mesh
|
||||
cm.setSourceRGBBlendFactor_(4)
|
||||
cm.setDestinationRGBBlendFactor_(5)
|
||||
cm.setSourceAlphaBlendFactor_(4)
|
||||
cm.setDestinationAlphaBlendFactor_(5)
|
||||
p, err = self._device.newRenderPipelineStateWithDescriptor_error_(mp, None)
|
||||
if p is None:
|
||||
raise RuntimeError(f"mesh pipeline failed: {err}")
|
||||
self._mesh_pipe = p
|
||||
|
||||
# ---- CPU staging buffers --------------------------------------
|
||||
def _init_skel_cpu_buffer(self) -> None:
|
||||
"""Preallocate the CPU staging buffer for skeleton segments.
|
||||
|
||||
SKEL_MAX_SEGS * 2 * SKEL_VERT_FLOATS floats : each segment = 2 verts × 5 floats
|
||||
(x, y, z, conf, pid). Idempotent — no-op if already allocated.
|
||||
"""
|
||||
if getattr(self, "_skel_cpu_buf", None) is None:
|
||||
self._skel_cpu_buf = np.zeros(SKEL_MAX_SEGS * 2 * SKEL_VERT_FLOATS, dtype=np.float32)
|
||||
|
||||
def _init_mesh_cpu_buffer(self) -> None:
|
||||
if getattr(self, "_mesh_cpu_buf", None) is None:
|
||||
self._mesh_cpu_buf = np.zeros(
|
||||
MESH_MAX_VERTS * MESH_VERT_FLOATS, dtype=np.float32,
|
||||
)
|
||||
|
||||
# ---- Uniforms helpers ------------------------------------------
|
||||
def _update_uniforms(self) -> int:
|
||||
s = self._state
|
||||
now = time.monotonic()
|
||||
with s.lock():
|
||||
# Flash de foudre : decay exponentiel ~600 ms
|
||||
dt_flash = now - s.last_lightning_t
|
||||
flash = max(0.0, 1.0 - dt_flash * 1.7) if dt_flash < 1.0 else 0.0
|
||||
# Positions des mains (point 0 = poignet) — pour mode hands3d
|
||||
lh_wrist = s.left_hand_kp[0] if s.hands_present else None
|
||||
rh_wrist = s.right_hand_kp[0] if s.hands_present else None
|
||||
hlx = (lh_wrist.x if lh_wrist else 0.5) * 2 - 1
|
||||
hly = 1 - (lh_wrist.y if lh_wrist else 0.5) * 2
|
||||
hrx = (rh_wrist.x if rh_wrist else 0.5) * 2 - 1
|
||||
hry = 1 - (rh_wrist.y if rh_wrist else 0.5) * 2
|
||||
uniforms = struct.pack(
|
||||
f"{UNIFORM_FLOATS}f",
|
||||
s.elapsed(), # 1
|
||||
min(1.0, s.rms * 3.0), # 2
|
||||
min(1.0, s.swpc_kp / 9.0), # 3
|
||||
max(-0.1, min(0.1, s.netz_dev)), # 4
|
||||
flash, # 5
|
||||
s.swpc_flare_norm, # 6
|
||||
max(0.0, min(1.0, (s.swpc_wind_speed - 280.0) / 600.0)), # 7
|
||||
max(-1.0, min(1.0, s.swpc_bz / 15.0)), # 8
|
||||
min(1.0, s.social_rate / 50.0), # 9
|
||||
1.0 if s.pose_alive() else 0.0, # 10
|
||||
float(s.pose_count), # 11
|
||||
float(s.width), # 12
|
||||
float(s.height), # 13
|
||||
float(s.viz_mode), # 14
|
||||
hlx, hly, hrx, hry, # 15-18 (mains)
|
||||
0.0, 0.0, # 19-20 pad
|
||||
)
|
||||
n_segs = self._update_skeleton(s)
|
||||
n_tris = self._update_mesh(s)
|
||||
# Copie via memoryview (pyobjc API : varlist.as_buffer(N))
|
||||
mv = self._uniforms_buf.contents().as_buffer(UNIFORM_SIZE)
|
||||
mv[:] = uniforms
|
||||
# Log debug toutes les 120 frames (~2s) — confirme que mesh draw
|
||||
if not hasattr(self, "_dbg_n"):
|
||||
self._dbg_n = 0
|
||||
self._dbg_n += 1
|
||||
if self._dbg_n % 120 == 0:
|
||||
LOG.info("render: %d segs, %d tris (face=%d hand=%d body=%d)",
|
||||
n_segs, n_tris,
|
||||
len(self._state.persons_face),
|
||||
len(self._state.persons_hands),
|
||||
len(self._state.persons_body))
|
||||
return n_segs, n_tris
|
||||
|
||||
def _update_skeleton(self, s: State) -> int:
|
||||
"""Remplit self._skel_buf avec les segments visibles. Retourne le
|
||||
nombre de segments (2 verts chacun).
|
||||
|
||||
Priorise MediaPipe (body 33 + face 478 + 2 mains 21) si disponible
|
||||
et present ; sinon fallback COCO 17 keypoints YOLO."""
|
||||
if not s.pose_alive():
|
||||
return 0
|
||||
|
||||
buf = self._skel_cpu_buf
|
||||
segs = 0
|
||||
|
||||
def push(A, B, conf, pid):
|
||||
"""Empile un segment (2 verts) dans le buffer CPU prealloque."""
|
||||
nonlocal segs
|
||||
if segs >= SKEL_MAX_SEGS:
|
||||
return False
|
||||
ax = A.x * 2.0 - 1.0; ay = 1.0 - A.y * 2.0
|
||||
bx = B.x * 2.0 - 1.0; by = 1.0 - B.y * 2.0
|
||||
i = segs * 10
|
||||
buf[i+0] = ax; buf[i+1] = ay; buf[i+2] = float(A.z); buf[i+3] = conf; buf[i+4] = float(pid)
|
||||
buf[i+5] = bx; buf[i+6] = by; buf[i+7] = float(B.z); buf[i+8] = conf; buf[i+9] = float(pid)
|
||||
segs += 1
|
||||
return True
|
||||
|
||||
if self._mp_bones is not None and (
|
||||
s.persons_body or s.persons_face or s.persons_hands or
|
||||
s.body_present or s.face_present or s.hands_present
|
||||
):
|
||||
body_bones, face_bones, lhand_bones, _ = self._mp_bones
|
||||
# ----- MULTI-PERSONNE : pid = ID stable du tracker -----
|
||||
# (track_id persiste entre frames, palette se stabilise)
|
||||
ids_b = s.persons_body_ids or list(range(len(s.persons_body)))
|
||||
ids_f = s.persons_face_ids or list(range(len(s.persons_face)))
|
||||
ids_h = s.persons_hands_ids or list(range(len(s.persons_hands)))
|
||||
for i, body_kp in enumerate(s.persons_body):
|
||||
pid = ids_b[i] if i < len(ids_b) else i
|
||||
for a, b in body_bones:
|
||||
if a >= len(body_kp) or b >= len(body_kp): continue
|
||||
A = body_kp[a]; B = body_kp[b]
|
||||
if A.c < 0.15 or B.c < 0.15: continue
|
||||
if not push(A, B, min(A.c, B.c), pid): break
|
||||
for i, face_kp in enumerate(s.persons_face):
|
||||
pid = ids_f[i] if i < len(ids_f) else i
|
||||
for a, b in face_bones:
|
||||
if a >= len(face_kp) or b >= len(face_kp): continue
|
||||
if not push(face_kp[a], face_kp[b], 1.0, pid): break
|
||||
if segs >= SKEL_MAX_SEGS: break
|
||||
for i, hand_kp in enumerate(s.persons_hands):
|
||||
pid = ids_h[i] if i < len(ids_h) else i
|
||||
for a, b in lhand_bones:
|
||||
if a >= len(hand_kp) or b >= len(hand_kp): continue
|
||||
# Decalage palette mains (+5) pour les distinguer
|
||||
if not push(hand_kp[a], hand_kp[b], 1.0, pid + 5): break
|
||||
# ----- FALLBACK single-person si persons_* vides -----
|
||||
if not (s.persons_body or s.persons_face or s.persons_hands):
|
||||
if s.body_present:
|
||||
for a, b in body_bones:
|
||||
if a >= len(s.body_kp) or b >= len(s.body_kp): continue
|
||||
A = s.body_kp[a]; B = s.body_kp[b]
|
||||
if A.c < 0.15 or B.c < 0.15: continue
|
||||
if not push(A, B, min(A.c, B.c), 0): break
|
||||
if s.face_present:
|
||||
for a, b in face_bones:
|
||||
if a >= len(s.face_kp) or b >= len(s.face_kp): continue
|
||||
if not push(s.face_kp[a], s.face_kp[b], 1.0, 0): break
|
||||
if s.hands_present:
|
||||
for kp_list in (s.left_hand_kp, s.right_hand_kp):
|
||||
if not any(p.x != 0.0 or p.y != 0.0 for p in kp_list):
|
||||
continue
|
||||
for a, b in lhand_bones:
|
||||
if a >= len(kp_list) or b >= len(kp_list): continue
|
||||
if not push(kp_list[a], kp_list[b], 1.0, 0): break
|
||||
else:
|
||||
# Fallback COCO 17 (YOLO legacy)
|
||||
for a, b in COCO_BONES:
|
||||
A = s.pose_kp[a]; B = s.pose_kp[b]
|
||||
if A.c < 0.2 or B.c < 0.2: continue
|
||||
if not push(A, B, min(A.c, B.c), 0): break
|
||||
|
||||
if segs == 0:
|
||||
return 0
|
||||
data = self._skel_cpu_buf[: segs * 2 * SKEL_VERT_FLOATS].tobytes()
|
||||
mv = self._skel_buf.contents().as_buffer(len(data))
|
||||
mv[:] = data
|
||||
return segs
|
||||
|
||||
def _update_mesh(self, s: State) -> int:
|
||||
"""Remplit self._mesh_buf avec des triangles face/hand/body.
|
||||
|
||||
Retourne le nombre de triangles ecrits (chacun = 3 vertices).
|
||||
Filtre les triangles dont au moins un sommet a confiance < 0.3.
|
||||
"""
|
||||
if not s.pose_alive():
|
||||
return 0
|
||||
if not (s.persons_face or s.persons_hands or s.persons_body):
|
||||
return 0
|
||||
|
||||
n_verts = 0
|
||||
|
||||
def push_tri(kp_list, i, j, k, pid: int) -> bool:
|
||||
"""Pousse un triangle (3 verts). Retourne False si buffer plein
|
||||
ou triangle invalide (confiance basse)."""
|
||||
nonlocal n_verts
|
||||
tris = n_verts // 3
|
||||
if tris >= MESH_MAX_TRIS or n_verts + 3 > MESH_MAX_VERTS:
|
||||
return False
|
||||
if i >= len(kp_list) or j >= len(kp_list) or k >= len(kp_list):
|
||||
return True # skip mais continue
|
||||
A = kp_list[i]; B = kp_list[j]; C = kp_list[k]
|
||||
if A.c < 0.15 or B.c < 0.15 or C.c < 0.15:
|
||||
return True
|
||||
ax = A.x * 2.0 - 1.0; ay = 1.0 - A.y * 2.0
|
||||
bx = B.x * 2.0 - 1.0; by = 1.0 - B.y * 2.0
|
||||
cx = C.x * 2.0 - 1.0; cy = 1.0 - C.y * 2.0
|
||||
conf = min(A.c, B.c, C.c)
|
||||
fpid = float(pid)
|
||||
base = n_verts * 5
|
||||
self._mesh_cpu_buf[base + 0] = ax
|
||||
self._mesh_cpu_buf[base + 1] = ay
|
||||
self._mesh_cpu_buf[base + 2] = float(A.z)
|
||||
self._mesh_cpu_buf[base + 3] = conf
|
||||
self._mesh_cpu_buf[base + 4] = fpid
|
||||
self._mesh_cpu_buf[base + 5] = bx
|
||||
self._mesh_cpu_buf[base + 6] = by
|
||||
self._mesh_cpu_buf[base + 7] = float(B.z)
|
||||
self._mesh_cpu_buf[base + 8] = conf
|
||||
self._mesh_cpu_buf[base + 9] = fpid
|
||||
self._mesh_cpu_buf[base + 10] = cx
|
||||
self._mesh_cpu_buf[base + 11] = cy
|
||||
self._mesh_cpu_buf[base + 12] = float(C.z)
|
||||
self._mesh_cpu_buf[base + 13] = conf
|
||||
self._mesh_cpu_buf[base + 14] = fpid
|
||||
n_verts += 3
|
||||
return True
|
||||
|
||||
ids_b = s.persons_body_ids or list(range(len(s.persons_body)))
|
||||
ids_f = s.persons_face_ids or list(range(len(s.persons_face)))
|
||||
ids_h = s.persons_hands_ids or list(range(len(s.persons_hands)))
|
||||
|
||||
# Body
|
||||
for i, body_kp in enumerate(s.persons_body):
|
||||
pid = ids_b[i] if i < len(ids_b) else i
|
||||
for a, b, c in BODY_TRIANGLES:
|
||||
if not push_tri(body_kp, a, b, c, pid):
|
||||
break
|
||||
if n_verts >= MESH_MAX_VERTS:
|
||||
break
|
||||
|
||||
# Face — utilise triangulation Delaunay dynamique sur les XY,
|
||||
# fallback sur FACE_TRIANGLES statique si Delaunay echoue.
|
||||
for i, face_kp in enumerate(s.persons_face):
|
||||
pid = ids_f[i] if i < len(ids_f) else i
|
||||
pts_xy = [(kp.x, kp.y) for kp in face_kp]
|
||||
tri_list = build_face_triangles_dynamic(pts_xy) or FACE_TRIANGLES
|
||||
for a, b, c in tri_list:
|
||||
if not push_tri(face_kp, a, b, c, pid):
|
||||
break
|
||||
if n_verts >= MESH_MAX_VERTS:
|
||||
break
|
||||
|
||||
# Hands — decalage palette +5 comme dans le skel
|
||||
for i, hand_kp in enumerate(s.persons_hands):
|
||||
pid = (ids_h[i] if i < len(ids_h) else i) + 5
|
||||
for a, b, c in HAND_TRIANGLES:
|
||||
if not push_tri(hand_kp, a, b, c, pid):
|
||||
break
|
||||
if n_verts >= MESH_MAX_VERTS:
|
||||
break
|
||||
|
||||
if n_verts == 0:
|
||||
return 0
|
||||
# Slice is exact — stale floats beyond n_verts*MESH_VERT_FLOATS never reach the GPU.
|
||||
data = self._mesh_cpu_buf[: n_verts * MESH_VERT_FLOATS].tobytes()
|
||||
mv = self._mesh_buf.contents().as_buffer(len(data))
|
||||
mv[:] = data
|
||||
return n_verts // 3
|
||||
|
||||
# ---- MTKViewDelegate ------------------------------------------
|
||||
def mtkView_drawableSizeWillChange_(self, view, size): # noqa: N802
|
||||
with self._state.lock():
|
||||
self._state.width = int(size.width)
|
||||
self._state.height = int(size.height)
|
||||
|
||||
def drawInMTKView_(self, view): # noqa: N802
|
||||
n_segs, n_tris = self._update_uniforms()
|
||||
rpd = view.currentRenderPassDescriptor()
|
||||
drawable = view.currentDrawable()
|
||||
if rpd is None or drawable is None:
|
||||
return
|
||||
cb = self._queue.commandBuffer()
|
||||
enc = cb.renderCommandEncoderWithDescriptor_(rpd)
|
||||
|
||||
# 1) background fullscreen tri
|
||||
enc.setRenderPipelineState_(self._bg_pipe)
|
||||
enc.setFragmentBuffer_offset_atIndex_(self._uniforms_buf, 0, 0)
|
||||
enc.drawPrimitives_vertexStart_vertexCount_(MTLPrimitiveTypeTriangle, 0, 3)
|
||||
|
||||
# 2) mesh overlay (triangles face/hand/body) — DESSOUS le skel
|
||||
if n_tris > 0:
|
||||
enc.setRenderPipelineState_(self._mesh_pipe)
|
||||
enc.setVertexBuffer_offset_atIndex_(self._mesh_buf, 0, 0)
|
||||
enc.setVertexBuffer_offset_atIndex_(self._uniforms_buf, 0, 1)
|
||||
enc.drawPrimitives_vertexStart_vertexCount_(
|
||||
MTL_PRIMITIVE_TRIANGLE, 0, n_tris * 3)
|
||||
|
||||
# 3) skeleton overlay (lignes par-dessus le mesh)
|
||||
if n_segs > 0:
|
||||
enc.setRenderPipelineState_(self._skel_pipe)
|
||||
enc.setVertexBuffer_offset_atIndex_(self._skel_buf, 0, 0)
|
||||
# SceneUniforms a buffer(1) du vertex pour acceder a U.rms etc
|
||||
enc.setVertexBuffer_offset_atIndex_(self._uniforms_buf, 0, 1)
|
||||
enc.drawPrimitives_vertexStart_vertexCount_(
|
||||
MTLPrimitiveTypeLine, 0, n_segs * 2)
|
||||
|
||||
enc.endEncoding()
|
||||
cb.presentDrawable_(drawable)
|
||||
cb.commit()
|
||||
|
||||
def device(self):
|
||||
return self._device
|
||||
@@ -0,0 +1,235 @@
|
||||
"""Bench Multi-HMR CoreML — compute_units sweep + section split.
|
||||
|
||||
Bench Multi-HMR `.mlpackage` inference latency on M5 (or any Apple
|
||||
Silicon). Decomposes the per-frame cost into copy_in / predict /
|
||||
copy_out so we can see where time goes, then sweeps compute_units
|
||||
(CPU_AND_GPU vs ALL vs CPU_AND_NE vs CPU_ONLY) and tests the
|
||||
"reused MLMultiArray buffer" optimization.
|
||||
|
||||
Usage:
|
||||
uv run --project data_only_viz \
|
||||
python -m data_only_viz.scripts.bench_multihmr_coreml
|
||||
|
||||
The result reproduces the 2026-05-14 finding: predict() is ~99% of
|
||||
latency, copy_in is <2 ms, copy_out is <1 ms. None of the I/O
|
||||
micro-optims (reused buffer, vImage preprocess, async copy) can
|
||||
help meaningfully — only changing the model itself does (INT8 quant
|
||||
via `scripts/quantize_multihmr_int8.py`, lower resolution, or a
|
||||
smaller architecture).
|
||||
|
||||
Pause the live worker before running for clean numbers:
|
||||
pgrep -f 'data_only_viz.main.*multi-hmr' | xargs kill -STOP
|
||||
# ...run bench...
|
||||
pgrep -f 'data_only_viz.main.*multi-hmr' | xargs kill -CONT
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from Foundation import NSURL
|
||||
|
||||
from data_only_viz.multihmr_coreml import (
|
||||
DEFAULT_MLPACKAGE,
|
||||
_load_frameworks,
|
||||
_mlarray_to_np,
|
||||
_np_to_mlarray,
|
||||
)
|
||||
|
||||
H = W = 672
|
||||
NITER = 30
|
||||
NWARM = 5
|
||||
|
||||
|
||||
def _make_inputs():
|
||||
img = np.random.rand(1, 3, H, W).astype(np.float32)
|
||||
focal = float(H)
|
||||
K = np.array(
|
||||
[[[focal, 0, H / 2], [0, focal, H / 2], [0, 0, 1.0]]],
|
||||
dtype=np.float32,
|
||||
)
|
||||
return img, K
|
||||
|
||||
|
||||
def _load_model(compute_units: int, mlpackage: Path):
|
||||
ns = _load_frameworks()
|
||||
MLModel = ns["MLModel"]
|
||||
MLModelConfiguration = ns["MLModelConfiguration"]
|
||||
cfg = MLModelConfiguration.alloc().init()
|
||||
cfg.setComputeUnits_(compute_units)
|
||||
url = NSURL.fileURLWithPath_(str(mlpackage))
|
||||
compiled = MLModel.compileModelAtURL_error_(url, None)
|
||||
if compiled is None:
|
||||
raise RuntimeError(f"compile failed cu={compute_units}")
|
||||
model = MLModel.modelWithContentsOfURL_configuration_error_(
|
||||
compiled, cfg, None)
|
||||
if model is None:
|
||||
raise RuntimeError(f"load failed cu={compute_units}")
|
||||
return model, ns
|
||||
|
||||
|
||||
def _stats(ts):
|
||||
ts = sorted(ts)
|
||||
return (ts[len(ts) // 2],
|
||||
ts[len(ts) // 10],
|
||||
ts[(len(ts) * 9) // 10])
|
||||
|
||||
|
||||
def bench_basic(label: str, compute_units: int, mlpackage: Path):
|
||||
try:
|
||||
model, ns = _load_model(compute_units, mlpackage)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"[{label}] LOAD FAILED: {e}")
|
||||
return None
|
||||
MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"]
|
||||
MLFeatureValue = ns["MLFeatureValue"]
|
||||
img, K = _make_inputs()
|
||||
for _ in range(NWARM):
|
||||
img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K)
|
||||
feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml),
|
||||
"cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)}
|
||||
prov = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
out = model.predictionFromFeatures_error_(prov, None)
|
||||
if out is None:
|
||||
print(f"[{label}] predict returned None")
|
||||
return None
|
||||
ts = []
|
||||
for _ in range(NITER):
|
||||
t0 = time.perf_counter()
|
||||
img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K)
|
||||
feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml),
|
||||
"cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)}
|
||||
prov = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
out = model.predictionFromFeatures_error_(prov, None)
|
||||
for name in out.featureNames():
|
||||
fv = out.featureValueForName_(name)
|
||||
ml = fv.multiArrayValue()
|
||||
if ml is None:
|
||||
continue
|
||||
_ = _mlarray_to_np(ml)
|
||||
ts.append((time.perf_counter() - t0) * 1e3)
|
||||
med, p10, p90 = _stats(ts)
|
||||
print(f"[{label:34s}] med={med:6.1f}ms p10={p10:6.1f} "
|
||||
f"p90={p90:6.1f} fps={1000/med:5.1f}")
|
||||
return med
|
||||
|
||||
|
||||
def bench_reused_input(label: str, compute_units: int, mlpackage: Path):
|
||||
try:
|
||||
model, ns = _load_model(compute_units, mlpackage)
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f"[{label}] LOAD FAILED: {e}")
|
||||
return None
|
||||
MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"]
|
||||
MLFeatureValue = ns["MLFeatureValue"]
|
||||
img, K = _make_inputs()
|
||||
img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K)
|
||||
ptr_img = img_ml.dataPointer()
|
||||
addr_img = int(ptr_img) if isinstance(ptr_img, int) else \
|
||||
ctypes.cast(ptr_img, ctypes.c_void_p).value
|
||||
ptr_k = k_ml.dataPointer()
|
||||
addr_k = int(ptr_k) if isinstance(ptr_k, int) else \
|
||||
ctypes.cast(ptr_k, ctypes.c_void_p).value
|
||||
img_bytes = img.nbytes
|
||||
k_bytes = K.nbytes
|
||||
feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml),
|
||||
"cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)}
|
||||
for _ in range(NWARM):
|
||||
ctypes.memmove(addr_img, img.ctypes.data, img_bytes)
|
||||
ctypes.memmove(addr_k, K.ctypes.data, k_bytes)
|
||||
prov = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
_ = model.predictionFromFeatures_error_(prov, None)
|
||||
ts = []
|
||||
for _ in range(NITER):
|
||||
t0 = time.perf_counter()
|
||||
ctypes.memmove(addr_img, img.ctypes.data, img_bytes)
|
||||
ctypes.memmove(addr_k, K.ctypes.data, k_bytes)
|
||||
prov = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
out = model.predictionFromFeatures_error_(prov, None)
|
||||
for name in out.featureNames():
|
||||
fv = out.featureValueForName_(name)
|
||||
ml = fv.multiArrayValue()
|
||||
if ml is None:
|
||||
continue
|
||||
_ = _mlarray_to_np(ml)
|
||||
ts.append((time.perf_counter() - t0) * 1e3)
|
||||
med, p10, p90 = _stats(ts)
|
||||
print(f"[{label:34s}] med={med:6.1f}ms p10={p10:6.1f} "
|
||||
f"p90={p90:6.1f} fps={1000/med:5.1f}")
|
||||
return med
|
||||
|
||||
|
||||
def bench_section_split(compute_units: int, mlpackage: Path):
|
||||
model, ns = _load_model(compute_units, mlpackage)
|
||||
MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"]
|
||||
MLFeatureValue = ns["MLFeatureValue"]
|
||||
img, K = _make_inputs()
|
||||
for _ in range(NWARM):
|
||||
img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K)
|
||||
feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml),
|
||||
"cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)}
|
||||
prov = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
_ = model.predictionFromFeatures_error_(prov, None)
|
||||
t_in, t_pred, t_out = [], [], []
|
||||
for _ in range(NITER):
|
||||
t0 = time.perf_counter()
|
||||
img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K)
|
||||
feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml),
|
||||
"cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)}
|
||||
prov = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
t1 = time.perf_counter()
|
||||
out = model.predictionFromFeatures_error_(prov, None)
|
||||
t2 = time.perf_counter()
|
||||
for name in out.featureNames():
|
||||
fv = out.featureValueForName_(name)
|
||||
ml = fv.multiArrayValue()
|
||||
if ml is None:
|
||||
continue
|
||||
_ = _mlarray_to_np(ml)
|
||||
t3 = time.perf_counter()
|
||||
t_in.append((t1 - t0) * 1e3)
|
||||
t_pred.append((t2 - t1) * 1e3)
|
||||
t_out.append((t3 - t2) * 1e3)
|
||||
mi = lambda a: sorted(a)[len(a) // 2]
|
||||
print("[section-split CPU_AND_GPU]")
|
||||
print(f" copy_in : {mi(t_in):6.2f} ms")
|
||||
print(f" predict : {mi(t_pred):6.2f} ms")
|
||||
print(f" copy_out : {mi(t_out):6.2f} ms")
|
||||
print(f" total : {mi(t_in)+mi(t_pred)+mi(t_out):6.2f} ms")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
mlpackage = DEFAULT_MLPACKAGE
|
||||
if len(argv) > 1:
|
||||
mlpackage = Path(argv[1])
|
||||
if not mlpackage.exists():
|
||||
print(f"mlpackage missing: {mlpackage}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"bench target: {mlpackage}")
|
||||
print("=" * 70)
|
||||
print("Section split (alloc/predict/copy)")
|
||||
print("=" * 70)
|
||||
bench_section_split(1, mlpackage)
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("Compute-units sweep (30 iter median)")
|
||||
print("=" * 70)
|
||||
bench_basic("A. CPU_AND_GPU (baseline)", 1, mlpackage)
|
||||
bench_basic("B. ALL (ANE+GPU+CPU)", 2, mlpackage)
|
||||
bench_basic("C. CPU_AND_NE (ANE-only)", 3, mlpackage)
|
||||
bench_basic("D. CPU_ONLY", 0, mlpackage)
|
||||
bench_reused_input("E. CPU_AND_GPU + reused buffer", 1, mlpackage)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Record webcam frames + timestamps for action-head training.
|
||||
|
||||
Usage:
|
||||
uv run python -m data_only_viz.scripts.capture_actions \
|
||||
--session sess03 --duration 600
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
|
||||
LOG = logging.getLogger("capture_actions")
|
||||
RAW_DIR = Path("~/.cache/av-live-action/raw").expanduser()
|
||||
|
||||
|
||||
def capture(session: str, duration_s: float,
|
||||
cam_index: int = 0, fps: int = 30,
|
||||
size: int = 672) -> Path:
|
||||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||||
out = RAW_DIR / f"{session}.mp4"
|
||||
ts_out = RAW_DIR / f"{session}.ts.txt"
|
||||
cap = cv2.VideoCapture(cam_index)
|
||||
if not cap.isOpened():
|
||||
raise RuntimeError(f"cannot open camera {cam_index}")
|
||||
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
||||
writer = cv2.VideoWriter(str(out), fourcc, fps, (size, size))
|
||||
try:
|
||||
t_start = time.perf_counter()
|
||||
with ts_out.open("w") as ts_f:
|
||||
n = 0
|
||||
while time.perf_counter() - t_start < duration_s:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
LOG.warning("frame read failed")
|
||||
break
|
||||
h, w = frame.shape[:2]
|
||||
side = min(h, w)
|
||||
y0 = (h - side) // 2
|
||||
x0 = (w - side) // 2
|
||||
crop = frame[y0:y0 + side, x0:x0 + side]
|
||||
resized = cv2.resize(crop, (size, size))
|
||||
writer.write(resized)
|
||||
ts_f.write(f"{n} {time.perf_counter() - t_start:.6f}\n")
|
||||
n += 1
|
||||
cv2.imshow("capture (q=quit)", resized)
|
||||
if cv2.waitKey(1) & 0xFF == ord("q"):
|
||||
break
|
||||
LOG.info("wrote %s (%d frames)", out, n)
|
||||
return out
|
||||
finally:
|
||||
cap.release()
|
||||
writer.release()
|
||||
cv2.destroyAllWindows()
|
||||
|
||||
|
||||
def _cli() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--session", required=True)
|
||||
p.add_argument("--duration", type=float, default=600.0)
|
||||
p.add_argument("--cam-index", type=int, default=0)
|
||||
p.add_argument("--fps", type=int, default=30)
|
||||
args = p.parse_args()
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format="%(asctime)s [%(name)s] %(message)s")
|
||||
capture(args.session, args.duration,
|
||||
cam_index=args.cam_index, fps=args.fps)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_cli()
|
||||
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Conversion des modeles pose vers CoreML .mlpackage pour ANE/M5.
|
||||
|
||||
Pipeline cible :
|
||||
1. YOLO11n-pose (ultralytics) — detection + pose 17 kp COCO en
|
||||
un seul modele top-down "tout-en-un". C'est notre baseline
|
||||
prefere : install simple, export CoreML natif via ultralytics,
|
||||
fonctionne sur ANE en INT8/FP16.
|
||||
2. (Optionnel) RTMPose-m via mmpose — pose top-down plus precise,
|
||||
necessite des bboxes en entree (paire avec un detecteur). Skip
|
||||
si mmpose n'est pas installe.
|
||||
3. (Optionnel) DWPose-m via mmpose — 133 kp body+face+hands, le
|
||||
plus complet. Souvent difficile a exporter en CoreML (operateurs
|
||||
custom). Si l'export echoue, on retombe sur YOLO11n-pose seul.
|
||||
|
||||
Usage :
|
||||
uv run python -m data_only_viz.scripts.convert_coreml
|
||||
uv run python -m data_only_viz.scripts.convert_coreml --force
|
||||
|
||||
Sortie :
|
||||
~/.cache/av-live-coreml/yolo11n-pose.mlpackage
|
||||
~/.cache/av-live-coreml/rtmpose-m.mlpackage (si possible)
|
||||
~/.cache/av-live-coreml/dwpose-m.mlpackage (si possible)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
LOG = logging.getLogger("convert_coreml")
|
||||
|
||||
CACHE_DIR = Path.home() / ".cache" / "av-live-coreml"
|
||||
YOLO_NAME = "yolo11n-pose"
|
||||
YOLO_MLPACKAGE = CACHE_DIR / f"{YOLO_NAME}.mlpackage"
|
||||
|
||||
|
||||
def _du_mb(path: Path) -> float:
|
||||
"""Taille (MB) d'un dossier ou d'un fichier."""
|
||||
if not path.exists():
|
||||
return 0.0
|
||||
if path.is_file():
|
||||
return path.stat().st_size / (1024 * 1024)
|
||||
total = 0
|
||||
for p in path.rglob("*"):
|
||||
if p.is_file():
|
||||
total += p.stat().st_size
|
||||
return total / (1024 * 1024)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. YOLO11n-pose : export tout-en-un via ultralytics
|
||||
# ---------------------------------------------------------------------------
|
||||
def convert_yolo11n_pose(force: bool = False) -> Path | None:
|
||||
"""Telecharge YOLO11n-pose .pt et l'exporte en CoreML .mlpackage."""
|
||||
if YOLO_MLPACKAGE.exists() and not force:
|
||||
LOG.info("[yolo11n-pose] deja present : %s (%.1f MB)",
|
||||
YOLO_MLPACKAGE, _du_mb(YOLO_MLPACKAGE))
|
||||
return YOLO_MLPACKAGE
|
||||
try:
|
||||
from ultralytics import YOLO
|
||||
except ImportError:
|
||||
LOG.error("[yolo11n-pose] ultralytics manquant — "
|
||||
"uv pip install ultralytics coremltools")
|
||||
return None
|
||||
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
LOG.info("[yolo11n-pose] telechargement du checkpoint .pt ...")
|
||||
# ultralytics resout 'yolo11n-pose.pt' automatiquement depuis ses
|
||||
# assets GitHub. Le fichier atterit dans CWD ; on chdir dans cache.
|
||||
cwd_before = Path.cwd()
|
||||
try:
|
||||
import os
|
||||
os.chdir(CACHE_DIR)
|
||||
model = YOLO(f"{YOLO_NAME}.pt")
|
||||
LOG.info("[yolo11n-pose] export CoreML (ANE/FP16) ...")
|
||||
# nms=True : le modele inclut deja le NMS, simplifie le post-process
|
||||
# int8=False : on garde FP16, plus sur pour la pose (precision sub-pix)
|
||||
out = model.export(format="coreml", nms=True, half=True, imgsz=640)
|
||||
out_path = Path(out)
|
||||
# ultralytics nomme le fichier 'yolo11n-pose.mlpackage'
|
||||
if out_path.exists() and out_path != YOLO_MLPACKAGE:
|
||||
if YOLO_MLPACKAGE.exists():
|
||||
shutil.rmtree(YOLO_MLPACKAGE)
|
||||
shutil.move(str(out_path), str(YOLO_MLPACKAGE))
|
||||
LOG.info("[yolo11n-pose] export OK : %s (%.1f MB)",
|
||||
YOLO_MLPACKAGE, _du_mb(YOLO_MLPACKAGE))
|
||||
return YOLO_MLPACKAGE
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("[yolo11n-pose] export echoue : %s", e)
|
||||
return None
|
||||
finally:
|
||||
import os
|
||||
os.chdir(cwd_before)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. RTMPose-m (top-down, 17 kp) — via mmpose si dispo
|
||||
# ---------------------------------------------------------------------------
|
||||
def convert_rtmpose_m(force: bool = False) -> Path | None:
|
||||
"""Stub : mmpose n'a pas d'export CoreML natif. Skip pour l'instant.
|
||||
|
||||
NOTE : la voie pratique serait de passer par ONNX puis coremltools.
|
||||
On laisse le scaffold ici mais on ne tente pas la conversion par defaut
|
||||
car la chaine PyTorch -> ONNX -> CoreML pour RTMPose demande des patches
|
||||
sur les ops Argmax + heatmap decoding.
|
||||
"""
|
||||
out = CACHE_DIR / "rtmpose-m.mlpackage"
|
||||
if out.exists() and not force:
|
||||
LOG.info("[rtmpose-m] deja present : %s", out)
|
||||
return out
|
||||
LOG.info("[rtmpose-m] skip (export ONNX->CoreML non implemente — "
|
||||
"voir https://github.com/open-mmlab/mmdeploy)")
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. DWPose-m (133 kp body+face+hands)
|
||||
# ---------------------------------------------------------------------------
|
||||
def convert_dwpose_m(force: bool = False) -> Path | None:
|
||||
"""Stub : DWPose utilise les memes ops que RTMPose + un distillation
|
||||
head. Conversion CoreML tres flaky en pratique. Skip et fallback YOLO.
|
||||
"""
|
||||
out = CACHE_DIR / "dwpose-m.mlpackage"
|
||||
if out.exists() and not force:
|
||||
LOG.info("[dwpose-m] deja present : %s", out)
|
||||
return out
|
||||
LOG.info("[dwpose-m] skip (export instable — fallback YOLO11n-pose)")
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rapport final
|
||||
# ---------------------------------------------------------------------------
|
||||
def report(models: dict[str, Path | None]) -> None:
|
||||
print()
|
||||
print("=" * 68)
|
||||
print(" CoreML pose models — rapport")
|
||||
print("=" * 68)
|
||||
print(f" Cache dir : {CACHE_DIR}")
|
||||
print()
|
||||
for name, p in models.items():
|
||||
if p is None or not p.exists():
|
||||
print(f" [-] {name:20s} ABSENT")
|
||||
continue
|
||||
size_mb = _du_mb(p)
|
||||
print(f" [+] {name:20s} {size_mb:6.1f} MB {p.name}")
|
||||
print()
|
||||
print(" I/O attendu YOLO11n-pose CoreML :")
|
||||
print(" input : image 640x640 BGR (pixel buffer accepte via Vision)")
|
||||
print(" output: var-shape 'output0' (1, N, 56) = [box(4)+conf(1)+kp(17*3)]")
|
||||
print(" ou 'var_xxx' tenseur post-NMS (depend de la version ultralytics)")
|
||||
print()
|
||||
print(" Pour ANE compatibility : verifier dans Xcode Quick Look")
|
||||
print(" - ouvrir le .mlpackage")
|
||||
print(" - onglet Performance → 'Compute units: Neural Engine'")
|
||||
print(" - latency cible M5 : <8 ms par frame")
|
||||
print()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(prog="convert_coreml")
|
||||
parser.add_argument("--force", action="store_true",
|
||||
help="re-export meme si .mlpackage deja present")
|
||||
parser.add_argument("-v", "--verbose", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s — %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
LOG.info("cache dir : %s", CACHE_DIR)
|
||||
|
||||
results = {
|
||||
"yolo11n-pose": convert_yolo11n_pose(force=args.force),
|
||||
"rtmpose-m": convert_rtmpose_m(force=args.force),
|
||||
"dwpose-m": convert_dwpose_m(force=args.force),
|
||||
}
|
||||
report(results)
|
||||
# Exit code : 0 si au moins YOLO11n-pose est dispo (cas nominal).
|
||||
return 0 if results["yolo11n-pose"] is not None else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert DINOv2 ViT-S/14 to a CoreML .mlpackage for ANE-friendly inference.
|
||||
|
||||
The wrapped module takes (1, 3, 224, 224) RGB float32 in [0, 1], applies
|
||||
ImageNet normalization internally, runs the ViT, and returns the CLS
|
||||
embedding (1, 384) L2-normalised. We trace + convert with
|
||||
``coremltools.convert(... compute_units=ComputeUnit.ALL, compute_precision=FP16)``.
|
||||
|
||||
Run with the Python 3.12 venv that has coremltools and torch::
|
||||
|
||||
/tmp/coreml312/bin/python -m data_only_viz.scripts.convert_dinov2 [--force]
|
||||
|
||||
Output:
|
||||
~/.cache/av-live-multihmr/dinov2_vits14.mlpackage
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
LOG = logging.getLogger("convert_dinov2")
|
||||
|
||||
OUT_DIR = Path.home() / ".cache" / "av-live-multihmr"
|
||||
OUT_PATH = OUT_DIR / "dinov2_vits14.mlpackage"
|
||||
|
||||
_IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
||||
_IMAGENET_STD = (0.229, 0.224, 0.225)
|
||||
|
||||
|
||||
def _build_wrapper():
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
backbone = torch.hub.load(
|
||||
"facebookresearch/dinov2",
|
||||
"dinov2_vits14",
|
||||
source="github",
|
||||
trust_repo=True,
|
||||
)
|
||||
backbone.eval()
|
||||
|
||||
# Pretrained pos_embed is at 37x37 (518/14). We pre-resample to
|
||||
# 16x16 (224/14) once so the traced graph never needs an upsample.
|
||||
pe = backbone.pos_embed.data # (1, 1+37*37, 384)
|
||||
cls_pe = pe[:, :1]
|
||||
patch_pe = pe[:, 1:]
|
||||
n_old = int(round((patch_pe.shape[1]) ** 0.5))
|
||||
dim = patch_pe.shape[-1]
|
||||
patch_pe = patch_pe.reshape(1, n_old, n_old, dim).permute(0, 3, 1, 2)
|
||||
patch_pe = F.interpolate(patch_pe, size=(16, 16), mode="bilinear",
|
||||
align_corners=False)
|
||||
patch_pe = patch_pe.permute(0, 2, 3, 1).reshape(1, 16 * 16, dim)
|
||||
new_pe = torch.cat([cls_pe, patch_pe], dim=1).contiguous()
|
||||
backbone.pos_embed = nn.Parameter(new_pe, requires_grad=False)
|
||||
|
||||
mean = torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1)
|
||||
std = torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1)
|
||||
|
||||
class DinoV2Wrapper(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.backbone = backbone
|
||||
self.register_buffer("mean", mean)
|
||||
self.register_buffer("std", std)
|
||||
|
||||
def forward(self, x):
|
||||
x = (x - self.mean) / self.std
|
||||
bb = self.backbone
|
||||
x = bb.patch_embed(x)
|
||||
# cls_token is (1,1,384). Concat directly (B=1 fixed).
|
||||
x = torch.cat((bb.cls_token, x), dim=1)
|
||||
x = x + bb.pos_embed
|
||||
for blk in bb.blocks:
|
||||
x = blk(x)
|
||||
x = bb.norm(x)
|
||||
cls = x[:, 0]
|
||||
cls = cls / (cls.norm(dim=-1, keepdim=True) + 1e-8)
|
||||
return cls
|
||||
|
||||
return DinoV2Wrapper().eval()
|
||||
|
||||
|
||||
def _patch_coremltools_cast():
|
||||
"""coremltools 9.0 _cast assumes x.val is a 0-d scalar. With recent
|
||||
torch (2.12) some aten::Int args land as 1-D length-1 arrays. Patch
|
||||
the helper to flatten before scalar-casting."""
|
||||
from coremltools.converters.mil.frontend.torch import ops as _ops
|
||||
from coremltools.converters.mil.mil import Builder as mb
|
||||
|
||||
_orig = _ops._cast
|
||||
|
||||
def _patched_cast(context, node, dtype, dtype_name):
|
||||
# Inputs are read inside _orig from context; we wrap the failure
|
||||
# path by checking the first input's val first.
|
||||
inputs = _ops._get_inputs(context, node, expected=1)
|
||||
x = inputs[0]
|
||||
if x.can_be_folded_to_const():
|
||||
val = x.val
|
||||
if hasattr(val, "shape") and getattr(val, "shape", ()) != ():
|
||||
# 1-D length-1 (or all-ones shape) -> extract scalar
|
||||
import numpy as _np
|
||||
arr = _np.asarray(val).reshape(-1)
|
||||
if arr.size == 1:
|
||||
res = mb.const(val=dtype(arr[0]), name=node.name)
|
||||
context.add(res, node.name)
|
||||
return
|
||||
return _orig(context, node, dtype, dtype_name)
|
||||
|
||||
_ops._cast = _patched_cast
|
||||
|
||||
|
||||
def convert(force: bool = False) -> Path:
|
||||
import torch
|
||||
import coremltools as ct
|
||||
_patch_coremltools_cast()
|
||||
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if OUT_PATH.exists() and not force:
|
||||
LOG.info("already converted: %s", OUT_PATH)
|
||||
return OUT_PATH
|
||||
|
||||
LOG.info("loading DINOv2 ViT-S/14 ...")
|
||||
wrap = _build_wrapper()
|
||||
example = torch.rand(1, 3, 224, 224, dtype=torch.float32)
|
||||
with torch.no_grad():
|
||||
ref_out = wrap(example)
|
||||
LOG.info("torch out shape=%s norm=%.4f", tuple(ref_out.shape),
|
||||
float(ref_out.norm(dim=-1).mean()))
|
||||
|
||||
LOG.info("tracing ...")
|
||||
with torch.no_grad():
|
||||
traced = torch.jit.trace(wrap, example, strict=False)
|
||||
|
||||
LOG.info("ct.convert (mlprogram FP16, computeUnits=ALL) ...")
|
||||
mlmodel = ct.convert(
|
||||
traced,
|
||||
source="pytorch",
|
||||
convert_to="mlprogram",
|
||||
inputs=[ct.TensorType(name="image", shape=example.shape,
|
||||
dtype=np.float32)],
|
||||
outputs=[ct.TensorType(name="embedding", dtype=np.float32)],
|
||||
compute_precision=ct.precision.FLOAT16,
|
||||
compute_units=ct.ComputeUnit.ALL,
|
||||
minimum_deployment_target=ct.target.macOS14,
|
||||
)
|
||||
mlmodel.short_description = "DINOv2 ViT-S/14 person re-id (384-D, L2)"
|
||||
mlmodel.save(str(OUT_PATH))
|
||||
LOG.info("saved %s", OUT_PATH)
|
||||
|
||||
pred = mlmodel.predict({"image": example.numpy().astype(np.float32)})
|
||||
coreml_out = list(pred.values())[0].reshape(-1)
|
||||
ref_np = ref_out.numpy().reshape(-1)
|
||||
cos = float(np.dot(coreml_out, ref_np) /
|
||||
(np.linalg.norm(coreml_out) * np.linalg.norm(ref_np) + 1e-8))
|
||||
LOG.info("CoreML vs Torch cosine on random input: %.4f", cos)
|
||||
return OUT_PATH
|
||||
|
||||
|
||||
def bench(n_iter: int = 30) -> None:
|
||||
import coremltools as ct
|
||||
LOG.info("bench: load mlpackage ...")
|
||||
m = ct.models.MLModel(str(OUT_PATH),
|
||||
compute_units=ct.ComputeUnit.ALL)
|
||||
crop = np.random.rand(1, 3, 224, 224).astype(np.float32)
|
||||
for _ in range(3):
|
||||
m.predict({"image": crop})
|
||||
times = []
|
||||
for _ in range(n_iter):
|
||||
t0 = time.perf_counter()
|
||||
m.predict({"image": crop})
|
||||
times.append((time.perf_counter() - t0) * 1e3)
|
||||
times.sort()
|
||||
p50 = times[len(times) // 2]
|
||||
p95 = times[int(len(times) * 0.95)]
|
||||
LOG.info("bench %d iter: p50=%.2f ms p95=%.2f ms mean=%.2f ms (~%.1f fps)",
|
||||
n_iter, p50, p95, sum(times) / len(times), 1000.0 / p50)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format="%(asctime)s %(name)s %(message)s")
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--force", action="store_true")
|
||||
ap.add_argument("--bench-only", action="store_true")
|
||||
ap.add_argument("--n-iter", type=int, default=30)
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.bench_only:
|
||||
convert(force=args.force)
|
||||
bench(n_iter=args.n_iter)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,571 @@
|
||||
"""Task 3 — Convert FULL Multi-HMR (backbone + head) to CoreML
|
||||
avec apply_topk(K=4) + fixed-shape tuple output.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
CACHE = Path.home() / ".cache" / "av-live-multihmr"
|
||||
_CKPT_NAME = os.environ.get("MULTIHMR_CKPT_NAME", "multiHMR_672_S.pt")
|
||||
CKPT = CACHE / "checkpoints" / _CKPT_NAME
|
||||
_OUT_NAME = os.environ.get("MULTIHMR_OUT_NAME",
|
||||
_CKPT_NAME.replace(".pt", ".mlpackage").lower())
|
||||
MULTIHMR_REPO = CACHE / "multi-hmr"
|
||||
|
||||
sys.path.insert(0, str(MULTIHMR_REPO))
|
||||
for mod in ("pyrender", "pyvista", "anny"):
|
||||
sys.modules.setdefault(mod, types.ModuleType(mod))
|
||||
|
||||
DEVICE = "cpu" # trace on CPU (CoreML mlprogram doesn't care)
|
||||
IMG_SIZE = 672
|
||||
K_PERSONS = 4
|
||||
|
||||
|
||||
# === apply_topk replacement (validated equivalent in Task 2) ===
|
||||
def apply_topk(K, _scores):
|
||||
if isinstance(K, list):
|
||||
K = K[0]
|
||||
B, H, W, C = _scores.shape
|
||||
flat = _scores.reshape(B, -1)
|
||||
_, idx_flat = torch.topk(flat, k=K, dim=1)
|
||||
wc = W * C
|
||||
idx_b = (torch.arange(B, device=_scores.device)
|
||||
.unsqueeze(1).expand(-1, K).reshape(-1).long())
|
||||
idx_flat_flat = idx_flat.reshape(-1)
|
||||
idx_h = (idx_flat_flat // wc).long()
|
||||
idx_w = ((idx_flat_flat // C) % W).long()
|
||||
idx_c = (idx_flat_flat % C).long()
|
||||
return (idx_b, idx_h, idx_w, idx_c)
|
||||
|
||||
|
||||
# === Patch coremltools _cast (validated probe v4) ===
|
||||
# Patch _auto_val pour coercer values 1-d size-1 -> 0-d
|
||||
def _install_auto_val_patch():
|
||||
from coremltools.converters.mil.mil import operation as _opmod
|
||||
from coremltools.converters.mil.mil.operation import mil_list
|
||||
|
||||
_orig_auto_val = _opmod.Operation._auto_val
|
||||
|
||||
def _patched_auto_val(self, output_types):
|
||||
try:
|
||||
return _orig_auto_val(self, output_types)
|
||||
except ValueError as e:
|
||||
if "zero-rank" not in str(e):
|
||||
raise
|
||||
# Retry avec coercion 1-d size-1 -> 0-d
|
||||
try:
|
||||
vals = self.value_inference()
|
||||
except NotImplementedError:
|
||||
return tuple(None for _ in output_types)
|
||||
if not isinstance(vals, (tuple, list)):
|
||||
vals = (vals,)
|
||||
for val in vals:
|
||||
if val is None:
|
||||
return tuple(None for _ in output_types)
|
||||
auto = []
|
||||
for t, v in zip(output_types, vals):
|
||||
bv = t()
|
||||
if isinstance(v, mil_list):
|
||||
bv.val = v.ls
|
||||
else:
|
||||
if isinstance(v, np.ndarray) and v.ndim > 0 and v.size == 1:
|
||||
# Coerce 1-d size-1 -> 0-d ndarray (val setter
|
||||
# accepte np.generic ou ndarray ndim==0).
|
||||
v = np.asarray(v.reshape(()))
|
||||
elif isinstance(v, (int, float)) and not isinstance(
|
||||
v, (np.generic,)):
|
||||
v = np.asarray(v)
|
||||
bv.val = v
|
||||
auto.append(bv)
|
||||
return auto
|
||||
|
||||
_opmod.Operation._auto_val = _patched_auto_val
|
||||
|
||||
|
||||
def _patched_cast(context, node, dtype, dtype_str):
|
||||
from coremltools.converters.mil import Builder as mb
|
||||
from coremltools.converters.mil.frontend.torch import ops as _ops
|
||||
inputs = _ops._get_inputs(context, node, expected=1)
|
||||
x = inputs[0]
|
||||
if x.val is not None:
|
||||
try:
|
||||
const_val = dtype(x.val)
|
||||
except TypeError:
|
||||
arr = np.asarray(x.val)
|
||||
if arr.size == 1:
|
||||
const_val = dtype(arr.item())
|
||||
else:
|
||||
res = mb.cast(x=x, dtype=dtype_str, name=node.name)
|
||||
context.add(res)
|
||||
return
|
||||
res = mb.const(val=const_val, name=node.name)
|
||||
else:
|
||||
res = mb.cast(x=x, dtype=dtype_str, name=node.name)
|
||||
context.add(res)
|
||||
|
||||
|
||||
prev = os.getcwd()
|
||||
try:
|
||||
os.chdir(MULTIHMR_REPO)
|
||||
from model import Model
|
||||
import model as model_mod
|
||||
|
||||
# Inject topk replacement
|
||||
print("==> Patching apply_threshold -> apply_topk(K=4)")
|
||||
model_mod.apply_threshold = lambda thr, scores: apply_topk(K_PERSONS, scores)
|
||||
|
||||
torch_dev = torch.device(DEVICE)
|
||||
ckpt = torch.load(str(CKPT), map_location=torch_dev, weights_only=False)
|
||||
kw = {k: v for k, v in vars(ckpt["args"]).items()}
|
||||
kw["type"] = ckpt["args"].train_return_type
|
||||
kw["img_size"] = ckpt["args"].img_size[0]
|
||||
print(f"==> Loading Multi-HMR ViT-S 672 (params count tbd)")
|
||||
model = Model(**kw).to(torch_dev)
|
||||
model.load_state_dict(ckpt["model_state_dict"], strict=False)
|
||||
model.eval()
|
||||
finally:
|
||||
os.chdir(prev)
|
||||
|
||||
|
||||
# === Pre-compute interpolate_pos_encoding (probe v4 fix) ===
|
||||
# Multi-HMR's backbone is DINOv2 ViT-S/14 — same dynamic interpolation
|
||||
# problem that planted la conversion sur backbone seul.
|
||||
if hasattr(model.backbone, "encoder") and hasattr(model.backbone.encoder,
|
||||
"interpolate_pos_encoding"):
|
||||
print("==> Patching backbone.encoder.interpolate_pos_encoding (pre-compute)")
|
||||
bk = model.backbone.encoder
|
||||
with torch.no_grad():
|
||||
dummy_x = torch.rand(1, 3, IMG_SIZE, IMG_SIZE)
|
||||
dummy_p = bk.patch_embed(dummy_x)
|
||||
cls = bk.cls_token.expand(dummy_p.shape[0], -1, -1)
|
||||
x_full = torch.cat((cls, dummy_p), dim=1)
|
||||
cached_pe = bk.interpolate_pos_encoding(
|
||||
x_full, IMG_SIZE, IMG_SIZE).detach()
|
||||
bk.register_buffer("_cached_pos_embed", cached_pe)
|
||||
|
||||
def fixed_pe(self, x, w, h):
|
||||
return self._cached_pos_embed.to(x.dtype)
|
||||
bk.interpolate_pos_encoding = types.MethodType(fixed_pe, bk)
|
||||
print(f" cached shape {tuple(cached_pe.shape)}")
|
||||
|
||||
# === Patch utils.camera.inverse_perspective_projection ===
|
||||
# torch.inverse(K) plante coremltools (op non implementee). Comme K est
|
||||
# fixe (camera intrinsics avec focal=IMG_SIZE), on pre-calcule K_inv
|
||||
# en closed-form et on l'utilise comme buffer module-level.
|
||||
print("==> Patching roma.rotmat_to_rotvec (branchless atan2)")
|
||||
# roma.rotmat_to_rotvec utilise torch.empty + 8 index_put_ qui se
|
||||
# traduisent en CoreML par scatter_nd successifs sur un buffer
|
||||
# garbage-initialise. Resultat : cellules non touchees restent NaN,
|
||||
# propagees via quat normalization -> v3d/transl all-NaN.
|
||||
# Remplacement branchless via atan2 : pas de torch.empty, pas
|
||||
# d'index_put_, juste des stack/clamp/norm/atan2 stables CoreML.
|
||||
# Precision vs roma original : 2.26e-6 L_inf sur batch random.
|
||||
import roma as _roma
|
||||
|
||||
def _rotmat_to_rotvec_branchless(R, eps=1e-6):
|
||||
w = torch.stack([
|
||||
R[..., 2, 1] - R[..., 1, 2],
|
||||
R[..., 0, 2] - R[..., 2, 0],
|
||||
R[..., 1, 0] - R[..., 0, 1],
|
||||
], dim=-1) * 0.5
|
||||
trace = R[..., 0, 0] + R[..., 1, 1] + R[..., 2, 2]
|
||||
cos_theta = ((trace - 1.0) * 0.5).clamp(-1.0, 1.0)
|
||||
sin_theta = torch.norm(w, dim=-1)
|
||||
theta = torch.atan2(sin_theta, cos_theta)
|
||||
sin_theta_safe = sin_theta.clamp(min=eps)
|
||||
return w * (theta / sin_theta_safe).unsqueeze(-1)
|
||||
|
||||
_roma.rotmat_to_rotvec = _rotmat_to_rotvec_branchless
|
||||
|
||||
|
||||
print("==> Patching utils.camera.inverse_perspective_projection")
|
||||
import utils.camera as _camera
|
||||
|
||||
# Pre-compute K_inv closed-form pour notre K standard
|
||||
focal_val = float(IMG_SIZE)
|
||||
cx = cy = IMG_SIZE / 2.0
|
||||
_K_INV_PRE = torch.tensor([
|
||||
[[1.0 / focal_val, 0.0, -cx / focal_val],
|
||||
[0.0, 1.0 / focal_val, -cy / focal_val],
|
||||
[0.0, 0.0, 1.0]]
|
||||
])
|
||||
|
||||
def inverse_perspective_projection_fixed(points, K, distance):
|
||||
"""Bypass torch.inverse + einsum + matmul pour eviter le bug
|
||||
coremltools de broadcast batch 1->K sur ces ops. K_inv etant
|
||||
fixe et structure (diag + translate), on ecrit les composantes
|
||||
explicitement en ops elementaires.
|
||||
|
||||
K_inv = [[1/f, 0, -cx/f], [0, 1/f, -cy/f], [0, 0, 1]]
|
||||
Pour points (b, N, 3) : out = points @ K_inv.T donne :
|
||||
out[..., 0] = points[..., 0]/f - (cx/f) * points[..., 2]
|
||||
out[..., 1] = points[..., 1]/f - (cy/f) * points[..., 2]
|
||||
out[..., 2] = points[..., 2]
|
||||
"""
|
||||
points_hom = torch.cat([points, torch.ones_like(points[..., :1])], -1)
|
||||
inv_f = 1.0 / focal_val
|
||||
cx_over_f = cx / focal_val
|
||||
cy_over_f = cy / focal_val
|
||||
x = points_hom[..., 0:1]
|
||||
y = points_hom[..., 1:2]
|
||||
z = points_hom[..., 2:3]
|
||||
out0 = x * inv_f - z * cx_over_f
|
||||
out1 = y * inv_f - z * cy_over_f
|
||||
out2 = z
|
||||
points = torch.cat([out0, out1, out2], dim=-1)
|
||||
if distance is None:
|
||||
return points
|
||||
points = points * distance
|
||||
return points
|
||||
|
||||
_camera.inverse_perspective_projection = inverse_perspective_projection_fixed
|
||||
# Aussi patcher le re-export dans utils/__init__.py et model.py
|
||||
import utils as _utils_pkg
|
||||
_utils_pkg.inverse_perspective_projection = inverse_perspective_projection_fixed
|
||||
# model.py importe directement : monkey-patch sur le module
|
||||
model_mod.inverse_perspective_projection = inverse_perspective_projection_fixed
|
||||
# Idem smpl_layer
|
||||
import blocks.smpl_layer as _smpl_layer
|
||||
_smpl_layer.inverse_perspective_projection = inverse_perspective_projection_fixed
|
||||
|
||||
# Aussi perspective_projection (utilise dans smpl_layer.py:143-144 pour
|
||||
# j2d et v2d) -> rewrite einsum en matmul pour le meme broadcast bug.
|
||||
def perspective_projection_fixed(x, K):
|
||||
"""Element-wise rewrite de la projection perspective avec K fixe
|
||||
(focal=IMG_SIZE, cx=cy=IMG_SIZE/2). Bypass matmul/einsum pour eviter
|
||||
les bugs broadcast coremltools.
|
||||
K = [[f, 0, cx], [0, f, cy], [0, 0, 1]]
|
||||
out[..., 0] = f * x_norm + cx * z_norm (mais on veut [..., :2])
|
||||
= f * (x/z) + cx
|
||||
out[..., 1] = f * (y/z) + cy
|
||||
"""
|
||||
z = x[..., 2:3]
|
||||
px = x[..., 0:1] / z * focal_val + cx
|
||||
py = x[..., 1:2] / z * focal_val + cy
|
||||
return torch.cat([px, py], dim=-1)
|
||||
|
||||
_camera.perspective_projection = perspective_projection_fixed
|
||||
_utils_pkg.perspective_projection = perspective_projection_fixed
|
||||
_smpl_layer.perspective_projection = perspective_projection_fixed
|
||||
|
||||
|
||||
# === Wrapper qui produit tuple fixe ===
|
||||
class TracedMHMR(nn.Module):
|
||||
"""Wrap Multi-HMR pour trace : output tuple de tensors fixes,
|
||||
pas de list-of-dicts."""
|
||||
|
||||
def __init__(self, m: Model):
|
||||
super().__init__()
|
||||
self.m = m
|
||||
|
||||
def forward(self, x: torch.Tensor, cam_K: torch.Tensor):
|
||||
# Call original forward with is_training=False ; apply_topk
|
||||
# garantit toujours K=4 detections donc le loop dans le forward
|
||||
# est unroll-friendly.
|
||||
humans = self.m(x, is_training=False, nms_kernel_size=5,
|
||||
det_thresh=0.0, K=cam_K)
|
||||
# humans est une list[dict] de longueur 4. Stack en tensors.
|
||||
if len(humans) == 0:
|
||||
# Should not happen with apply_topk, but defensive
|
||||
zeros = torch.zeros(K_PERSONS, 10475, 3)
|
||||
zeros_p = torch.zeros(K_PERSONS, 3)
|
||||
zeros_s = torch.zeros(K_PERSONS)
|
||||
zeros_b = torch.zeros(K_PERSONS, 10)
|
||||
zeros_j = torch.zeros(K_PERSONS, 127, 3)
|
||||
return zeros, zeros_p, zeros_s, zeros_b, zeros_b, zeros_j
|
||||
v3d = torch.stack([h["v3d"] for h in humans])
|
||||
transl = torch.stack([h["transl_pelvis"] for h in humans])
|
||||
scores = torch.stack([
|
||||
h["scores"] if h["scores"].dim() > 0 else h["scores"].unsqueeze(0)
|
||||
for h in humans
|
||||
]).squeeze(-1)
|
||||
shape = torch.stack([h["shape"] for h in humans])
|
||||
expr = torch.stack([h["expression"] for h in humans])
|
||||
# Joints (SMPL-X). smplx.create(use_pca=False) populates
|
||||
# output.joints of shape (B, 127, 3) which is then carried as
|
||||
# 'j3d' in smpl_layer.py:148 already in camera space (same
|
||||
# transl_up applied as v3d). The first 55 are the standard
|
||||
# SMPL-X joints (22 body + jaw + 2 eyes + 30 fingers); the
|
||||
# remaining 72 are face/landmark anchors. Downstream code can
|
||||
# slice [..., :55, :] if it only needs the skeleton.
|
||||
j3d = torch.stack([h["j3d"] for h in humans])
|
||||
return v3d, transl, scores, shape, expr, j3d
|
||||
|
||||
|
||||
wrapper = TracedMHMR(model).eval()
|
||||
|
||||
# Sanity forward
|
||||
focal = float(IMG_SIZE)
|
||||
example_K = torch.tensor(
|
||||
[[[focal, 0.0, IMG_SIZE / 2.0],
|
||||
[0.0, focal, IMG_SIZE / 2.0],
|
||||
[0.0, 0.0, 1.0]]], dtype=torch.float32)
|
||||
example_x = torch.rand(1, 3, IMG_SIZE, IMG_SIZE)
|
||||
|
||||
print("==> Sanity forward")
|
||||
with torch.no_grad():
|
||||
v3d, transl, scores, shape, expr, joints = wrapper(example_x, example_K)
|
||||
print(f" v3d: {tuple(v3d.shape)}, transl: {tuple(transl.shape)},")
|
||||
print(f" scores: {tuple(scores.shape)}, shape: {tuple(shape.shape)},")
|
||||
print(f" expr: {tuple(expr.shape)}, joints: {tuple(joints.shape)}")
|
||||
|
||||
print("==> torch.jit.trace")
|
||||
try:
|
||||
traced = torch.jit.trace(wrapper, (example_x, example_K), strict=False)
|
||||
print(" trace OK")
|
||||
except Exception as e:
|
||||
print(f" trace FAILED: {type(e).__name__}: {e}")
|
||||
raise
|
||||
|
||||
# === CoreML convert ===
|
||||
print("==> coremltools.convert")
|
||||
import coremltools as ct
|
||||
from coremltools.converters.mil.frontend.torch import ops as _ops
|
||||
_ops._cast = _patched_cast
|
||||
_install_auto_val_patch()
|
||||
|
||||
# Instrument convert_single_node pour logger le node responsable
|
||||
# de l'erreur (cascade-debug helper).
|
||||
_orig_csn = _ops.convert_single_node
|
||||
|
||||
|
||||
def _csn_logged(context, node):
|
||||
try:
|
||||
_orig_csn(context, node)
|
||||
except Exception:
|
||||
try:
|
||||
k = node.kind() if callable(node.kind) else str(node.kind)
|
||||
print(f" >>> FAIL on torch node kind={k}")
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
_ops.convert_single_node = _csn_logged
|
||||
|
||||
# Patch tile op pour gerer reps=[] (no-op = return input unchanged).
|
||||
# Le pattern apparait quand torch.repeat(*[]) ou expand sur dim ratée.
|
||||
from coremltools.converters.mil.mil.ops.defs.iOS15 import tensor_operation as _tens_op
|
||||
_orig_tile_type_inf = _tens_op.tile.type_inference
|
||||
|
||||
|
||||
def _tile_type_inf_safe(self):
|
||||
reps = self.reps.val if self.reps.val is not None else self.reps
|
||||
try:
|
||||
return _orig_tile_type_inf(self)
|
||||
except ValueError as e:
|
||||
if "reps" in str(e) and "0" in str(e):
|
||||
print(f" >>> tile no-op : reps empty, returning input shape")
|
||||
# No-op : return type of input x unchanged
|
||||
return self.x.sym_type
|
||||
raise
|
||||
|
||||
|
||||
_tens_op.tile.type_inference = _tile_type_inf_safe
|
||||
|
||||
# Register `new_ones` converter (aten::new_ones).
|
||||
# Signature: new_ones(self, size, dtype=None, layout=None, ...)
|
||||
# Equivalent : fill(shape=size, value=1.0) cast vers self.dtype.
|
||||
from coremltools.converters.mil import Builder as _mb
|
||||
from coremltools.converters.mil.frontend.torch.ops import (
|
||||
_get_inputs, register_torch_op as _reg)
|
||||
|
||||
|
||||
from coremltools.converters.mil.frontend.torch.torch_op_registry import (
|
||||
_TORCH_OPS_REGISTRY)
|
||||
|
||||
|
||||
def _maybe_register(name, fn):
|
||||
"""Register fn under torch op name only if not already registered."""
|
||||
try:
|
||||
if name not in _TORCH_OPS_REGISTRY.name_to_func_mapping:
|
||||
_TORCH_OPS_REGISTRY.register_func(fn, [name], override=False)
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
def _new_ones(context, node):
|
||||
inputs = _get_inputs(context, node, min_expected=2)
|
||||
size = inputs[1]
|
||||
if isinstance(size, (list, tuple)):
|
||||
from coremltools.converters.mil import Builder as mb
|
||||
# Reshape chaque element a rank 1 avant concat (sinon mix 0d/1d
|
||||
# plante avec "Input has rank 0 != other inputs rank 1").
|
||||
size_1d = []
|
||||
for v in size:
|
||||
r = v.rank if hasattr(v, "rank") else None
|
||||
if r == 0:
|
||||
v = mb.expand_dims(x=v, axes=[0])
|
||||
size_1d.append(v)
|
||||
size = mb.concat(values=size_1d, axis=0)
|
||||
res = _mb.fill(shape=size, value=1.0, name=node.name)
|
||||
context.add(res, node.name)
|
||||
|
||||
|
||||
_maybe_register("new_ones", _new_ones)
|
||||
|
||||
|
||||
# Patch global concat type_inference : auto-promote 0d → 1d.
|
||||
_orig_concat_ti = _tens_op.concat.type_inference
|
||||
|
||||
|
||||
def _concat_ti_auto_promote(self):
|
||||
try:
|
||||
return _orig_concat_ti(self)
|
||||
except ValueError as e:
|
||||
if "rank 0" in str(e) and "rank 1" in str(e):
|
||||
# Find 0d inputs and replace via expand_dims
|
||||
from coremltools.converters.mil import Builder as mb
|
||||
promoted = []
|
||||
for v in self.values:
|
||||
if v.rank == 0:
|
||||
v = mb.expand_dims(x=v, axes=[0])
|
||||
promoted.append(v)
|
||||
self.values = promoted
|
||||
return _orig_concat_ti(self)
|
||||
raise
|
||||
|
||||
|
||||
_tens_op.concat.type_inference = _concat_ti_auto_promote
|
||||
|
||||
|
||||
# Override clamp_min : promote dtypes (original assert sans promotion).
|
||||
from coremltools.converters.mil.frontend.torch.ops import (
|
||||
promote_input_dtypes)
|
||||
|
||||
|
||||
def _clamp_min_promote(context, node):
|
||||
inputs = _get_inputs(context, node, expected=2)
|
||||
x, y = promote_input_dtypes([inputs[0], inputs[1]])
|
||||
out = _mb.maximum(x=x, y=y, name=node.name)
|
||||
context.add(out)
|
||||
|
||||
|
||||
_TORCH_OPS_REGISTRY.name_to_func_mapping["clamp_min"] = _clamp_min_promote
|
||||
|
||||
|
||||
def _clamp_max_promote(context, node):
|
||||
inputs = _get_inputs(context, node, expected=2)
|
||||
x, y = promote_input_dtypes([inputs[0], inputs[1]])
|
||||
out = _mb.minimum(x=x, y=y, name=node.name)
|
||||
context.add(out)
|
||||
|
||||
|
||||
_TORCH_OPS_REGISTRY.name_to_func_mapping["clamp_max"] = _clamp_max_promote
|
||||
|
||||
|
||||
# Override diagonal pour supporter dim1=1, dim2=2 sur tensor (B, N, N).
|
||||
# Multi-HMR via roma.rotmat_to_rotvec utilise .diagonal(dim1=1, dim2=2).
|
||||
def _diagonal_general(context, node):
|
||||
inputs = _get_inputs(context, node, expected=[1, 4])
|
||||
x = inputs[0]
|
||||
offset = inputs[1].val if len(inputs) > 1 and inputs[1] is not None else 0
|
||||
dim1 = inputs[2].val if len(inputs) > 2 and inputs[2] is not None else 0
|
||||
dim2 = inputs[3].val if len(inputs) > 3 and inputs[3] is not None else 1
|
||||
|
||||
# Pour notre cas type (B, N, N) avec dim1=1 dim2=2 et offset=0 :
|
||||
# reshape (B, N, N) -> (B, N*N), gather indices [0, N+1, 2N+2, ...].
|
||||
if offset == 0 and x.rank == 3 and dim1 == 1 and dim2 == 2:
|
||||
N = x.shape[1]
|
||||
# Indices diagonale aplatis : i * N + i
|
||||
diag_idx = np.array([i * N + i for i in range(N)],
|
||||
dtype=np.int32)
|
||||
x_flat = _mb.reshape(x=x, shape=[x.shape[0], N * N])
|
||||
out = _mb.gather(x=x_flat, indices=diag_idx, axis=1,
|
||||
name=node.name)
|
||||
context.add(out)
|
||||
return
|
||||
|
||||
# Fallback : on garde le path original (offset=0 dim1=0 dim2=1)
|
||||
if offset == 0 and dim1 == 0 and dim2 == 1:
|
||||
diag = _mb.band_part(x=x, lower=0, upper=0, name=node.name)
|
||||
context.add(diag)
|
||||
return
|
||||
|
||||
raise NotImplementedError(
|
||||
f"diagonal: offset={offset} dim1={dim1} dim2={dim2} rank={x.rank} "
|
||||
"non gere — etendre _diagonal_general")
|
||||
|
||||
|
||||
_TORCH_OPS_REGISTRY.name_to_func_mapping["diagonal"] = _diagonal_general
|
||||
|
||||
|
||||
# Instrument reshape pour logger node source au moment de l'erreur.
|
||||
from coremltools.converters.mil.mil.ops.defs.iOS15 import tensor_transformation as _tt
|
||||
_orig_reshape_ti = _tt.reshape.type_inference
|
||||
|
||||
|
||||
def _reshape_ti_logged(self):
|
||||
try:
|
||||
return _orig_reshape_ti(self)
|
||||
except ValueError as e:
|
||||
if "Invalid target shape" in str(e):
|
||||
try:
|
||||
from_shape = list(self.x.shape)
|
||||
target = list(self.shape.val) if hasattr(self.shape, "val") else "?"
|
||||
print(f" >>> RESHAPE FAIL : name={self.name} from={from_shape} target={target}")
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
_tt.reshape.type_inference = _reshape_ti_logged
|
||||
|
||||
try:
|
||||
mlmodel = ct.convert(
|
||||
traced,
|
||||
inputs=[
|
||||
ct.TensorType(shape=(1, 3, IMG_SIZE, IMG_SIZE),
|
||||
name="image", dtype=np.float32),
|
||||
ct.TensorType(shape=(1, 3, 3), name="cam_K", dtype=np.float32),
|
||||
],
|
||||
compute_units=ct.ComputeUnit.CPU_AND_GPU,
|
||||
minimum_deployment_target=ct.target.macOS15,
|
||||
convert_to="mlprogram",
|
||||
# FP32 mandatory : FP16 (global ou hybride op_selector) degrade
|
||||
# visiblement le mesh sur poses extremes. INT8 weight quant
|
||||
# teste 2026-05-14 : aucun gain sur GPU compute-bound.
|
||||
compute_precision=ct.precision.FLOAT32,
|
||||
)
|
||||
out_path = f"/tmp/{_OUT_NAME}"
|
||||
mlmodel.save(out_path)
|
||||
print(f" CONVERT OK -> {out_path}")
|
||||
# Dump output names + shapes so we can wire OUT_* constants.
|
||||
try:
|
||||
spec = mlmodel.get_spec()
|
||||
print("==> mlpackage outputs:")
|
||||
for o in spec.description.output:
|
||||
mt = o.type.multiArrayType
|
||||
shape = list(mt.shape) if mt is not None else []
|
||||
print(f" {o.name} shape={shape}")
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" spec dump failed: {e}")
|
||||
except Exception as e:
|
||||
print(f" CONVERT FAILED: {type(e).__name__}: {e}")
|
||||
raise
|
||||
|
||||
# === Bench ===
|
||||
print("==> bench 30 iter")
|
||||
img = np.random.rand(1, 3, IMG_SIZE, IMG_SIZE).astype(np.float32)
|
||||
cam = np.array([[[focal, 0, IMG_SIZE/2],
|
||||
[0, focal, IMG_SIZE/2],
|
||||
[0, 0, 1]]], dtype=np.float32)
|
||||
for _ in range(3):
|
||||
_ = mlmodel.predict({"image": img, "cam_K": cam})
|
||||
t = []
|
||||
for _ in range(30):
|
||||
t0 = time.perf_counter()
|
||||
_ = mlmodel.predict({"image": img, "cam_K": cam})
|
||||
t.append((time.perf_counter() - t0) * 1000)
|
||||
t.sort()
|
||||
print(f" CoreML full Multi-HMR median={t[15]:.1f} ms "
|
||||
f"p10={t[3]:.1f} p90={t[27]:.1f} min={t[0]:.1f}")
|
||||
print(f" Target was <60ms (12-25 fps). Achieved: {1000.0/t[15]:.1f} fps")
|
||||
@@ -0,0 +1,113 @@
|
||||
"""DINOv2 ViT-S 672x672 backbone CoreML conversion + bench.
|
||||
|
||||
Probe v4 (2026-05-13) — résultat : conversion OK avec 2 patches,
|
||||
bench M5 CoreML CPU_AND_GPU = 25 ms vs PyTorch MPS = 275 ms = 11.8x
|
||||
speedup. ANE compute unit n'apporte rien (et ralentit) sur ce modele.
|
||||
|
||||
Patches requis :
|
||||
1. Pre-calculer interpolate_pos_encoding en buffer fige (sinon
|
||||
coremltools rejette l'interpolation dynamique).
|
||||
2. Patcher coremltools._cast pour gerer val non-scalaire via
|
||||
numpy.asarray().item() ou fallback mb.cast (sinon plante
|
||||
`dtype(x.val)` sur shape arithmetic int().
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import types
|
||||
import numpy as np
|
||||
import torch
|
||||
import coremltools as ct
|
||||
|
||||
H = W = 672
|
||||
|
||||
|
||||
def _patched_cast(context, node, dtype, dtype_str):
|
||||
"""Wrap coremltools _cast pour gerer x.val non-0d."""
|
||||
from coremltools.converters.mil import Builder as mb
|
||||
from coremltools.converters.mil.frontend.torch import ops as _ops
|
||||
inputs = _ops._get_inputs(context, node, expected=1)
|
||||
x = inputs[0]
|
||||
if x.val is not None:
|
||||
v = x.val
|
||||
try:
|
||||
const_val = dtype(v)
|
||||
except TypeError:
|
||||
arr = np.asarray(v)
|
||||
if arr.size == 1:
|
||||
const_val = dtype(arr.item())
|
||||
else:
|
||||
res = mb.cast(x=x, dtype=dtype_str, name=node.name)
|
||||
context.add(res)
|
||||
return
|
||||
res = mb.const(val=const_val, name=node.name)
|
||||
else:
|
||||
res = mb.cast(x=x, dtype=dtype_str, name=node.name)
|
||||
context.add(res)
|
||||
|
||||
|
||||
def install_coreml_patches() -> None:
|
||||
from coremltools.converters.mil.frontend.torch import ops as _ops
|
||||
_ops._cast = _patched_cast
|
||||
|
||||
|
||||
def build_dinov2_with_fixed_pos_embed():
|
||||
model = torch.hub.load(
|
||||
"facebookresearch/dinov2", "dinov2_vits14",
|
||||
pretrained=True, trust_repo=True)
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
dummy_p = model.patch_embed(torch.rand(1, 3, H, W))
|
||||
cls = model.cls_token.expand(dummy_p.shape[0], -1, -1)
|
||||
x_full = torch.cat((cls, dummy_p), dim=1)
|
||||
cached_pe = model.interpolate_pos_encoding(x_full, H, W).detach()
|
||||
model.register_buffer("_cached_pos_embed", cached_pe)
|
||||
|
||||
def fixed_pe(self, x, w, h):
|
||||
return self._cached_pos_embed.to(x.dtype)
|
||||
model.interpolate_pos_encoding = types.MethodType(fixed_pe, model)
|
||||
return model
|
||||
|
||||
|
||||
def convert(model, out_path: str) -> ct.models.MLModel:
|
||||
example = torch.rand(1, 3, H, W)
|
||||
traced = torch.jit.trace(model, example, strict=False)
|
||||
mlmodel = ct.convert(
|
||||
traced,
|
||||
inputs=[ct.TensorType(
|
||||
shape=(1, 3, H, W), name="image", dtype=np.float32)],
|
||||
compute_units=ct.ComputeUnit.CPU_AND_GPU,
|
||||
minimum_deployment_target=ct.target.macOS15,
|
||||
convert_to="mlprogram",
|
||||
)
|
||||
mlmodel.save(out_path)
|
||||
return mlmodel
|
||||
|
||||
|
||||
def bench(mlmodel, n: int = 50) -> dict:
|
||||
img = np.random.rand(1, 3, H, W).astype(np.float32)
|
||||
for _ in range(5):
|
||||
_ = mlmodel.predict({"image": img})
|
||||
t = []
|
||||
for _ in range(n):
|
||||
t0 = time.perf_counter()
|
||||
_ = mlmodel.predict({"image": img})
|
||||
t.append((time.perf_counter() - t0) * 1000)
|
||||
t.sort()
|
||||
return {
|
||||
"median_ms": t[n // 2],
|
||||
"p10_ms": t[max(0, int(n * 0.1) - 1)],
|
||||
"p90_ms": t[min(n - 1, int(n * 0.9))],
|
||||
"min_ms": t[0],
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
install_coreml_patches()
|
||||
model = build_dinov2_with_fixed_pos_embed()
|
||||
out = "/tmp/dinov2_vits14_672.mlpackage"
|
||||
print(f"==> convert -> {out}")
|
||||
mlmodel = convert(model, out)
|
||||
print(f"==> bench 50 iter")
|
||||
stats = bench(mlmodel)
|
||||
print(f" CoreML CPU_AND_GPU : {stats}")
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Extrait les 13776 triangles SMPL (6890 vertices) et les serialise en
|
||||
binaire little-endian (uint32) pour consommation par l'app Swift RealityKit.
|
||||
|
||||
Strategie : tente d'abord d'extraire depuis nlf_data_files.zip si present,
|
||||
sinon charge le modele TorchScript et tente d'acceder aux faces embarquees,
|
||||
sinon telecharge le fichier SMPL faces standard depuis un repo open-source.
|
||||
"""
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
CACHE = Path.home() / ".cache" / "av-live-nlf"
|
||||
OUT = (Path(__file__).parent.parent.parent
|
||||
/ "launcher" / "AV-Live-Body" / "Resources" / "smpl_faces.bin")
|
||||
|
||||
# SMPL standard : 13776 triangles, 6890 vertices
|
||||
EXPECTED_FACES = 13776
|
||||
EXPECTED_VERTS = 6890
|
||||
|
||||
|
||||
def try_from_data_files() -> np.ndarray | None:
|
||||
"""Tente d'extraire depuis nlf_data_files.zip."""
|
||||
import zipfile
|
||||
zf = CACHE / "nlf_data_files.zip"
|
||||
if not zf.exists():
|
||||
return None
|
||||
with zipfile.ZipFile(zf) as z:
|
||||
for name in z.namelist():
|
||||
if "smpl" in name.lower() and name.endswith(".npy"):
|
||||
with z.open(name) as f:
|
||||
arr = np.load(f)
|
||||
if arr.shape == (EXPECTED_FACES, 3):
|
||||
return arr
|
||||
return None
|
||||
|
||||
|
||||
def try_from_torchscript() -> np.ndarray | None:
|
||||
"""Charge le checkpoint et cherche les faces SMPL."""
|
||||
try:
|
||||
import torch
|
||||
import torchvision # noqa: F401 - register torchvision::nms op for TorchScript
|
||||
ckpt = CACHE / "nlf_l_multi.torchscript"
|
||||
if not ckpt.exists():
|
||||
return None
|
||||
model = torch.jit.load(str(ckpt), map_location="cpu")
|
||||
for name, buf in model.named_buffers():
|
||||
if buf.shape == (EXPECTED_FACES, 3):
|
||||
print(f"Found faces in buffer '{name}'")
|
||||
return buf.numpy().astype(np.int32)
|
||||
for attr in dir(model):
|
||||
try:
|
||||
val = getattr(model, attr)
|
||||
if hasattr(val, 'shape') and val.shape == (EXPECTED_FACES, 3):
|
||||
print(f"Found faces in attr '{attr}'")
|
||||
return val.numpy().astype(np.int32) if hasattr(val, 'numpy') else np.array(val, dtype=np.int32)
|
||||
except Exception:
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f"TorchScript extraction failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def download_smpl_faces() -> np.ndarray:
|
||||
"""Telecharge les faces SMPL standard depuis un repo open-source.
|
||||
|
||||
Strategie multi-URL : essaie plusieurs sources, la premiere qui repond
|
||||
avec le bon shape (13776, 3) gagne. Aucun de ces fichiers ne contient
|
||||
de poids SMPL proprietaires, juste la topologie publique du mesh.
|
||||
"""
|
||||
import urllib.request
|
||||
import tempfile
|
||||
|
||||
candidates = [
|
||||
# HMR (akanazawa) ships the standard SMPL face topology as a public .npy
|
||||
# — verified (13776, 3) uint32, max index 6889.
|
||||
"https://github.com/akanazawa/hmr/raw/master/src/tf_smpl/smpl_faces.npy",
|
||||
]
|
||||
last_err = None
|
||||
for url in candidates:
|
||||
print(f"Downloading SMPL faces from {url}...")
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".npy", delete=False) as tmp:
|
||||
urllib.request.urlretrieve(url, tmp.name)
|
||||
faces = np.load(tmp.name)
|
||||
if faces.shape == (EXPECTED_FACES, 3):
|
||||
print(f" -> OK: {faces.shape} {faces.dtype}")
|
||||
return faces.astype(np.int32)
|
||||
print(f" -> wrong shape {faces.shape}, skip")
|
||||
except Exception as e:
|
||||
print(f" -> failed: {e}")
|
||||
last_err = e
|
||||
raise RuntimeError(f"All SMPL face download candidates failed: {last_err}")
|
||||
|
||||
|
||||
def main():
|
||||
faces = try_from_data_files()
|
||||
if faces is None:
|
||||
print("nlf_data_files.zip absent ou faces non trouvees, essai TorchScript...")
|
||||
faces = try_from_torchscript()
|
||||
if faces is None:
|
||||
print("TorchScript: faces non trouvees dans les buffers, download fallback...")
|
||||
faces = download_smpl_faces()
|
||||
|
||||
print(f"SMPL faces: {faces.shape} dtype={faces.dtype}")
|
||||
assert faces.shape == (EXPECTED_FACES, 3), f"shape attendu ({EXPECTED_FACES}, 3), got {faces.shape}"
|
||||
assert faces.max() < EXPECTED_VERTS, f"index max {faces.max()} >= {EXPECTED_VERTS}"
|
||||
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(OUT, "wb") as f:
|
||||
for tri in faces:
|
||||
for idx in tri:
|
||||
f.write(struct.pack("<I", int(idx)))
|
||||
size = OUT.stat().st_size
|
||||
expected_size = EXPECTED_FACES * 3 * 4
|
||||
print(f"Wrote {size} bytes to {OUT} (expected {expected_size})")
|
||||
assert size == expected_size
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Extrait la liste des 20908 triangles du modele SMPL-X NEUTRAL et
|
||||
les serialise en binaire little-endian (int32) pour consommation par
|
||||
l'app Swift RealityKit.
|
||||
|
||||
Necessite SMPLX_NEUTRAL.npz dans ~/.cache/av-live-multihmr/models/smplx/
|
||||
(inscription manuelle sur smpl-x.is.tue.mpg.de — licence MPII).
|
||||
"""
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
CACHE = Path.home() / ".cache" / "av-live-multihmr"
|
||||
SMPLX = CACHE / "models" / "smplx" / "SMPLX_NEUTRAL.npz"
|
||||
OUT = (Path(__file__).parent.parent.parent
|
||||
/ "launcher" / "AV-Live-Body" / "Resources" / "smplx_faces.bin")
|
||||
|
||||
EXPECTED_FACES = 20908
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not SMPLX.exists():
|
||||
print(f"SMPL-X manquant : {SMPLX}")
|
||||
print("Voir data_only_viz/scripts/setup_multihmr.sh pour la procedure.")
|
||||
return 1
|
||||
npz = np.load(SMPLX)
|
||||
faces = npz["f"]
|
||||
print(f"SMPL-X faces : {faces.shape} dtype={faces.dtype}")
|
||||
assert faces.shape == (EXPECTED_FACES, 3), (
|
||||
f"shape attendu ({EXPECTED_FACES}, 3), got {faces.shape}")
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(OUT, "wb") as f:
|
||||
for tri in faces:
|
||||
for idx in tri:
|
||||
f.write(struct.pack("<i", int(idx)))
|
||||
size = OUT.stat().st_size
|
||||
expected = EXPECTED_FACES * 3 * 4
|
||||
print(f"Wrote {size} bytes to {OUT} (expected {expected})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Extract j3d (32 SMPL-X joint anchors) from a recorded MP4 using the
|
||||
Multi-HMR CoreML backend, write per-frame per-person jsonl rows.
|
||||
|
||||
Usage:
|
||||
uv run python -m data_only_viz.scripts.extract_j3d_offline \
|
||||
--session sess03 \
|
||||
--video ~/.cache/av-live-action/raw/sess03.mp4 \
|
||||
--out ~/.cache/av-live-action/raw/sess03.jsonl
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from data_only_viz.action_head import EXPR_DIM, HANDS_KP_DIMS, HANDS_KP_TOTAL
|
||||
from data_only_viz.action_head_pub import (
|
||||
SMPLX_JOINT_ANCHOR_VERTS,
|
||||
SMPLX_UPPER_LIP_VERT,
|
||||
SMPLX_LOWER_LIP_VERT,
|
||||
)
|
||||
from data_only_viz.multihmr_coreml import MultiHMRCoreMLBackend
|
||||
|
||||
LOG = logging.getLogger("extract_j3d_offline")
|
||||
IMG_SIZE = 672
|
||||
DEFAULT_OUT_DIR = Path("~/.cache/av-live-action/raw").expanduser()
|
||||
|
||||
|
||||
def _default_K(size: int = IMG_SIZE) -> np.ndarray:
|
||||
"""Synthetic camera intrinsics, focal ~ image size, principal point centred."""
|
||||
f = float(size)
|
||||
cx = cy = f * 0.5
|
||||
return np.array(
|
||||
[[f, 0.0, cx], [0.0, f, cy], [0.0, 0.0, 1.0]],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
|
||||
def _frame_to_chw(frame_bgr: np.ndarray, size: int = IMG_SIZE) -> np.ndarray:
|
||||
"""BGR uint8 (H, W, 3) -> float32 CHW (3, size, size) in [0, 1]."""
|
||||
h, w = frame_bgr.shape[:2]
|
||||
side = min(h, w)
|
||||
y0 = (h - side) // 2
|
||||
x0 = (w - side) // 2
|
||||
crop = frame_bgr[y0:y0 + side, x0:x0 + side]
|
||||
resized = cv2.resize(crop, (size, size))
|
||||
rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
|
||||
return rgb.transpose(2, 0, 1) # CHW
|
||||
|
||||
|
||||
def _person_to_j3d32(
|
||||
person: dict,
|
||||
anchors: tuple[int, ...],
|
||||
) -> tuple[np.ndarray, np.ndarray, float] | None:
|
||||
"""Return (j3d32, expression, mouth_open) or None if v3d absent/too small."""
|
||||
v3d = person.get("v3d")
|
||||
if v3d is None:
|
||||
return None
|
||||
# CoreMLArray wraps numpy but lacks __array__; unwrap before asarray.
|
||||
if hasattr(v3d, "numpy") and not isinstance(v3d, np.ndarray):
|
||||
v3d = v3d.numpy()
|
||||
v3d_np = np.asarray(v3d, dtype=np.float32)
|
||||
if v3d_np.shape[0] < max(anchors) + 1:
|
||||
return None
|
||||
j3d32 = v3d_np[list(anchors)].astype(np.float32)
|
||||
# expression
|
||||
expr = person.get("expression")
|
||||
if expr is not None:
|
||||
if hasattr(expr, "numpy") and not isinstance(expr, np.ndarray):
|
||||
expr = expr.numpy()
|
||||
expr_np = np.asarray(expr, dtype=np.float32).flatten()
|
||||
else:
|
||||
expr_np = np.zeros(EXPR_DIM, dtype=np.float32)
|
||||
# mouth_open
|
||||
if v3d_np.shape[0] > max(SMPLX_UPPER_LIP_VERT, SMPLX_LOWER_LIP_VERT):
|
||||
mouth = float(np.linalg.norm(
|
||||
v3d_np[SMPLX_UPPER_LIP_VERT] - v3d_np[SMPLX_LOWER_LIP_VERT]
|
||||
))
|
||||
else:
|
||||
mouth = 0.0
|
||||
return j3d32, expr_np, mouth
|
||||
|
||||
|
||||
def extract(session: str, video: Path, out: Path,
|
||||
det_thresh: float = 0.3,
|
||||
mlpackage_path: Path | None = None,
|
||||
anchors: tuple[int, ...] = SMPLX_JOINT_ANCHOR_VERTS) -> int:
|
||||
"""Returns the number of (frame, person) rows written."""
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
cap = cv2.VideoCapture(str(video))
|
||||
if not cap.isOpened():
|
||||
raise RuntimeError(f"cannot open {video}")
|
||||
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
||||
backend = MultiHMRCoreMLBackend(mlpackage_path) if mlpackage_path \
|
||||
else MultiHMRCoreMLBackend()
|
||||
K = _default_K(IMG_SIZE)
|
||||
n_frames = 0
|
||||
n_rows = 0
|
||||
with out.open("w") as f:
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
break
|
||||
chw = _frame_to_chw(frame)
|
||||
try:
|
||||
persons = backend.infer(chw, K, det_thresh=det_thresh)
|
||||
except Exception:
|
||||
LOG.exception("infer failed at frame=%d", n_frames)
|
||||
n_frames += 1
|
||||
continue
|
||||
ts = n_frames / fps
|
||||
for i, person in enumerate(persons):
|
||||
result = _person_to_j3d32(person, anchors)
|
||||
if result is None:
|
||||
continue
|
||||
j3d32, expr_np, mouth = result
|
||||
f.write(json.dumps({
|
||||
"ts": ts,
|
||||
"session": session,
|
||||
"pid": int(person.get("pid", i)),
|
||||
"j3d": j3d32.tolist(),
|
||||
"expression": expr_np.tolist(),
|
||||
"mouth_open": mouth,
|
||||
"hands_kp": np.zeros(
|
||||
(HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32
|
||||
).tolist(),
|
||||
}) + "\n")
|
||||
n_rows += 1
|
||||
n_frames += 1
|
||||
if n_frames % 100 == 0:
|
||||
LOG.info("frame=%d rows=%d", n_frames, n_rows)
|
||||
cap.release()
|
||||
LOG.info("done: %d frames, %d rows -> %s", n_frames, n_rows, out)
|
||||
return n_rows
|
||||
|
||||
|
||||
def _cli() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--session", required=True)
|
||||
p.add_argument("--video", required=True, type=Path)
|
||||
p.add_argument("--out", type=Path)
|
||||
p.add_argument("--det-thresh", type=float, default=0.3)
|
||||
p.add_argument("--mlpackage", type=Path, default=None)
|
||||
args = p.parse_args()
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format="%(asctime)s [%(name)s] %(message)s")
|
||||
DEFAULT_OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
out = args.out or (DEFAULT_OUT_DIR / f"{args.session}.jsonl")
|
||||
extract(args.session, args.video, out,
|
||||
det_thresh=args.det_thresh, mlpackage_path=args.mlpackage)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_cli()
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Extract action-head v3 jsonl rows from a recorded MP4 using MediaPipe
|
||||
Holistic. Populates real hands_kp (42, 3) and mouth_open (face lips
|
||||
distance), unlike extract_j3d_offline.py (SMPL-X path) which writes zeros
|
||||
for hands_kp.
|
||||
|
||||
Output jsonl row format (matches dataset.py load_frames_jsonl) :
|
||||
|
||||
{
|
||||
"ts": float seconds,
|
||||
"session": str,
|
||||
"pid": int (always 0 — Holistic is single-person),
|
||||
"j3d": [[32, 3]] floats (body22 + 10 fingertips),
|
||||
"expression": [10] zeros (MediaPipe has no SMPL-X PCA),
|
||||
"mouth_open": float (lips inner distance),
|
||||
"hands_kp": [[42, 3]] floats (21 L + 21 R, zero-padded if absent),
|
||||
}
|
||||
|
||||
Usage :
|
||||
|
||||
uv run python -m data_only_viz.scripts.extract_mediapipe_offline \
|
||||
--session sess03 \
|
||||
--video ~/.cache/av-live-action/raw/sess03.mp4 \
|
||||
--out ~/.cache/av-live-action/raw/sess03_mp.jsonl
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from data_only_viz.action_head import (
|
||||
EXPR_DIM,
|
||||
HANDS_KP_DIMS,
|
||||
HANDS_KP_PER_HAND,
|
||||
HANDS_KP_TOTAL,
|
||||
J3D_BODY,
|
||||
J3D_FINGERS,
|
||||
J3D_FINGERS_PER_HAND,
|
||||
J3D_JOINTS,
|
||||
)
|
||||
from data_only_viz.action_head_pub import (
|
||||
MEDIAPIPE_HAND_FINGERTIPS,
|
||||
MEDIAPIPE_LIP_LOWER_INNER,
|
||||
MEDIAPIPE_LIP_UPPER_INNER,
|
||||
MEDIAPIPE_TO_22,
|
||||
)
|
||||
|
||||
LOG = logging.getLogger("extract_mediapipe_offline")
|
||||
DEFAULT_OUT_DIR = Path("~/.cache/av-live-action/raw").expanduser()
|
||||
|
||||
|
||||
def _build_landmarker():
|
||||
"""Build a MediaPipe HolisticLandmarker in VIDEO running mode."""
|
||||
from mediapipe.tasks.python import vision
|
||||
from mediapipe.tasks.python.core.base_options import BaseOptions
|
||||
from data_only_viz.holistic import _ensure_model
|
||||
model_path = _ensure_model()
|
||||
opts = vision.HolisticLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(model_path)),
|
||||
running_mode=vision.RunningMode.VIDEO,
|
||||
min_pose_detection_confidence=0.3,
|
||||
min_pose_landmarks_confidence=0.3,
|
||||
min_face_detection_confidence=0.3,
|
||||
min_face_landmarks_confidence=0.3,
|
||||
min_hand_landmarks_confidence=0.3,
|
||||
)
|
||||
return vision.HolisticLandmarker.create_from_options(opts)
|
||||
|
||||
|
||||
def _lmk_list_to_array(lmks) -> np.ndarray | None:
|
||||
"""Convert MediaPipe NormalizedLandmark / Landmark list to (N, 3) array."""
|
||||
if lmks is None:
|
||||
return None
|
||||
try:
|
||||
return np.asarray(
|
||||
[(lm.x, lm.y, getattr(lm, "z", 0.0)) for lm in lmks],
|
||||
dtype=np.float32,
|
||||
)
|
||||
except (AttributeError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def _build_j3d32(body3d_arr: np.ndarray | None,
|
||||
hands_kp42: np.ndarray) -> np.ndarray | None:
|
||||
"""Map MediaPipe body3d (33, 3) + hands_kp (42, 3) -> j3d (32, 3).
|
||||
|
||||
body22 indices via MEDIAPIPE_TO_22, fingertips from hands_kp idx
|
||||
MEDIAPIPE_HAND_FINGERTIPS (4, 8, 12, 16, 20) for each side.
|
||||
"""
|
||||
if body3d_arr is None or body3d_arr.shape[0] < 33:
|
||||
return None
|
||||
body22 = body3d_arr[list(MEDIAPIPE_TO_22)].astype(np.float32)
|
||||
tips = np.zeros((J3D_FINGERS, 3), dtype=np.float32)
|
||||
for side_idx in (0, 1):
|
||||
base = side_idx * HANDS_KP_PER_HAND
|
||||
for k, mp_tip in enumerate(MEDIAPIPE_HAND_FINGERTIPS):
|
||||
if base + mp_tip < hands_kp42.shape[0]:
|
||||
tips[side_idx * J3D_FINGERS_PER_HAND + k] = hands_kp42[base + mp_tip]
|
||||
return np.concatenate([body22, tips], axis=0)
|
||||
|
||||
|
||||
def _mouth_open(face_arr: np.ndarray | None) -> float:
|
||||
if face_arr is None or face_arr.shape[0] <= MEDIAPIPE_LIP_LOWER_INNER:
|
||||
return 0.0
|
||||
upper = face_arr[MEDIAPIPE_LIP_UPPER_INNER]
|
||||
lower = face_arr[MEDIAPIPE_LIP_LOWER_INNER]
|
||||
return float(np.linalg.norm(upper - lower))
|
||||
|
||||
|
||||
def _hands_kp42(left_arr: np.ndarray | None,
|
||||
right_arr: np.ndarray | None) -> np.ndarray:
|
||||
out = np.zeros((HANDS_KP_TOTAL, HANDS_KP_DIMS), dtype=np.float32)
|
||||
if left_arr is not None and left_arr.shape[0] >= HANDS_KP_PER_HAND:
|
||||
out[:HANDS_KP_PER_HAND] = left_arr[:HANDS_KP_PER_HAND]
|
||||
if right_arr is not None and right_arr.shape[0] >= HANDS_KP_PER_HAND:
|
||||
out[HANDS_KP_PER_HAND:] = right_arr[:HANDS_KP_PER_HAND]
|
||||
return out
|
||||
|
||||
|
||||
def extract(session: str, video: Path, out: Path) -> int:
|
||||
"""Run MediaPipe Holistic on every frame of video, write jsonl rows.
|
||||
|
||||
Returns the number of frames where at least body3d was detected
|
||||
(rows written). Frames with no person are silently skipped.
|
||||
"""
|
||||
import mediapipe as mp
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
cap = cv2.VideoCapture(str(video))
|
||||
if not cap.isOpened():
|
||||
raise RuntimeError(f"cannot open {video}")
|
||||
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
||||
landmarker = _build_landmarker()
|
||||
n_frames = 0
|
||||
n_rows = 0
|
||||
expr_zeros_list = np.zeros(EXPR_DIM, dtype=np.float32).tolist()
|
||||
try:
|
||||
with out.open("w") as f:
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
break
|
||||
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
|
||||
ts_ms = int(n_frames * 1000 / fps)
|
||||
try:
|
||||
res = landmarker.detect_for_video(mp_img, ts_ms)
|
||||
except Exception:
|
||||
LOG.exception("detect failed at frame=%d", n_frames)
|
||||
n_frames += 1
|
||||
continue
|
||||
body3d = _lmk_list_to_array(
|
||||
getattr(res, "pose_world_landmarks", None)
|
||||
)
|
||||
face_arr = _lmk_list_to_array(
|
||||
getattr(res, "face_landmarks", None)
|
||||
)
|
||||
left_arr = _lmk_list_to_array(
|
||||
getattr(res, "left_hand_landmarks", None)
|
||||
)
|
||||
right_arr = _lmk_list_to_array(
|
||||
getattr(res, "right_hand_landmarks", None)
|
||||
)
|
||||
hands_kp42 = _hands_kp42(left_arr, right_arr)
|
||||
j3d32 = _build_j3d32(body3d, hands_kp42)
|
||||
if j3d32 is None:
|
||||
n_frames += 1
|
||||
continue
|
||||
ts = n_frames / fps
|
||||
mouth = _mouth_open(face_arr)
|
||||
f.write(json.dumps({
|
||||
"ts": ts,
|
||||
"session": session,
|
||||
"pid": 0,
|
||||
"j3d": j3d32.tolist(),
|
||||
"expression": expr_zeros_list,
|
||||
"mouth_open": mouth,
|
||||
"hands_kp": hands_kp42.tolist(),
|
||||
}) + "\n")
|
||||
n_rows += 1
|
||||
n_frames += 1
|
||||
if n_frames % 100 == 0:
|
||||
LOG.info("frame=%d rows=%d", n_frames, n_rows)
|
||||
finally:
|
||||
cap.release()
|
||||
landmarker.close()
|
||||
LOG.info("done : %d frames, %d rows -> %s", n_frames, n_rows, out)
|
||||
return n_rows
|
||||
|
||||
|
||||
def _cli() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--session", required=True)
|
||||
p.add_argument("--video", required=True, type=Path)
|
||||
p.add_argument("--out", type=Path)
|
||||
args = p.parse_args()
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format="%(asctime)s [%(name)s] %(message)s")
|
||||
DEFAULT_OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
out = args.out or (DEFAULT_OUT_DIR / f"{args.session}_mp.jsonl")
|
||||
extract(args.session, args.video, out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_cli()
|
||||
@@ -0,0 +1,607 @@
|
||||
"""Multi-HMR inference server (TCP, coremltools backend).
|
||||
|
||||
Runs on a remote Mac (macm1 in the AV-Live cluster), loads the
|
||||
mlpackage via coremltools (Python 3.12), and serves frames over TCP.
|
||||
|
||||
Protocol (little-endian, persistent connection):
|
||||
|
||||
Request:
|
||||
[4 bytes uint32 payload_len]
|
||||
[4 bytes magic "REQ\\x01"]
|
||||
[1 byte uint8 format_id] # 1 = raw RGB uint8 HWC, 2 = JPEG
|
||||
[3 bytes padding]
|
||||
[variable image bytes] # IMG_BYTES if format=1, else JPEG bytes
|
||||
[9 float32 LE = 36 bytes K] # always last 36 bytes
|
||||
|
||||
Response:
|
||||
[4 bytes uint32 payload_len]
|
||||
[4 bytes magic "RSP\\x01"]
|
||||
[4 bytes int32 status] # 0 = OK, 1 = error
|
||||
[v3d: 4*10475*3 float32]
|
||||
[transl: 4*1*3 float32]
|
||||
[scores: 4 float32]
|
||||
[betas: 4*10 float32]
|
||||
[expr: 4*10 float32]
|
||||
|
||||
Connection handler runs a 3-thread pipeline: reader -> worker -> writer.
|
||||
While the worker predicts frame N, the reader has already buffered frame
|
||||
N+1 so the next predict can start the instant the previous response is
|
||||
handed to the writer. Queue depth is 2 to absorb network jitter.
|
||||
|
||||
Bench mode (--bench): synthetic frames against the loaded backend.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
LOG = logging.getLogger("multihmr_server")
|
||||
|
||||
IMG_SIZE = 672
|
||||
N_PERSONS_FIXED = 4
|
||||
N_VERTS = 10475
|
||||
|
||||
MAGIC_REQ = b"REQ\x01"
|
||||
MAGIC_RSP = b"RSP\x01"
|
||||
|
||||
FORMAT_RAW = 1
|
||||
FORMAT_JPEG = 2
|
||||
|
||||
IMG_BYTES = IMG_SIZE * IMG_SIZE * 3 # 1_354_752
|
||||
K_BYTES = 9 * 4 # 36
|
||||
REQ_HEADER = 4 + 1 + 3 # magic + fmt u8 + 3 pad
|
||||
|
||||
V3D_BYTES = N_PERSONS_FIXED * N_VERTS * 3 * 4
|
||||
TRANSL_BYTES = N_PERSONS_FIXED * 1 * 3 * 4
|
||||
SCORES_BYTES = N_PERSONS_FIXED * 4
|
||||
BETAS_BYTES = N_PERSONS_FIXED * 10 * 4
|
||||
EXPR_BYTES = N_PERSONS_FIXED * 10 * 4
|
||||
RSP_HEADER = 4 + 4
|
||||
RSP_PAYLOAD_LEN = (RSP_HEADER + V3D_BYTES + TRANSL_BYTES
|
||||
+ SCORES_BYTES + BETAS_BYTES + EXPR_BYTES)
|
||||
|
||||
|
||||
DEFAULT_MLPACKAGE = Path(
|
||||
os.environ.get("MULTIHMR_MLPACKAGE")
|
||||
or str(Path.home() / ".cache" / "av-live-multihmr"
|
||||
/ "multihmr_full_672_s.mlpackage"))
|
||||
|
||||
OUT_V3D = "var_2420"
|
||||
OUT_TRANSL = "var_2423"
|
||||
OUT_SCORES = "var_2436"
|
||||
OUT_BETAS = "var_2439"
|
||||
OUT_EXPR = "var_2442"
|
||||
OUT_JOINTS = "var_2445" # (4, 127, 3) SMPL-X joints incl fingers
|
||||
N_JOINTS = 127
|
||||
|
||||
|
||||
def recv_exact(sock: socket.socket, n: int) -> bytes:
|
||||
buf = bytearray(n)
|
||||
view = memoryview(buf)
|
||||
pos = 0
|
||||
while pos < n:
|
||||
got = sock.recv_into(view[pos:])
|
||||
if got == 0:
|
||||
raise ConnectionError("peer closed")
|
||||
pos += got
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def encode_response(v3d: np.ndarray, transl: np.ndarray,
|
||||
scores: np.ndarray, betas: np.ndarray,
|
||||
expr: np.ndarray, status: int = 0) -> bytes:
|
||||
parts = [
|
||||
struct.pack("<I", RSP_PAYLOAD_LEN),
|
||||
MAGIC_RSP,
|
||||
struct.pack("<i", status),
|
||||
np.ascontiguousarray(v3d, dtype=np.float32).tobytes(),
|
||||
np.ascontiguousarray(transl, dtype=np.float32).tobytes(),
|
||||
np.ascontiguousarray(scores, dtype=np.float32).tobytes(),
|
||||
np.ascontiguousarray(betas, dtype=np.float32).tobytes(),
|
||||
np.ascontiguousarray(expr, dtype=np.float32).tobytes(),
|
||||
]
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def decode_request(payload: bytes) -> tuple[np.ndarray, np.ndarray, float]:
|
||||
"""Decode a request payload (without the leading 4-byte length).
|
||||
|
||||
Returns (image_uint8_hwc, K_33_f32, decode_ms_overhead).
|
||||
"""
|
||||
if len(payload) < REQ_HEADER + K_BYTES:
|
||||
raise ValueError(f"req payload too short: {len(payload)}")
|
||||
magic = payload[:4]
|
||||
if magic != MAGIC_REQ:
|
||||
raise ValueError(f"bad magic {magic!r}")
|
||||
fmt = payload[4]
|
||||
# payload[5:8] reserved.
|
||||
img_end = len(payload) - K_BYTES
|
||||
img_bytes = payload[REQ_HEADER:img_end]
|
||||
K = np.frombuffer(payload, dtype="<f4", count=9,
|
||||
offset=img_end).reshape(3, 3).astype(np.float32)
|
||||
t0 = time.monotonic()
|
||||
if fmt == FORMAT_RAW:
|
||||
if len(img_bytes) != IMG_BYTES:
|
||||
raise ValueError(
|
||||
f"raw img bytes {len(img_bytes)} != {IMG_BYTES}")
|
||||
img = np.frombuffer(img_bytes, dtype=np.uint8).reshape(
|
||||
IMG_SIZE, IMG_SIZE, 3)
|
||||
elif fmt == FORMAT_JPEG:
|
||||
import cv2
|
||||
arr = np.frombuffer(img_bytes, dtype=np.uint8)
|
||||
bgr = cv2.imdecode(arr, cv2.IMREAD_COLOR)
|
||||
if bgr is None:
|
||||
raise ValueError("cv2.imdecode failed")
|
||||
if bgr.shape[:2] != (IMG_SIZE, IMG_SIZE):
|
||||
bgr = cv2.resize(bgr, (IMG_SIZE, IMG_SIZE))
|
||||
img = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
|
||||
else:
|
||||
raise ValueError(f"unknown format_id {fmt}")
|
||||
decode_ms = (time.monotonic() - t0) * 1e3
|
||||
return img, K, decode_ms
|
||||
|
||||
|
||||
ML_DTYPE_FLOAT16 = 65552
|
||||
ML_DTYPE_FLOAT32 = 65568
|
||||
ML_DTYPE_DOUBLE = 65600
|
||||
|
||||
|
||||
def _np_to_mlarray(arr: np.ndarray, MLMultiArray):
|
||||
"""Create a contiguous float32 MLMultiArray from a numpy array."""
|
||||
import ctypes
|
||||
arr = np.ascontiguousarray(arr, dtype=np.float32)
|
||||
shape = [int(s) for s in arr.shape]
|
||||
ml = MLMultiArray.alloc().initWithShape_dataType_error_(
|
||||
shape, ML_DTYPE_FLOAT32, None)
|
||||
if ml is None:
|
||||
raise RuntimeError("MLMultiArray alloc failed")
|
||||
ptr = ml.dataPointer()
|
||||
addr = int(ptr) if isinstance(ptr, int) else ctypes.cast(
|
||||
ptr, ctypes.c_void_p).value
|
||||
if addr is None:
|
||||
raise RuntimeError("MLMultiArray dataPointer null")
|
||||
ctypes.memmove(addr, arr.ctypes.data, arr.nbytes)
|
||||
return ml
|
||||
|
||||
|
||||
def _mlarray_to_np(ml) -> np.ndarray:
|
||||
"""Copy an MLMultiArray (FLOAT16/32/64) to numpy float32."""
|
||||
import ctypes
|
||||
shape = tuple(int(s) for s in ml.shape())
|
||||
dtype_id = int(ml.dataType())
|
||||
count = 1
|
||||
for s in shape:
|
||||
count *= s
|
||||
ptr = ml.dataPointer()
|
||||
addr = int(ptr) if isinstance(ptr, int) else ctypes.cast(
|
||||
ptr, ctypes.c_void_p).value
|
||||
if addr is None:
|
||||
raise RuntimeError("MLMultiArray dataPointer null")
|
||||
if dtype_id == ML_DTYPE_FLOAT16:
|
||||
raw = (ctypes.c_uint16 * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).view(np.float16).astype(np.float32)
|
||||
elif dtype_id == ML_DTYPE_FLOAT32:
|
||||
raw = (ctypes.c_float * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).copy()
|
||||
elif dtype_id == ML_DTYPE_DOUBLE:
|
||||
raw = (ctypes.c_double * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).astype(np.float32)
|
||||
else:
|
||||
raise RuntimeError(f"unsupported MLMultiArray dtype {dtype_id}")
|
||||
return arr.reshape(shape)
|
||||
|
||||
|
||||
class CoreMLModel:
|
||||
"""pyobjc-direct CoreML wrapper. Drops the ~30 ms coremltools.MLModel.predict
|
||||
overhead by using CoreML.framework directly (MLDictionaryFeatureProvider
|
||||
+ MLMultiArray ctypes memcpy). Fallback to coremltools if pyobjc missing,
|
||||
via MULTIHMR_SERVER_BACKEND=coremltools env."""
|
||||
|
||||
def __init__(self, mlpackage_path: Path) -> None:
|
||||
self.path = Path(mlpackage_path)
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"mlpackage missing: {self.path}")
|
||||
backend = os.environ.get(
|
||||
"MULTIHMR_SERVER_BACKEND", "pyobjc").strip().lower()
|
||||
cu_env = os.environ.get(
|
||||
"COREML_COMPUTE_UNITS", "cpu_and_gpu").strip().lower()
|
||||
if backend == "pyobjc":
|
||||
self._use_pyobjc = True
|
||||
self._init_pyobjc(cu_env)
|
||||
else:
|
||||
self._use_pyobjc = False
|
||||
self._init_coremltools(cu_env)
|
||||
|
||||
def _init_pyobjc(self, cu_env: str) -> None:
|
||||
import objc
|
||||
from Foundation import NSURL
|
||||
ns: dict = {}
|
||||
objc.loadBundle("CoreML", ns,
|
||||
"/System/Library/Frameworks/CoreML.framework")
|
||||
cu_map = {"cpu_only": 0, "cpu_and_gpu": 1, "all": 2,
|
||||
"cpu_and_ne": 3}
|
||||
cu = cu_map.get(cu_env, 1)
|
||||
MLModel = ns["MLModel"]
|
||||
MLModelConfiguration = ns["MLModelConfiguration"]
|
||||
cfg = MLModelConfiguration.alloc().init()
|
||||
try:
|
||||
cfg.setComputeUnits_(cu)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
url = NSURL.fileURLWithPath_(str(self.path))
|
||||
compiled_url = MLModel.compileModelAtURL_error_(url, None)
|
||||
if compiled_url is None:
|
||||
raise RuntimeError(f"compileModelAtURL failed for {self.path}")
|
||||
model = MLModel.modelWithContentsOfURL_configuration_error_(
|
||||
compiled_url, cfg, None)
|
||||
if model is None:
|
||||
raise RuntimeError(f"MLModel load failed for {compiled_url}")
|
||||
self._model = model
|
||||
self._ns = ns
|
||||
LOG.info("loading mlpackage %s via pyobjc (computeUnit=%s)",
|
||||
self.path.name, cu_env)
|
||||
|
||||
def _init_coremltools(self, cu_env: str) -> None:
|
||||
import coremltools as ct
|
||||
from coremltools.models import MLModel as CTMLModel
|
||||
cu_map = {
|
||||
"cpu_only": ct.ComputeUnit.CPU_ONLY,
|
||||
"cpu_and_gpu": ct.ComputeUnit.CPU_AND_GPU,
|
||||
"all": ct.ComputeUnit.ALL,
|
||||
"cpu_and_ne": ct.ComputeUnit.CPU_AND_NE,
|
||||
}
|
||||
cu = cu_map.get(cu_env, ct.ComputeUnit.CPU_AND_GPU)
|
||||
LOG.info("loading mlpackage %s via coremltools (computeUnit=%s)",
|
||||
self.path.name, cu_env)
|
||||
self.model = CTMLModel(str(self.path), compute_units=cu)
|
||||
|
||||
def predict(self, image_uint8_hwc: np.ndarray, K_33: np.ndarray
|
||||
) -> dict[str, np.ndarray]:
|
||||
img_chw = image_uint8_hwc.transpose(2, 0, 1).astype(np.float32) / 255.0
|
||||
img4 = img_chw[np.newaxis, ...]
|
||||
K = K_33.astype(np.float32)
|
||||
if K.ndim == 2:
|
||||
K = K[np.newaxis, ...]
|
||||
if self._use_pyobjc:
|
||||
return self._predict_pyobjc(img4, K)
|
||||
return self.model.predict({"image": img4, "cam_K": K})
|
||||
|
||||
def _predict_pyobjc(self, image_4d: np.ndarray, K_33: np.ndarray
|
||||
) -> dict[str, np.ndarray]:
|
||||
ns = self._ns
|
||||
MLMultiArray = ns["MLMultiArray"]
|
||||
MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"]
|
||||
MLFeatureValue = ns["MLFeatureValue"]
|
||||
img_ml = _np_to_mlarray(image_4d, MLMultiArray)
|
||||
k_ml = _np_to_mlarray(K_33, MLMultiArray)
|
||||
feats = {
|
||||
"image": MLFeatureValue.featureValueWithMultiArray_(img_ml),
|
||||
"cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml),
|
||||
}
|
||||
provider = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
if provider is None:
|
||||
raise RuntimeError("MLDictionaryFeatureProvider alloc failed")
|
||||
out = self._model.predictionFromFeatures_error_(provider, None)
|
||||
if out is None:
|
||||
raise RuntimeError("MLModel predict failed")
|
||||
names = [str(n) for n in out.featureNames()]
|
||||
result: dict[str, np.ndarray] = {}
|
||||
for n in names:
|
||||
fv = out.featureValueForName_(n)
|
||||
if fv is None:
|
||||
continue
|
||||
ml = fv.multiArrayValue()
|
||||
if ml is None:
|
||||
continue
|
||||
result[n] = _mlarray_to_np(ml)
|
||||
return result
|
||||
|
||||
|
||||
def _zero_outputs() -> tuple[np.ndarray, ...]:
|
||||
return (
|
||||
np.zeros((N_PERSONS_FIXED, N_VERTS, 3), dtype=np.float32),
|
||||
np.zeros((N_PERSONS_FIXED, 1, 3), dtype=np.float32),
|
||||
np.zeros((N_PERSONS_FIXED,), dtype=np.float32),
|
||||
np.zeros((N_PERSONS_FIXED, 10), dtype=np.float32),
|
||||
np.zeros((N_PERSONS_FIXED, 10), dtype=np.float32),
|
||||
)
|
||||
|
||||
|
||||
def _extract_outputs(raw: dict[str, np.ndarray]
|
||||
) -> tuple[np.ndarray, ...]:
|
||||
v3d = np.asarray(raw[OUT_V3D], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED, N_VERTS, 3)
|
||||
transl = np.asarray(raw[OUT_TRANSL], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED, 1, 3)
|
||||
scores = np.asarray(raw[OUT_SCORES], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED)
|
||||
betas = np.asarray(raw[OUT_BETAS], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED, 10)
|
||||
expr = np.asarray(raw[OUT_EXPR], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED, 10)
|
||||
return v3d, transl, scores, betas, expr
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self, model: CoreMLModel, host: str, port: int) -> None:
|
||||
self.model = model
|
||||
self.host = host
|
||||
self.port = port
|
||||
self._stop = threading.Event()
|
||||
self._sock: socket.socket | None = None
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
if self._sock is not None:
|
||||
try:
|
||||
self._sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def serve(self) -> None:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
sock.bind((self.host, self.port))
|
||||
sock.listen(4)
|
||||
sock.settimeout(1.0)
|
||||
self._sock = sock
|
||||
LOG.info("listening %s:%d", self.host, self.port)
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
conn, addr = sock.accept()
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
break
|
||||
LOG.info("client connected %s", addr)
|
||||
try:
|
||||
self._handle_pipelined(conn)
|
||||
except (ConnectionError, BrokenPipeError, OSError) as e:
|
||||
LOG.info("client disconnected: %s", e)
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
LOG.info("server stopped")
|
||||
|
||||
# -- pipelined per-connection handler -----------------------------
|
||||
|
||||
def _handle_pipelined(self, conn: socket.socket) -> None:
|
||||
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
conn_stop = threading.Event()
|
||||
|
||||
# raw requests in, encoded responses out.
|
||||
req_q: queue.Queue[bytes] = queue.Queue(maxsize=2)
|
||||
rsp_q: queue.Queue[bytes] = queue.Queue(maxsize=2)
|
||||
|
||||
# stats
|
||||
served = {"n": 0, "t0": time.monotonic(),
|
||||
"sum_decode": 0.0, "sum_pred": 0.0,
|
||||
"sum_encode": 0.0}
|
||||
|
||||
def reader() -> None:
|
||||
try:
|
||||
while not conn_stop.is_set() and not self._stop.is_set():
|
||||
len_buf = recv_exact(conn, 4)
|
||||
payload_len = struct.unpack("<I", len_buf)[0]
|
||||
if payload_len > 8 * 1024 * 1024:
|
||||
raise ValueError(f"reqlen too big {payload_len}")
|
||||
payload = recv_exact(conn, payload_len)
|
||||
req_q.put(payload)
|
||||
except (ConnectionError, BrokenPipeError, OSError) as e:
|
||||
LOG.info("reader exit: %s", e)
|
||||
finally:
|
||||
conn_stop.set()
|
||||
# poison-pill the worker
|
||||
try:
|
||||
req_q.put_nowait(b"")
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
def worker() -> None:
|
||||
try:
|
||||
while not conn_stop.is_set() and not self._stop.is_set():
|
||||
try:
|
||||
payload = req_q.get(timeout=0.5)
|
||||
except queue.Empty:
|
||||
continue
|
||||
if payload == b"":
|
||||
break
|
||||
try:
|
||||
img, K, decode_ms = decode_request(payload)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("decode failed: %s", e)
|
||||
v3d, transl, scores, betas, expr = _zero_outputs()
|
||||
rsp_q.put(encode_response(
|
||||
v3d, transl, scores, betas, expr, status=1))
|
||||
continue
|
||||
t_pred = time.monotonic()
|
||||
try:
|
||||
raw = self.model.predict(img, K)
|
||||
v3d, transl, scores, betas, expr = _extract_outputs(
|
||||
raw)
|
||||
status = 0
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("predict failed: %s", e)
|
||||
v3d, transl, scores, betas, expr = _zero_outputs()
|
||||
status = 1
|
||||
t_pred_end = time.monotonic()
|
||||
t_enc = time.monotonic()
|
||||
rsp = encode_response(
|
||||
v3d, transl, scores, betas, expr, status=status)
|
||||
t_enc_end = time.monotonic()
|
||||
pred_ms = (t_pred_end - t_pred) * 1e3
|
||||
encode_ms = (t_enc_end - t_enc) * 1e3
|
||||
served["n"] += 1
|
||||
served["sum_decode"] += decode_ms
|
||||
served["sum_pred"] += pred_ms
|
||||
served["sum_encode"] += encode_ms
|
||||
rsp_q.put(rsp)
|
||||
now = time.monotonic()
|
||||
if served["n"] % 30 == 0:
|
||||
dt = now - served["t0"]
|
||||
fps = served["n"] / max(1e-6, dt)
|
||||
LOG.info(
|
||||
"served %d frames at %.1f fps over %.1f s "
|
||||
"(decode=%.1fms pred=%.1fms encode=%.1fms)",
|
||||
served["n"], fps, dt,
|
||||
served["sum_decode"] / served["n"],
|
||||
served["sum_pred"] / served["n"],
|
||||
served["sum_encode"] / served["n"])
|
||||
finally:
|
||||
conn_stop.set()
|
||||
try:
|
||||
rsp_q.put_nowait(b"")
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
def writer() -> None:
|
||||
try:
|
||||
while not conn_stop.is_set() and not self._stop.is_set():
|
||||
try:
|
||||
rsp = rsp_q.get(timeout=0.5)
|
||||
except queue.Empty:
|
||||
continue
|
||||
if rsp == b"":
|
||||
break
|
||||
conn.sendall(rsp)
|
||||
except (ConnectionError, BrokenPipeError, OSError) as e:
|
||||
LOG.info("writer exit: %s", e)
|
||||
finally:
|
||||
conn_stop.set()
|
||||
|
||||
t_r = threading.Thread(target=reader, name="srv-reader", daemon=True)
|
||||
t_w = threading.Thread(target=worker, name="srv-worker", daemon=True)
|
||||
t_x = threading.Thread(target=writer, name="srv-writer", daemon=True)
|
||||
t_r.start()
|
||||
t_w.start()
|
||||
t_x.start()
|
||||
t_r.join()
|
||||
t_w.join()
|
||||
t_x.join()
|
||||
dt = time.monotonic() - served["t0"]
|
||||
if served["n"] > 0:
|
||||
LOG.info("connection closed: served %d frames at %.1f fps "
|
||||
"over %.1f s", served["n"],
|
||||
served["n"] / max(1e-6, dt), dt)
|
||||
|
||||
|
||||
def run_bench(model: CoreMLModel, n: int = 30) -> None:
|
||||
"""Local synthetic bench (no socket)."""
|
||||
rng = np.random.default_rng(0)
|
||||
K = np.array([[672.0, 0.0, 336.0],
|
||||
[0.0, 672.0, 336.0],
|
||||
[0.0, 0.0, 1.0]], dtype=np.float32)
|
||||
img0 = rng.integers(0, 256, (IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)
|
||||
model.predict(img0, K)
|
||||
times = []
|
||||
for _ in range(n):
|
||||
img = rng.integers(0, 256, (IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)
|
||||
t0 = time.monotonic()
|
||||
model.predict(img, K)
|
||||
times.append((time.monotonic() - t0) * 1e3)
|
||||
ts = sorted(times)
|
||||
median = ts[len(ts) // 2]
|
||||
mean = sum(times) / len(times)
|
||||
p90 = ts[int(len(ts) * 0.9)]
|
||||
LOG.info("bench n=%d median=%.1fms mean=%.1fms p90=%.1fms (%.1f fps)",
|
||||
n, median, mean, p90, 1000.0 / median)
|
||||
|
||||
|
||||
def run_bench_async(model: CoreMLModel, host: str, port: int,
|
||||
n: int = 60) -> None:
|
||||
"""End-to-end pipeline bench via real socket loopback."""
|
||||
import threading
|
||||
server = Server(model, host, port)
|
||||
th = threading.Thread(target=server.serve, daemon=True)
|
||||
th.start()
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
from data_only_viz.multihmr_remote import MultiHMRRemoteBackend
|
||||
except ImportError:
|
||||
# When the server runs standalone, the client package may not be
|
||||
# importable. Skip with a friendly message.
|
||||
LOG.warning("data_only_viz package not importable on this host, "
|
||||
"skipping --bench-async")
|
||||
server.stop()
|
||||
return
|
||||
os.environ.setdefault("MULTIHMR_REMOTE_HOST", host)
|
||||
os.environ.setdefault("MULTIHMR_REMOTE_PORT", str(port))
|
||||
be = MultiHMRRemoteBackend(host=host, port=port)
|
||||
rng = np.random.default_rng(0)
|
||||
K = np.array([[672.0, 0.0, 336.0],
|
||||
[0.0, 672.0, 336.0],
|
||||
[0.0, 0.0, 1.0]], dtype=np.float32)
|
||||
t0 = time.monotonic()
|
||||
got = 0
|
||||
for _ in range(n):
|
||||
img = (rng.random((3, IMG_SIZE, IMG_SIZE), dtype=np.float32))
|
||||
out = be.infer(img, K)
|
||||
if out is not None:
|
||||
got += 1
|
||||
time.sleep(0.01)
|
||||
dt = time.monotonic() - t0
|
||||
LOG.info("bench-async submitted=%d got=%d in %.2fs (%.1f fps submit)",
|
||||
n, got, dt, n / max(1e-6, dt))
|
||||
be.close()
|
||||
server.stop()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(description="Multi-HMR TCP server")
|
||||
ap.add_argument("--mlpackage", type=Path, default=DEFAULT_MLPACKAGE)
|
||||
ap.add_argument("--host", default=os.environ.get(
|
||||
"MULTIHMR_SERVER_HOST", "0.0.0.0"))
|
||||
ap.add_argument("--port", type=int, default=int(os.environ.get(
|
||||
"MULTIHMR_SERVER_PORT", "57140")))
|
||||
ap.add_argument("--bench", action="store_true",
|
||||
help="local synthetic bench, no socket")
|
||||
ap.add_argument("--bench-async", action="store_true",
|
||||
help="loopback pipeline bench through real sockets")
|
||||
ap.add_argument("--bench-n", type=int, default=30)
|
||||
ap.add_argument("--log-level", default="INFO")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
logging.basicConfig(
|
||||
level=args.log_level.upper(),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
|
||||
model = CoreMLModel(args.mlpackage)
|
||||
|
||||
if args.bench:
|
||||
run_bench(model, n=args.bench_n)
|
||||
return 0
|
||||
if args.bench_async:
|
||||
run_bench_async(model, "127.0.0.1", args.port, n=args.bench_n)
|
||||
return 0
|
||||
|
||||
server = Server(model, args.host, args.port)
|
||||
|
||||
def _sigint(*_a):
|
||||
LOG.info("SIGINT received, stopping")
|
||||
server.stop()
|
||||
|
||||
signal.signal(signal.SIGINT, _sigint)
|
||||
signal.signal(signal.SIGTERM, _sigint)
|
||||
try:
|
||||
server.serve()
|
||||
except KeyboardInterrupt:
|
||||
server.stop()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Task 2 — Validate apply_topk(K=4) as drop-in replacement for
|
||||
apply_threshold in Multi-HMR head.
|
||||
|
||||
Compares v3d output between threshold-based (original) and topk-based
|
||||
(patched) Multi-HMR on the same input. Pass criterion: for the same
|
||||
detections (when K >= n_threshold_detected), v3d cosine similarity > 0.99.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
CACHE = Path.home() / ".cache" / "av-live-multihmr"
|
||||
CKPT = CACHE / "checkpoints" / "multiHMR_672_S.pt"
|
||||
MULTIHMR_REPO = CACHE / "multi-hmr"
|
||||
|
||||
sys.path.insert(0, str(MULTIHMR_REPO))
|
||||
for mod in ("pyrender", "pyvista", "anny"):
|
||||
sys.modules.setdefault(mod, types.ModuleType(mod))
|
||||
|
||||
DEVICE = "mps" if torch.backends.mps.is_available() else "cpu"
|
||||
IMG_SIZE = 672
|
||||
|
||||
|
||||
def apply_topk(K, _scores):
|
||||
"""Drop-in pour apply_threshold. _scores shape (B, H, W, C).
|
||||
Renvoie 4-tuple LongTensor (batch_idx, h_idx, w_idx, c_idx) chacun
|
||||
de longueur B*K (au lieu de variable). K candidate top-scoring
|
||||
tokens par image.
|
||||
"""
|
||||
if isinstance(K, list):
|
||||
K = K[0]
|
||||
B, H, W, C = _scores.shape
|
||||
flat = _scores.reshape(B, -1)
|
||||
_, idx_flat = torch.topk(flat, k=K, dim=1)
|
||||
wc = W * C
|
||||
idx_b = (torch.arange(B, device=_scores.device)
|
||||
.unsqueeze(1).expand(-1, K).reshape(-1))
|
||||
idx_flat_flat = idx_flat.reshape(-1)
|
||||
idx_h = idx_flat_flat // wc
|
||||
idx_w = (idx_flat_flat // C) % W
|
||||
idx_c = idx_flat_flat % C
|
||||
return (idx_b.long(), idx_h.long(), idx_w.long(), idx_c.long())
|
||||
|
||||
|
||||
prev = os.getcwd()
|
||||
try:
|
||||
os.chdir(MULTIHMR_REPO)
|
||||
from model import Model
|
||||
import model as model_mod
|
||||
torch_dev = torch.device(DEVICE)
|
||||
ckpt = torch.load(str(CKPT), map_location=torch_dev, weights_only=False)
|
||||
kw = {k: v for k, v in vars(ckpt["args"]).items()}
|
||||
kw["type"] = ckpt["args"].train_return_type
|
||||
kw["img_size"] = ckpt["args"].img_size[0]
|
||||
model = Model(**kw).to(torch_dev)
|
||||
model.load_state_dict(ckpt["model_state_dict"], strict=False)
|
||||
model.eval()
|
||||
finally:
|
||||
os.chdir(prev)
|
||||
|
||||
focal = float(IMG_SIZE)
|
||||
K_mat = torch.tensor([[[focal, 0.0, IMG_SIZE / 2.0],
|
||||
[0.0, focal, IMG_SIZE / 2.0],
|
||||
[0.0, 0.0, 1.0]]], device=DEVICE)
|
||||
|
||||
# Use a real test image (multi-hmr example or webcam capture)
|
||||
import cv2
|
||||
img_path = "/Users/electron/.cache/av-live-multihmr/multi-hmr/example_data/4446582661_b188f82f3c_c.jpg"
|
||||
img = cv2.imread(img_path)
|
||||
h, w = img.shape[:2]
|
||||
side = min(h, w)
|
||||
y0 = (h - side) // 2; x0 = (w - side) // 2
|
||||
img = cv2.resize(img[y0:y0+side, x0:x0+side], (IMG_SIZE, IMG_SIZE))
|
||||
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
||||
x = (torch.from_numpy(img_rgb).permute(2, 0, 1).float() / 255.0
|
||||
).unsqueeze(0).to(DEVICE)
|
||||
print(f"loaded image {img_path}")
|
||||
|
||||
# --- Pass 1 : original apply_threshold a tres bas seuil ---
|
||||
print("==> Pass 1 : apply_threshold(0.15)")
|
||||
with torch.no_grad():
|
||||
humans_orig = model(x, is_training=False, nms_kernel_size=5,
|
||||
det_thresh=0.15, K=K_mat)
|
||||
print(f" detected: {len(humans_orig)}")
|
||||
for i, h in enumerate(humans_orig[:4]):
|
||||
sc = h.get("scores", 0.0)
|
||||
if hasattr(sc, "item"):
|
||||
sc = sc.item()
|
||||
print(f" [{i}] score={sc:.3f} v3d.shape={tuple(h['v3d'].shape)}")
|
||||
|
||||
# --- Pass 2 : monkey-patch apply_threshold avec apply_topk(K=4) ---
|
||||
print("\n==> Pass 2 : apply_topk(K=4)")
|
||||
original_apply_threshold = model_mod.apply_threshold
|
||||
|
||||
def topk_wrapper(det_thresh, _scores):
|
||||
return apply_topk(4, _scores)
|
||||
|
||||
model_mod.apply_threshold = topk_wrapper
|
||||
|
||||
with torch.no_grad():
|
||||
humans_topk = model(x, is_training=False, nms_kernel_size=5,
|
||||
det_thresh=0.15, K=K_mat)
|
||||
print(f" detected: {len(humans_topk)}")
|
||||
for i, h in enumerate(humans_topk[:4]):
|
||||
sc = h.get("scores", 0.0)
|
||||
if hasattr(sc, "item"):
|
||||
sc = sc.item()
|
||||
print(f" [{i}] score={sc:.3f} v3d.shape={tuple(h['v3d'].shape)}")
|
||||
|
||||
# Restore
|
||||
model_mod.apply_threshold = original_apply_threshold
|
||||
|
||||
# --- Comparison ---
|
||||
print("\n==> Comparison")
|
||||
if len(humans_orig) == 0 or len(humans_topk) == 0:
|
||||
print(" NO DETECTIONS in one path — adjust threshold lower")
|
||||
sys.exit(0)
|
||||
|
||||
# Match by score (highest first in both)
|
||||
o = sorted(humans_orig, key=lambda h: -(
|
||||
h.get("scores", 0).item() if hasattr(h.get("scores", 0), "item")
|
||||
else h.get("scores", 0)))[:min(len(humans_orig), 4)]
|
||||
t = sorted(humans_topk, key=lambda h: -(
|
||||
h.get("scores", 0).item() if hasattr(h.get("scores", 0), "item")
|
||||
else h.get("scores", 0)))[:len(o)]
|
||||
|
||||
for i, (ho, ht) in enumerate(zip(o, t)):
|
||||
vo = ho["v3d"].detach().cpu().numpy().flatten()
|
||||
vt = ht["v3d"].detach().cpu().numpy().flatten()
|
||||
dot = float(np.dot(vo, vt))
|
||||
nv = float(np.linalg.norm(vo) * np.linalg.norm(vt) + 1e-9)
|
||||
cos = dot / nv
|
||||
mae = float(np.mean(np.abs(vo - vt)))
|
||||
sco = (ho.get("scores", 0).item()
|
||||
if hasattr(ho.get("scores", 0), "item") else ho.get("scores", 0))
|
||||
sct = (ht.get("scores", 0).item()
|
||||
if hasattr(ht.get("scores", 0), "item") else ht.get("scores", 0))
|
||||
print(f" [{i}] cosine={cos:.6f} mae={mae*1000:.3f}mm "
|
||||
f"score_orig={sco:.4f} score_topk={sct:.4f}")
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Quantize Multi-HMR mlpackage to INT8 (weight-only) for M5 speedup.
|
||||
|
||||
Run in the Python 3.12 conversion venv (coremltools cannot run on 3.14):
|
||||
|
||||
/tmp/coreml312/.venv/bin/python \
|
||||
data_only_viz/scripts/quantize_multihmr_int8.py
|
||||
|
||||
Produces `multihmr_full_672_s_int8.mlpackage` next to the FP32 file.
|
||||
Bench after with `scripts/coreml_full_probe.py` or just load with
|
||||
`MultiHMRCoreMLBackend(path=...new path...)`.
|
||||
|
||||
Strategy:
|
||||
- Linear 8-bit weight palettization (per-tensor symmetric). Activations
|
||||
stay FP16 — that's the "weight-only quant" path, lowest accuracy
|
||||
hit and what CoreML's GPU runtime accelerates best.
|
||||
- Skip the SMPL-X decoder branch ops that are sensitive to numeric
|
||||
drift (skipped by name pattern below — adjust if v3d shows mesh
|
||||
artefacts after quantization).
|
||||
|
||||
Validation:
|
||||
- After producing the int8 mlpackage, run the live worker briefly
|
||||
with COREML_MLPACKAGE pointing to the new file and visually check
|
||||
the mesh. If v3d shows tearing on extreme poses, retry with
|
||||
`granularity="per_channel"` instead of `per_tensor`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import coremltools as ct
|
||||
from coremltools.optimize.coreml import (
|
||||
linear_quantize_weights,
|
||||
OptimizationConfig,
|
||||
OpLinearQuantizerConfig,
|
||||
)
|
||||
except ImportError as e:
|
||||
print(f"coremltools missing in this venv: {e}", file=sys.stderr)
|
||||
print("Run from the Python 3.12 conversion venv (coremltools "
|
||||
"is not available on 3.14).", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
SRC = Path.home() / ".cache" / "av-live-multihmr" / \
|
||||
"multihmr_full_672_s.mlpackage"
|
||||
DST = Path.home() / ".cache" / "av-live-multihmr" / \
|
||||
"multihmr_full_672_s_int8.mlpackage"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not SRC.exists():
|
||||
print(f"source mlpackage missing: {SRC}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"loading FP32 model from {SRC}")
|
||||
model = ct.models.MLModel(str(SRC))
|
||||
|
||||
# Per-tensor symmetric int8 weight quant. Per-tensor keeps the
|
||||
# quantized model small and GPU-friendly; per-channel is a safer
|
||||
# fallback if mesh quality degrades.
|
||||
op_cfg = OpLinearQuantizerConfig(
|
||||
mode="linear_symmetric",
|
||||
dtype="int8",
|
||||
granularity="per_tensor",
|
||||
)
|
||||
cfg = OptimizationConfig(global_config=op_cfg)
|
||||
print("running linear_quantize_weights (per_tensor int8)...")
|
||||
quant = linear_quantize_weights(model, config=cfg)
|
||||
print(f"saving quantized model to {DST}")
|
||||
quant.save(str(DST))
|
||||
print("done. Test with:")
|
||||
print(f" COREML_MLPACKAGE={DST} \\\n"
|
||||
f" MULTIHMR_BACKEND=coreml \\\n"
|
||||
f" uv run --project data_only_viz \\\n"
|
||||
f" python -m data_only_viz.main --multi-hmr "
|
||||
f"--motion-gate 0")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
# Setup Multi-HMR : clone repo, telecharge checkpoint base (88 MB), prepare
|
||||
# le dossier SMPL-X. SMPLX_NEUTRAL.npz necessite une inscription manuelle
|
||||
# sur https://smpl-x.is.tue.mpg.de/ (academic license MPII).
|
||||
set -euo pipefail
|
||||
CACHE="$HOME/.cache/av-live-multihmr"
|
||||
mkdir -p "$CACHE/checkpoints" "$CACHE/models/smplx"
|
||||
|
||||
if [ ! -d "$CACHE/multi-hmr" ]; then
|
||||
echo "==> Clone Multi-HMR"
|
||||
git clone --depth=1 https://github.com/naver/multi-hmr.git "$CACHE/multi-hmr"
|
||||
fi
|
||||
|
||||
CKPT="$CACHE/checkpoints/multiHMR_672_S.pt"
|
||||
if [ ! -f "$CKPT" ]; then
|
||||
echo "==> Telechargement checkpoint multiHMR_672_S (ViT-S)"
|
||||
# Source primaire : Naver Labs Europe ; fallback : HuggingFace mirror
|
||||
if ! curl -fL --progress-bar \
|
||||
"https://download.europe.naverlabs.com/ComputerVision/MultiHMR/multiHMR_672_S.pt" \
|
||||
-o "$CKPT"; then
|
||||
echo "==> Fallback HuggingFace"
|
||||
curl -fL --progress-bar \
|
||||
"https://huggingface.co/naver/multiHMR_672_S/resolve/main/multiHMR_672_S.pt" \
|
||||
-o "$CKPT"
|
||||
fi
|
||||
fi
|
||||
|
||||
SMPLX="$CACHE/models/smplx/SMPLX_NEUTRAL.npz"
|
||||
if [ ! -f "$SMPLX" ]; then
|
||||
echo ""
|
||||
echo "MANUEL REQUIS :"
|
||||
echo " 1. Inscrivez-vous sur https://smpl-x.is.tue.mpg.de/"
|
||||
echo " 2. Telechargez 'SMPL-X v1.1 (NPZ + PKL)'"
|
||||
echo " 3. Extraire SMPLX_NEUTRAL.npz vers : $SMPLX"
|
||||
fi
|
||||
|
||||
# Mean params SMPL (init parameters) — necessaire au constructeur Model
|
||||
MEAN="$CACHE/models/smpl_mean_params.npz"
|
||||
if [ ! -f "$MEAN" ]; then
|
||||
echo "==> Telechargement smpl_mean_params (1.3 KB)"
|
||||
curl -fL --progress-bar \
|
||||
"https://openmmlab-share.oss-cn-hangzhou.aliyuncs.com/mmhuman3d/models/smpl_mean_params.npz?versionId=CAEQHhiBgICN6M3V6xciIDU1MzUzNjZjZGNiOTQ3OWJiZTJmNThiZmY4NmMxMTM4" \
|
||||
-o "$MEAN"
|
||||
fi
|
||||
|
||||
# Symlink relatif 'models' dans le repo Multi-HMR pour que SMPLX_DIR='models'
|
||||
# (utils/constants.py) trouve les .npz.
|
||||
if [ ! -e "$CACHE/multi-hmr/models" ]; then
|
||||
ln -sfn ../models "$CACHE/multi-hmr/models"
|
||||
fi
|
||||
|
||||
# CoreML conversion patches : remplace les torch.einsum dans utils/camera.py
|
||||
# par des ops element-wise (broadcast-friendly). Sans ca, ct.convert echoue
|
||||
# avec "Invalid target shape in reshape op ([1, N, 3] to [K*N, 3, 1])"
|
||||
# quand batch K detections != 1. Idempotent.
|
||||
CAM="$CACHE/multi-hmr/utils/camera.py"
|
||||
if [ -f "$CAM" ] && ! grep -q "_apply_intrinsics_componentwise" "$CAM"; then
|
||||
echo "==> Patch utils/camera.py (einsum -> componentwise)"
|
||||
python3 - "$CAM" <<'PYEOF'
|
||||
import sys, pathlib
|
||||
p = pathlib.Path(sys.argv[1])
|
||||
src = p.read_text()
|
||||
helper = '''
|
||||
def _apply_intrinsics_componentwise(K, y):
|
||||
"""CoreML-friendly: out[b,k,i] = sum_j K[b,i,j] * y[b,k,j]
|
||||
Replaces torch.einsum('bij,bkj->bki', K, y) with pure broadcast ops.
|
||||
"""
|
||||
K00 = K[:, 0:1, 0:1]; K01 = K[:, 0:1, 1:2]; K02 = K[:, 0:1, 2:3]
|
||||
K10 = K[:, 1:2, 0:1]; K11 = K[:, 1:2, 1:2]; K12 = K[:, 1:2, 2:3]
|
||||
K20 = K[:, 2:3, 0:1]; K21 = K[:, 2:3, 1:2]; K22 = K[:, 2:3, 2:3]
|
||||
y0 = y[:, :, 0:1]; y1 = y[:, :, 1:2]; y2 = y[:, :, 2:3]
|
||||
out0 = K00 * y0 + K01 * y1 + K02 * y2
|
||||
out1 = K10 * y0 + K11 * y1 + K12 * y2
|
||||
out2 = K20 * y0 + K21 * y1 + K22 * y2
|
||||
return torch.cat([out0, out1, out2], dim=-1)
|
||||
|
||||
|
||||
'''
|
||||
src = src.replace(
|
||||
"def perspective_projection(x, K):",
|
||||
helper + "def perspective_projection(x, K):",
|
||||
)
|
||||
src = src.replace(
|
||||
"y = torch.einsum('bij,bkj->bki', K, y) # (bs, N, 3)",
|
||||
"y = _apply_intrinsics_componentwise(K, y)",
|
||||
)
|
||||
src = src.replace(
|
||||
"points = torch.einsum('bij,bkj->bki', torch.inverse(K), points)",
|
||||
"points = _apply_intrinsics_componentwise(torch.inverse(K), points)",
|
||||
)
|
||||
p.write_text(src)
|
||||
print(" camera.py patched")
|
||||
PYEOF
|
||||
fi
|
||||
|
||||
# CoreML conversion patch : smplx/lbs.py landmarks einsum (mеme bug broadcast)
|
||||
# Patch best-effort sur tous les venvs presents (data_only_viz + /tmp/coreml312).
|
||||
for VENV in \
|
||||
"$(dirname "$(dirname "$(readlink -f "$0")")")/.venv" \
|
||||
"/tmp/coreml312"; do
|
||||
LBS="$VENV/lib/python3.14/site-packages/smplx/lbs.py"
|
||||
[ -f "$LBS" ] || LBS="$VENV/lib/python3.12/site-packages/smplx/lbs.py"
|
||||
if [ -f "$LBS" ] && grep -q "torch.einsum('blfi,blf->bli'" "$LBS"; then
|
||||
echo "==> Patch $LBS (landmarks einsum)"
|
||||
python3 - "$LBS" <<'PYEOF'
|
||||
import sys, pathlib
|
||||
p = pathlib.Path(sys.argv[1])
|
||||
s = p.read_text()
|
||||
s = s.replace(
|
||||
"landmarks = torch.einsum('blfi,blf->bli', [lmk_vertices, lmk_bary_coords])\n return landmarks",
|
||||
"# CoreML-friendly: replace einsum('blfi,blf->bli', ...) with broadcast+sum\n landmarks = (lmk_vertices * lmk_bary_coords.unsqueeze(-1)).sum(dim=2)\n return landmarks",
|
||||
)
|
||||
p.write_text(s)
|
||||
print(" smplx/lbs.py patched")
|
||||
PYEOF
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Setup OK. Cache : $CACHE"
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
CACHE="$HOME/.cache/av-live-nlf"
|
||||
mkdir -p "$CACHE"
|
||||
|
||||
# NLF Large multi-person TorchScript (470 MB)
|
||||
CKPT="$CACHE/nlf_l_multi.torchscript"
|
||||
if [ ! -f "$CKPT" ]; then
|
||||
echo "Downloading NLF-L multi-person (470 MB)..."
|
||||
curl -fL --progress-bar \
|
||||
"https://github.com/isarandi/nlf/releases/download/v0.3.2/nlf_l_multi_0.3.2.torchscript" \
|
||||
-o "$CKPT"
|
||||
fi
|
||||
|
||||
# NLF Small multi-person TorchScript (284 MB) — fallback plus rapide
|
||||
CKPT_S="$CACHE/nlf_s_multi.torchscript"
|
||||
if [ ! -f "$CKPT_S" ]; then
|
||||
echo "Downloading NLF-S multi-person (284 MB)..."
|
||||
curl -fL --progress-bar \
|
||||
"https://github.com/isarandi/nlf/releases/download/v0.2.2/nlf_s_multi_0.2.2.torchscript" \
|
||||
-o "$CKPT_S"
|
||||
fi
|
||||
|
||||
echo "Setup OK. Cache : $CACHE"
|
||||
ls -lh "$CACHE"/*.torchscript
|
||||
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
# Push the Multi-HMR mlpackage + server to macm1 (M1 Max, 32-core GPU)
|
||||
# and launch the inference server in the background.
|
||||
#
|
||||
# Prereqs on macm1 :
|
||||
# * passwordless ssh (Tailscale alias 'macm1' or LAN)
|
||||
# * uv installed
|
||||
# * Python 3.12 available via uv (uv pulls it)
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/setup_remote_macm1.sh
|
||||
# MACM1_HOST=clems@192.168.0.175 ./scripts/setup_remote_macm1.sh
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${MACM1_HOST:-macm1}"
|
||||
PORT="${MULTIHMR_SERVER_PORT:-57140}"
|
||||
MLPACKAGE_LOCAL="${MLPACKAGE_LOCAL:-$HOME/.cache/av-live-multihmr/multihmr_full_672_s.mlpackage}"
|
||||
REMOTE_TMP="/tmp/av-live-multihmr"
|
||||
REMOTE_VENV="/tmp/av-live-multihmr/venv"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
echo "==> Target host : $HOST"
|
||||
echo "==> mlpackage : $MLPACKAGE_LOCAL"
|
||||
|
||||
if [ ! -d "$MLPACKAGE_LOCAL" ]; then
|
||||
echo "ERROR: mlpackage missing at $MLPACKAGE_LOCAL" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Creating remote tmp dir"
|
||||
ssh "$HOST" "mkdir -p $REMOTE_TMP"
|
||||
|
||||
echo "==> rsync mlpackage (~70 MB, may take a moment first time)"
|
||||
rsync -a --delete \
|
||||
"$MLPACKAGE_LOCAL/" \
|
||||
"$HOST:$REMOTE_TMP/multihmr_full_672_s.mlpackage/"
|
||||
|
||||
echo "==> rsync multihmr_server.py"
|
||||
rsync -a "$SCRIPT_DIR/multihmr_server.py" \
|
||||
"$HOST:$REMOTE_TMP/multihmr_server.py"
|
||||
|
||||
echo "==> Provision Python 3.12 venv with uv (idempotent)"
|
||||
ssh "$HOST" "bash -lc 'set -e
|
||||
if [ ! -x $REMOTE_VENV/bin/python ]; then
|
||||
uv venv --python 3.12 $REMOTE_VENV --quiet
|
||||
fi
|
||||
uv pip install --python $REMOTE_VENV/bin/python --quiet \
|
||||
coremltools numpy opencv-python-headless \
|
||||
pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CoreML
|
||||
'"
|
||||
|
||||
echo "==> Killing any stale server on :$PORT"
|
||||
ssh "$HOST" "bash -lc 'pkill -f multihmr_server.py 2>/dev/null || true; sleep 0.3'"
|
||||
|
||||
echo "==> Launching server (background)"
|
||||
ssh "$HOST" "bash -lc 'cd $REMOTE_TMP && \
|
||||
MULTIHMR_SERVER_PORT=$PORT \
|
||||
nohup $REMOTE_VENV/bin/python multihmr_server.py \
|
||||
--mlpackage $REMOTE_TMP/multihmr_full_672_s.mlpackage \
|
||||
--port $PORT \
|
||||
>> $REMOTE_TMP/server.log 2>&1 &
|
||||
echo \$! > $REMOTE_TMP/server.pid
|
||||
disown || true'"
|
||||
|
||||
echo "==> Waiting for server to be ready"
|
||||
REMOTE_ADDR=$(ssh "$HOST" 'echo $SSH_CONNECTION' | awk '{print $3}')
|
||||
# Fallback to host alias if SSH_CONNECTION trick fails.
|
||||
if [ -z "${REMOTE_ADDR:-}" ]; then REMOTE_ADDR="$HOST"; fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
if ssh "$HOST" "bash -lc 'nc -z 127.0.0.1 $PORT 2>/dev/null'"; then
|
||||
echo "==> Server up on $HOST:$PORT (probed via localhost on host)"
|
||||
echo "==> Reachable from this Mac at $REMOTE_ADDR:$PORT"
|
||||
ssh "$HOST" "tail -n 20 $REMOTE_TMP/server.log" || true
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "ERROR: server did not come up within 30s. Last log lines:" >&2
|
||||
ssh "$HOST" "tail -n 60 $REMOTE_TMP/server.log" || true
|
||||
exit 1
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
# Setup SMPLer-X-S inference pipeline pour ARM Mac (M5).
|
||||
#
|
||||
# Stratégie : SMPLer-X vendorise sa propre copie de mmpose dans
|
||||
# transformer_utils/mmpose/, donc on n'a besoin que de :
|
||||
# - mmcv-lite (pour `from mmcv import Config`)
|
||||
# - smplx (pour decode SMPL-X params)
|
||||
# - YOLO/Ultralytics (déjà dans extras pose) pour body detection
|
||||
#
|
||||
# On évite mmdet/mmcv-full/mmpose pip installs qui plantent sur ARM
|
||||
# Python 3.14.
|
||||
set -euo pipefail
|
||||
|
||||
CACHE="$HOME/.cache/av-live-smplerx"
|
||||
# Repo source = git submodule electron-rare/SMPLer-X (fork S-Lab 1.0)
|
||||
# initialise au niveau du repo AV-Live racine.
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
REPO="$REPO_ROOT/third_party/SMPLer-X"
|
||||
|
||||
mkdir -p "$CACHE/checkpoints" "$CACHE/models/smplx"
|
||||
|
||||
if [ ! -d "$REPO/main" ]; then
|
||||
echo "==> Init git submodule SMPLer-X"
|
||||
( cd "$REPO_ROOT" && git submodule update --init third_party/SMPLer-X )
|
||||
fi
|
||||
|
||||
CKPT="$CACHE/checkpoints/smpler_x_s32.pth.tar"
|
||||
if [ ! -f "$CKPT" ]; then
|
||||
echo "==> Telechargement SMPLer-X-S checkpoint (ViT-S, ~150 MB)"
|
||||
curl -fL --progress-bar \
|
||||
"https://huggingface.co/caizhongang/SMPLer-X/resolve/main/smpler_x_s32.pth.tar" \
|
||||
-o "$CKPT" \
|
||||
|| { echo "ERREUR download checkpoint"; exit 1; }
|
||||
fi
|
||||
|
||||
SMPLX="$CACHE/models/smplx/SMPLX_NEUTRAL.npz"
|
||||
SHARED="$HOME/.cache/av-live-multihmr/models/smplx/SMPLX_NEUTRAL.npz"
|
||||
if [ ! -f "$SMPLX" ]; then
|
||||
if [ -f "$SHARED" ]; then
|
||||
echo "==> Symlink SMPLX_NEUTRAL.npz depuis cache Multi-HMR"
|
||||
ln -sf "$SHARED" "$SMPLX"
|
||||
else
|
||||
echo ""
|
||||
echo "MANUEL REQUIS :"
|
||||
echo " 1. Inscrivez-vous sur https://smpl-x.is.tue.mpg.de/"
|
||||
echo " 2. Telechargez 'SMPL-X v1.1 (NPZ + PKL)'"
|
||||
echo " 3. Extraire SMPLX_NEUTRAL.npz vers : $SMPLX"
|
||||
echo " OU lance d'abord scripts/setup_multihmr.sh et symlink."
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Setup OK. Cache : $CACHE"
|
||||
echo "Files :"
|
||||
ls -lh "$CACHE/checkpoints/" 2>/dev/null
|
||||
echo ""
|
||||
echo "Next: ajouter 'mmcv-lite' au pyproject.toml et 'uv sync --extra smplerx'"
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
# Train action-head on MacStudio M3 Ultra (Tailscale 100.116.92.12).
|
||||
#
|
||||
# SSH direct grosmac→studio is broken since reboot 2026-05-12 ;
|
||||
# we route via electron-server bastion (cf. CLAUDE.md root).
|
||||
#
|
||||
# Usage:
|
||||
# ./train_on_studio.sh # uses defaults
|
||||
# ./train_on_studio.sh --epochs 80 --lr 5e-4
|
||||
#
|
||||
# Local layout :
|
||||
# ~/.cache/av-live-action/dataset/dataset.jsonl (input)
|
||||
# ~/.cache/av-live-action/checkpoints/ (output, after rsync back)
|
||||
#
|
||||
# Remote layout :
|
||||
# studio:~/av-live-action/repo/ (rsynced code subset)
|
||||
# studio:~/av-live-action/dataset/ (rsynced dataset)
|
||||
# studio:~/av-live-action/checkpoints/ (training output)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BASTION_USER_HOST="${BASTION_USER_HOST:-electron-server}"
|
||||
STUDIO_USER_HOST="${STUDIO_USER_HOST:-clems@100.116.92.12}"
|
||||
STUDIO_USER="${STUDIO_USER:-clems}"
|
||||
STUDIO_UV="${STUDIO_UV:-/opt/homebrew/bin/uv}"
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
|
||||
LOCAL_CACHE="$HOME/.cache/av-live-action"
|
||||
LOCAL_DATASET="$LOCAL_CACHE/dataset"
|
||||
LOCAL_CKPT="$LOCAL_CACHE/checkpoints"
|
||||
|
||||
REMOTE_ROOT="/Users/${STUDIO_USER}/av-live-action"
|
||||
REMOTE_REPO="$REMOTE_ROOT/repo"
|
||||
REMOTE_DATASET="$REMOTE_ROOT/dataset"
|
||||
REMOTE_CKPT="$REMOTE_ROOT/checkpoints"
|
||||
|
||||
DATASET_FILE="${DATASET_FILE:-$LOCAL_DATASET/dataset.jsonl}"
|
||||
CKPT_NAME="${CKPT_NAME:-action_head.pt}"
|
||||
|
||||
# Quote train args defensively before forwarding through bastion ssh +
|
||||
# studio ssh (each layer reparses). Reject single quotes — they break
|
||||
# the single-quoted payload in bastion_ssh and could allow injection.
|
||||
for a in "$@"; do
|
||||
if [[ "$a" == *"'"* ]]; then
|
||||
printf '[train_on_studio] forbidden single quote in arg: %s\n' "$a" >&2
|
||||
exit 3
|
||||
fi
|
||||
done
|
||||
TRAIN_ARGS="$(printf '%q ' "$@")"
|
||||
|
||||
log() { printf '[train_on_studio] %s\n' "$*" >&2; }
|
||||
|
||||
[[ -f "$DATASET_FILE" ]] || { log "missing dataset: $DATASET_FILE"; exit 2; }
|
||||
mkdir -p "$LOCAL_CKPT"
|
||||
|
||||
bastion_ssh() {
|
||||
# The remote shell on the bastion must receive the studio command
|
||||
# as a single argument, otherwise `;` and `&&` are parsed
|
||||
# bastion-side instead of studio-side.
|
||||
# All paths in commands MUST be absolute (no $HOME, no ~) since
|
||||
# we use single-quotes for the studio-side payload.
|
||||
ssh -o ConnectTimeout=5 "$BASTION_USER_HOST" \
|
||||
"ssh -o ConnectTimeout=5 $STUDIO_USER_HOST '$*'"
|
||||
}
|
||||
|
||||
bastion_rsync() {
|
||||
# rsync via ssh ProxyJump through bastion. Direct grosmac->studio
|
||||
# known_hosts entry may be stale (SSH direct broken since reboot
|
||||
# 2026-05-12). accept-new lets us add the key on first use.
|
||||
local src="$1" dst="$2"
|
||||
rsync -avz --delete \
|
||||
-e "ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -A -J $BASTION_USER_HOST" \
|
||||
"$src" "$dst"
|
||||
}
|
||||
|
||||
log "== Studio reachability =="
|
||||
bastion_ssh "echo studio OK ; $STUDIO_UV --version"
|
||||
|
||||
log "== Push code subset =="
|
||||
bastion_ssh "mkdir -p $REMOTE_REPO/data_only_viz $REMOTE_DATASET $REMOTE_CKPT"
|
||||
rsync -avz --delete \
|
||||
--exclude='.venv/' --exclude='__pycache__/' --exclude='.pytest_cache/' \
|
||||
--exclude='.ruff_cache/' --exclude='*.pyc' --exclude='.DS_Store' \
|
||||
--exclude='web/' --exclude='shaders/' \
|
||||
-e "ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -A -J $BASTION_USER_HOST" \
|
||||
"$REPO_ROOT/data_only_viz/" \
|
||||
"$STUDIO_USER_HOST:av-live-action/repo/data_only_viz/"
|
||||
|
||||
log "== Push dataset =="
|
||||
bastion_rsync "$LOCAL_DATASET/" "$STUDIO_USER_HOST:av-live-action/dataset/"
|
||||
|
||||
log "== Remote uv sync =="
|
||||
# multihmr extra pulls torch (action-head training needs torch but no pyobjc).
|
||||
# We piggy-back on the multihmr extras since torch is the main thing we need.
|
||||
bastion_ssh "cd $REMOTE_REPO && $STUDIO_UV sync --no-progress --project data_only_viz --extra multihmr"
|
||||
|
||||
log "== Remote train (MPS) =="
|
||||
# cwd must be the PARENT of data_only_viz/ so the package is importable as
|
||||
# top-level. uv resolves the env via --project data_only_viz.
|
||||
bastion_ssh "cd $REMOTE_REPO && \
|
||||
$STUDIO_UV run --project data_only_viz python -m data_only_viz.training.train_action_head \
|
||||
--dataset $REMOTE_DATASET/$(basename "$DATASET_FILE") \
|
||||
--ckpt-out $REMOTE_CKPT/$CKPT_NAME \
|
||||
--device mps \
|
||||
$TRAIN_ARGS"
|
||||
|
||||
log "== Pull checkpoint back =="
|
||||
bastion_rsync "$STUDIO_USER_HOST:av-live-action/checkpoints/" "$LOCAL_CKPT/"
|
||||
|
||||
log "== Done. Checkpoint: $LOCAL_CKPT/$CKPT_NAME =="
|
||||
ls -la "$LOCAL_CKPT/$CKPT_NAME"
|
||||
@@ -0,0 +1,570 @@
|
||||
// scene.metal — fond reactif aux flux data_feeds + skeleton overlay.
|
||||
//
|
||||
// 9 modes visuels style demoscene 2023+ (raymarching SDF, fractales,
|
||||
// parallax, palette IQ). Reactivite open-data via SceneUniforms.
|
||||
// 0 storm fbm tissu palette Kp/Bz + lightning flash
|
||||
// 1 tunnel raymarched tube avec anneaux translucents (wind, RMS)
|
||||
// 2 plasma volumetric noise palette IQ (Kp, social_rate)
|
||||
// 3 kaleido fractal KIFS 6-fold rotation 3D (flare, time)
|
||||
// 4 voronoi cellular 3D crystal sphere (lightning, RMS)
|
||||
// 5 metaballs raymarched SDF metaballs colored shading (RMS, beat)
|
||||
// 6 starfield galaxy spiral parallax + god rays (wind, kp)
|
||||
// 7 bars 3D pillars en perspective avec depth fog (RMS+social)
|
||||
// 8 hands3d raymarching mandelbox-like + hands camera control
|
||||
|
||||
#include <metal_stdlib>
|
||||
using namespace metal;
|
||||
|
||||
struct SceneUniforms {
|
||||
float time;
|
||||
float rms;
|
||||
float kp_norm;
|
||||
float netz_dev;
|
||||
float lightning_flash;
|
||||
float flare;
|
||||
float wind_norm;
|
||||
float bz_norm;
|
||||
float social_rate;
|
||||
float pose_alive;
|
||||
float pose_count;
|
||||
float width;
|
||||
float height;
|
||||
float viz_mode;
|
||||
float hand_l_x;
|
||||
float hand_l_y;
|
||||
float hand_r_x;
|
||||
float hand_r_y;
|
||||
float _pad0;
|
||||
float _pad1;
|
||||
};
|
||||
|
||||
struct VsOut {
|
||||
float4 position [[position]];
|
||||
float2 uv;
|
||||
};
|
||||
|
||||
vertex VsOut bg_vertex(uint vid [[vertex_id]]) {
|
||||
float2 p = float2((vid << 1) & 2, vid & 2);
|
||||
VsOut o;
|
||||
o.position = float4(p * 2.0 - 1.0, 0.0, 1.0);
|
||||
o.uv = p;
|
||||
return o;
|
||||
}
|
||||
|
||||
// ===== Helpers ====================================================
|
||||
|
||||
float hash21(float2 p) {
|
||||
p = fract(p * float2(123.34, 456.21));
|
||||
p += dot(p, p + 45.32);
|
||||
return fract(p.x * p.y);
|
||||
}
|
||||
float hash31(float3 p) {
|
||||
p = fract(p * 0.1031);
|
||||
p += dot(p, p.yzx + 33.33);
|
||||
return fract((p.x + p.y) * p.z);
|
||||
}
|
||||
float noise2(float2 p) {
|
||||
float2 i = floor(p);
|
||||
float2 f = fract(p);
|
||||
float a = hash21(i);
|
||||
float b = hash21(i + float2(1, 0));
|
||||
float c = hash21(i + float2(0, 1));
|
||||
float d = hash21(i + float2(1, 1));
|
||||
float2 u = f * f * (3.0 - 2.0 * f);
|
||||
return mix(mix(a, b, u.x), mix(c, d, u.x), u.y);
|
||||
}
|
||||
float fbm(float2 p) {
|
||||
float v = 0.0, a = 0.5;
|
||||
for (int i = 0; i < 5; ++i) { v += a * noise2(p); p *= 2.13; a *= 0.5; }
|
||||
return v;
|
||||
}
|
||||
|
||||
// Palette cosinusoidale IQ : 3 tons doux
|
||||
float3 palIQ(float t, float3 a, float3 b, float3 c, float3 d) {
|
||||
return a + b * cos(6.28318 * (c * t + d));
|
||||
}
|
||||
|
||||
// Rotations
|
||||
float3 rotY(float3 p, float a) {
|
||||
float c = cos(a), s = sin(a);
|
||||
return float3(c * p.x + s * p.z, p.y, -s * p.x + c * p.z);
|
||||
}
|
||||
float3 rotX(float3 p, float a) {
|
||||
float c = cos(a), s = sin(a);
|
||||
return float3(p.x, c * p.y - s * p.z, s * p.y + c * p.z);
|
||||
}
|
||||
float3 rotZ(float3 p, float a) {
|
||||
float c = cos(a), s = sin(a);
|
||||
return float3(c * p.x - s * p.y, s * p.x + c * p.y, p.z);
|
||||
}
|
||||
|
||||
// SDF primitives
|
||||
float sdSphere(float3 p, float r) { return length(p) - r; }
|
||||
float sdBox(float3 p, float3 b) {
|
||||
float3 q = abs(p) - b;
|
||||
return length(max(q, 0.0)) + min(max(q.x, max(q.y, q.z)), 0.0);
|
||||
}
|
||||
float sdTorus(float3 p, float2 t) {
|
||||
float2 q = float2(length(p.xz) - t.x, p.y);
|
||||
return length(q) - t.y;
|
||||
}
|
||||
float smin(float a, float b, float k) {
|
||||
float h = clamp(0.5 + 0.5 * (b - a) / k, 0.0, 1.0);
|
||||
return mix(b, a, h) - k * h * (1.0 - h);
|
||||
}
|
||||
|
||||
float vignette(float2 p) {
|
||||
return 1.0 - smoothstep(0.6, 1.5, length(p));
|
||||
}
|
||||
|
||||
// ===== Modes =======================================================
|
||||
|
||||
// ---- 0 storm : tissu fbm reactif + bloom-fake ----
|
||||
float3 mode_storm(float2 p, constant SceneUniforms& U) {
|
||||
float storm = saturate(U.kp_norm * 1.0 + max(-U.bz_norm, 0.0) * 0.5);
|
||||
float speed = 0.08 + U.wind_norm * 1.5;
|
||||
float zoom = 1.8 - U.rms * 1.2;
|
||||
float n = fbm(p * zoom + float2(U.time * speed, U.time * speed * 0.7));
|
||||
n = pow(n, 1.2 - U.rms * 0.5);
|
||||
float netz = sin(U.time * 50.0 + U.netz_dev * 800.0) * 0.06;
|
||||
float3 base = palIQ(n + storm * 0.5,
|
||||
float3(0.10, 0.05, 0.20),
|
||||
float3(0.40, 0.30, 0.55),
|
||||
float3(1.0, 1.0, 1.0),
|
||||
float3(0.0, 0.33, 0.67));
|
||||
float bloom = smoothstep(0.7, 1.0, n);
|
||||
return base * (n * 1.4 + 0.3) + netz + U.rms * 1.2
|
||||
+ bloom * 0.7
|
||||
+ float3(1.0, 0.55, 0.1) * U.flare * 1.4
|
||||
+ float3(U.lightning_flash * 0.7);
|
||||
}
|
||||
|
||||
// ---- 1 tunnel : raymarched cylindrical tube avec anneaux ----
|
||||
float3 mode_tunnel(float2 p, constant SceneUniforms& U) {
|
||||
// Pseudo-3D tunnel: r/theta + scrolling z
|
||||
float r = length(p);
|
||||
float a = atan2(p.y, p.x);
|
||||
float z = U.time * (1.5 + U.wind_norm * 8.0 + U.rms * 4.0);
|
||||
// Repeat depth
|
||||
float d = 1.0 / max(r, 0.04) + z;
|
||||
// anneaux + spirale
|
||||
float ring = sin(d * 4.0) * 0.5 + 0.5;
|
||||
float spiral = sin(a * (8.0 + U.kp_norm * 6.0) + d * 0.6);
|
||||
float v = ring * (0.4 + 0.6 * spiral);
|
||||
// Iris central
|
||||
v *= smoothstep(0.05, 0.20, r);
|
||||
float3 base = palIQ(d * 0.06 + U.time * 0.05,
|
||||
float3(0.15, 0.05, 0.35),
|
||||
float3(0.55, 0.25, 0.35),
|
||||
float3(1.0, 1.0, 0.8),
|
||||
float3(0.0, 0.10, 0.20));
|
||||
float3 col = base * v;
|
||||
// Chromatic aberration fake : sample displaced
|
||||
float chrom = U.lightning_flash * 0.15;
|
||||
col.r *= 1.0 + chrom; col.b *= 1.0 - chrom;
|
||||
return col + float3(1.0, 0.7, 0.3) * U.flare * 1.5
|
||||
+ float3(U.lightning_flash * 0.6);
|
||||
}
|
||||
|
||||
// ---- 2 plasma : volumetric noise palette IQ ----
|
||||
float3 mode_plasma(float2 p, constant SceneUniforms& U) {
|
||||
float t = U.time * (0.5 + U.rms * 1.5);
|
||||
// 3 octaves de sin/cos en composition
|
||||
float v = sin(p.x * 4.0 + t)
|
||||
+ sin(p.y * 5.0 - t * 1.2)
|
||||
+ sin((p.x + p.y) * 3.5 + t * 0.7)
|
||||
+ sin(length(p) * (8.0 + U.kp_norm * 4.0) - t * 1.8);
|
||||
v = v * 0.25 + 0.5;
|
||||
// Fake volumetric "depth" : repeat layers
|
||||
float layer2 = sin(p.x * 2.0 - t * 0.5) * sin(p.y * 2.5 + t * 0.7);
|
||||
v = mix(v, v * 0.5 + 0.5 * (layer2 + 1.0) * 0.5, 0.35);
|
||||
float3 col = palIQ(v,
|
||||
float3(0.5),
|
||||
float3(0.5),
|
||||
float3(1.0, 1.0, 1.0),
|
||||
float3(0.0, 0.33, 0.67));
|
||||
col *= 0.8 + U.kp_norm * 0.7 + U.social_rate * 0.5;
|
||||
return col + float3(0.6, 0.3, 1.0) * U.lightning_flash * 0.5;
|
||||
}
|
||||
|
||||
// ---- 3 kaleido : KIFS fractal 6-fold avec rot 3D fake ----
|
||||
float3 mode_kaleido(float2 p, constant SceneUniforms& U) {
|
||||
float ang = U.time * 0.15 + U.flare * 2.0;
|
||||
float c = cos(ang), s = sin(ang);
|
||||
p = float2(c * p.x - s * p.y, s * p.x + c * p.y);
|
||||
float r = length(p);
|
||||
float a = atan2(p.y, p.x);
|
||||
float seg = 6.28318 / 6.0;
|
||||
a = abs(fmod(a + seg * 0.5, seg) - seg * 0.5);
|
||||
float2 q = float2(cos(a), sin(a)) * r;
|
||||
// Iteration KIFS-like
|
||||
float scale = 1.0;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
q = abs(q) - 0.35;
|
||||
if (q.y > q.x) q = q.yx;
|
||||
q *= 1.5; scale *= 1.5;
|
||||
}
|
||||
float v = length(q) / scale;
|
||||
float n = fbm(q * 3.0 + U.time * 0.2);
|
||||
float3 col = palIQ(v + n * 0.3,
|
||||
float3(0.20, 0.10, 0.30),
|
||||
float3(0.55, 0.40, 0.50),
|
||||
float3(1.0, 1.0, 0.5),
|
||||
float3(0.0, 0.25, 0.50));
|
||||
col = col * (1.0 - exp(-v * 6.0));
|
||||
return col * (0.8 + U.rms * 1.0)
|
||||
+ float3(1.0, 0.6, 0.2) * U.flare * 1.2;
|
||||
}
|
||||
|
||||
// ---- 4 voronoi : 3D crystalline cellular ----
|
||||
float3 mode_voronoi(float2 p, constant SceneUniforms& U) {
|
||||
// 3D voronoi : on echantillonne dans une grille 3D animee
|
||||
float t = U.time * (0.4 + U.rms * 1.0);
|
||||
float3 P = float3(p * 3.5, t);
|
||||
float3 ip = floor(P);
|
||||
float3 fp = fract(P);
|
||||
float d1 = 10.0, d2 = 10.0;
|
||||
for (int z = -1; z <= 1; ++z)
|
||||
for (int y = -1; y <= 1; ++y)
|
||||
for (int x = -1; x <= 1; ++x) {
|
||||
float3 g = float3(float(x), float(y), float(z));
|
||||
float3 o = float3(hash31(ip + g + 13.0),
|
||||
hash31(ip + g + 71.0),
|
||||
hash31(ip + g + 47.0));
|
||||
o = 0.5 + 0.5 * sin(t + 6.28 * o);
|
||||
float3 dv = g + o - fp;
|
||||
float d = dot(dv, dv);
|
||||
if (d < d1) { d2 = d1; d1 = d; }
|
||||
else if (d < d2) { d2 = d; }
|
||||
}
|
||||
d1 = sqrt(d1); d2 = sqrt(d2);
|
||||
float edge = smoothstep(0.0, 0.04, d2 - d1); // walls between cells
|
||||
float face = smoothstep(0.0, 0.6, d1);
|
||||
float3 base = palIQ(d1,
|
||||
float3(0.05, 0.08, 0.20),
|
||||
float3(0.45, 0.35, 0.55),
|
||||
float3(1.0, 1.0, 0.6),
|
||||
float3(0.2, 0.3, 0.0));
|
||||
return base * (1.0 - face) + float3(1.0) * (1.0 - edge) * 0.5
|
||||
+ U.lightning_flash * 0.8;
|
||||
}
|
||||
|
||||
// ---- 5 metaballs : raymarched SDF ----
|
||||
float metaballs_dist(float3 p, constant SceneUniforms& U) {
|
||||
float t = U.time * 0.7;
|
||||
float d = 100.0;
|
||||
for (int k = 0; k < 5; ++k) {
|
||||
float fk = float(k);
|
||||
float3 c = float3(
|
||||
sin(t * (0.6 + 0.13 * fk) + fk * 1.7) * 1.2,
|
||||
cos(t * (0.5 + 0.11 * fk) + fk * 2.1) * 1.0,
|
||||
sin(t * (0.4 + 0.09 * fk) + fk * 3.0) * 0.8
|
||||
);
|
||||
float radius = 0.45 + 0.15 * U.rms + 0.05 * sin(t + fk);
|
||||
d = smin(d, sdSphere(p - c, radius), 0.45);
|
||||
}
|
||||
return d;
|
||||
}
|
||||
float3 mode_metaballs(float2 p, constant SceneUniforms& U) {
|
||||
float3 ro = float3(0, 0, -3.5);
|
||||
float3 rd = normalize(float3(p, 1.5));
|
||||
float t = 0.0;
|
||||
float glow = 0.0;
|
||||
int i;
|
||||
for (i = 0; i < 64; ++i) {
|
||||
float3 pos = ro + rd * t;
|
||||
float d = metaballs_dist(pos, U);
|
||||
if (d < 0.01) break;
|
||||
glow += 0.02 / (1.0 + d * d * 4.0);
|
||||
t += d * 0.9;
|
||||
if (t > 8.0) break;
|
||||
}
|
||||
float3 col = float3(0);
|
||||
if (t < 8.0) {
|
||||
float3 pos = ro + rd * t;
|
||||
// normal via gradient
|
||||
float2 e = float2(0.001, 0);
|
||||
float3 n = normalize(float3(
|
||||
metaballs_dist(pos + e.xyy, U) - metaballs_dist(pos - e.xyy, U),
|
||||
metaballs_dist(pos + e.yxy, U) - metaballs_dist(pos - e.yxy, U),
|
||||
metaballs_dist(pos + e.yyx, U) - metaballs_dist(pos - e.yyx, U)));
|
||||
float3 lightDir = normalize(float3(0.6, 0.8, -0.5));
|
||||
float lambert = max(0.0, dot(n, lightDir));
|
||||
float fres = pow(1.0 - max(0.0, dot(n, -rd)), 2.0);
|
||||
col = palIQ(pos.x * 0.3 + pos.y * 0.2 + U.time * 0.1,
|
||||
float3(0.2, 0.0, 0.3),
|
||||
float3(0.5, 0.5, 0.4),
|
||||
float3(1.0),
|
||||
float3(0.0, 0.33, 0.67)) * lambert;
|
||||
col += float3(0.3, 0.7, 1.0) * fres * (0.7 + U.kp_norm);
|
||||
}
|
||||
col += float3(0.2, 0.6, 1.0) * glow * 1.5;
|
||||
return col + U.lightning_flash * 0.6;
|
||||
}
|
||||
|
||||
// ---- 6 starfield : galaxy spiral + parallax ----
|
||||
float3 mode_starfield(float2 p, constant SceneUniforms& U) {
|
||||
float warp = U.time * (1.5 + U.wind_norm * 6.0);
|
||||
// 3 layers of stars at different speeds
|
||||
float3 col = float3(0);
|
||||
for (int L = 0; L < 3; ++L) {
|
||||
float speed = (1.0 + float(L) * 0.5);
|
||||
float scale = 6.0 + float(L) * 4.0;
|
||||
for (int k = 0; k < 50; ++k) {
|
||||
float fk = float(k + L * 50);
|
||||
float r0 = hash21(float2(fk, 7.0 + float(L)));
|
||||
float a0 = hash21(float2(fk, 17.0 + float(L))) * 6.28;
|
||||
// Spirale galactique
|
||||
float angle = a0 + r0 * 4.0;
|
||||
float dist = fract(r0 + warp * 0.04 * speed) * 1.6;
|
||||
float2 q = float2(cos(angle + dist * 1.5),
|
||||
sin(angle + dist * 1.5)) * dist;
|
||||
float d = length(p - q);
|
||||
float bright = smoothstep(0.012 / speed, 0.0, d);
|
||||
col += float3(0.5 + r0 * 0.5, 0.7 - r0 * 0.3, 1.0) * bright
|
||||
* (1.4 - dist) * (1.0 / speed);
|
||||
}
|
||||
}
|
||||
// God rays subtils depuis le centre
|
||||
float ang = atan2(p.y, p.x);
|
||||
float rays = 0.5 + 0.5 * sin(ang * 8.0 + U.time);
|
||||
col += float3(0.3, 0.4, 0.7) * rays * (1.0 - length(p)) * 0.15
|
||||
* (0.5 + U.kp_norm);
|
||||
return col + U.flare * float3(1.0, 0.5, 0.2) * 0.4;
|
||||
}
|
||||
|
||||
// ---- 7 bars : 3D pillars en perspective ----
|
||||
float3 mode_bars(float2 p, constant SceneUniforms& U) {
|
||||
// Pseudo-3D : barres "horizontales" qui s'eloignent
|
||||
int nbars = 24;
|
||||
float t = U.time * 0.4;
|
||||
float3 col = float3(0);
|
||||
// Sky gradient
|
||||
float3 sky = mix(float3(0.05, 0.0, 0.15), float3(0.25, 0.1, 0.35),
|
||||
p.y * 0.5 + 0.5);
|
||||
col = sky;
|
||||
for (int i = 0; i < nbars; ++i) {
|
||||
float fi = float(i) / float(nbars);
|
||||
// Position en profondeur (z = 0 proche, 1 loin)
|
||||
float z = fract(fi + t * (0.15 + U.rms * 0.3));
|
||||
float perspective = 1.0 / (z + 0.1);
|
||||
float y_base = -0.6 + z * 1.2; // ligne d'horizon
|
||||
// Hauteur barre depend du bin "i" via hash + RMS
|
||||
float h0 = hash21(float2(float(i), 0.0));
|
||||
float h = sin(t * (0.5 + h0 * 4.0) + float(i)) * 0.5 + 0.5;
|
||||
h = h * (0.3 + U.rms * 1.5 + U.social_rate * 0.4);
|
||||
h = clamp(h, 0.02, 0.85);
|
||||
float bar_top = y_base + h * perspective * 0.3;
|
||||
// Largeur = 1 / nbars perspective
|
||||
float bx = (fi - 0.5) * perspective * 1.5;
|
||||
float bw = 0.5 / float(nbars) * perspective;
|
||||
if (abs(p.x - bx) < bw &&
|
||||
p.y > y_base && p.y < bar_top) {
|
||||
float3 c = palIQ(fi,
|
||||
float3(0.5), float3(0.5),
|
||||
float3(1.0, 1.0, 0.5),
|
||||
float3(0.0, 0.33, 0.67));
|
||||
// Fog selon z
|
||||
c *= 1.0 - z * 0.6;
|
||||
col = mix(col, c, 1.0 - z * 0.3);
|
||||
}
|
||||
}
|
||||
// Grille du sol scanline
|
||||
float floor_y = -0.6;
|
||||
if (p.y < floor_y) {
|
||||
float depth = (floor_y - p.y) * 4.0;
|
||||
float grid = step(0.95, fract(p.x * 8.0 / max(depth, 0.1)));
|
||||
grid += step(0.95, fract(depth * 4.0 + t));
|
||||
col += float3(0.2, 0.3, 0.6) * grid * 0.4;
|
||||
}
|
||||
return col + U.flare * float3(1.0, 0.5, 0.2) * 0.3;
|
||||
}
|
||||
|
||||
// ---- 8 hands3d : voyage 3D pilote par les mains ----
|
||||
float map_hands(float3 p, constant SceneUniforms& U) {
|
||||
float3 q = fmod(p + 2.0, 4.0) - 2.0;
|
||||
float d = length(q) - 0.6;
|
||||
float pulse = 0.8 + U.rms * 0.6;
|
||||
d = min(d, length(p) - pulse);
|
||||
d += sin(p.x * 2.0 + U.time) * 0.15 * U.kp_norm;
|
||||
return d;
|
||||
}
|
||||
float3 mode_hands3d(float2 p, constant SceneUniforms& U) {
|
||||
float hl_active = (abs(U.hand_l_x) + abs(U.hand_l_y)) > 0.01 ? 1.0 : 0.0;
|
||||
float hr_active = (abs(U.hand_r_x) + abs(U.hand_r_y)) > 0.01 ? 1.0 : 0.0;
|
||||
float3 cam_pos = float3(
|
||||
U.hand_l_x * 5.0,
|
||||
U.hand_l_y * 3.0,
|
||||
-U.time * (1.5 + U.hand_l_y * 4.0 * hl_active)
|
||||
);
|
||||
float yaw = U.hand_r_x * 1.2 * hr_active;
|
||||
float pitch = -U.hand_r_y * 0.8 * hr_active;
|
||||
float3 rd = normalize(float3(p.x, p.y, 1.5));
|
||||
rd = rotX(rd, pitch);
|
||||
rd = rotY(rd, yaw);
|
||||
float t = 0.0, glow = 0.0;
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
float3 pos = cam_pos + rd * t;
|
||||
float d = map_hands(pos, U);
|
||||
if (d < 0.005) break;
|
||||
glow += 0.02 / (1.0 + d * d * 8.0);
|
||||
t += d * 0.85;
|
||||
if (t > 30.0) break;
|
||||
}
|
||||
float3 col = float3(0);
|
||||
if (t < 30.0) {
|
||||
float3 pos = cam_pos + rd * t;
|
||||
float fog = 1.0 - saturate(t / 30.0);
|
||||
col = float3(
|
||||
0.5 + 0.5 * sin(pos.x * 0.4 + U.time),
|
||||
0.5 + 0.5 * sin(pos.y * 0.5 + U.time * 1.3),
|
||||
0.5 + 0.5 * sin(pos.z * 0.3 + U.time * 0.7)
|
||||
) * fog;
|
||||
}
|
||||
col += float3(0.2, 0.6, 1.0) * glow * 1.5;
|
||||
col += float3(1.0, 0.5, 0.0) * U.flare * 0.8;
|
||||
return col;
|
||||
}
|
||||
|
||||
// ---- 9 openpos : fond minimal radial pour faire ressortir le squelette ----
|
||||
// Le rendu des joints + bones se fait par le skel_pipeline rendu PAR-DESSUS
|
||||
// (cf renderer.py). On laisse juste un degrade radial sombre pour le contraste.
|
||||
float3 mode_openpos(float2 p, constant SceneUniforms& U) {
|
||||
float r = length(p);
|
||||
// Centre legerement plus clair, bords sombres. Touche de couleur
|
||||
// chaude au centre selon rms pour reagir a la musique.
|
||||
float3 inner = float3(0.05, 0.05, 0.10) + float3(0.30, 0.12, 0.18) * U.rms;
|
||||
float3 outer = float3(0.01, 0.01, 0.02);
|
||||
float3 col = mix(inner, outer, smoothstep(0.0, 1.4, r));
|
||||
// Grille de points discrete pour donner une ref de profondeur
|
||||
float2 g = fmod(p * 12.0, 2.0) - 1.0;
|
||||
float dot_grid = exp(-dot(g, g) * 6.0) * 0.04;
|
||||
col += float3(dot_grid);
|
||||
// Pulsation legere sur le kick / drop
|
||||
col *= 1.0 + U.rms * 0.4;
|
||||
return col;
|
||||
}
|
||||
|
||||
// ===== Fragment dispatcher =========================================
|
||||
|
||||
fragment float4 bg_fragment(VsOut in [[stage_in]],
|
||||
constant SceneUniforms& U [[buffer(0)]]) {
|
||||
float2 uv = in.uv;
|
||||
float2 p = uv * 2.0 - 1.0;
|
||||
p.x *= U.width / U.height;
|
||||
|
||||
int mode = int(U.viz_mode + 0.5);
|
||||
float3 color;
|
||||
if (mode == 1) color = mode_tunnel(p, U);
|
||||
else if (mode == 2) color = mode_plasma(p, U);
|
||||
else if (mode == 3) color = mode_kaleido(p, U);
|
||||
else if (mode == 4) color = mode_voronoi(p, U);
|
||||
else if (mode == 5) color = mode_metaballs(p, U);
|
||||
else if (mode == 6) color = mode_starfield(p, U);
|
||||
else if (mode == 7) color = mode_bars(p, U);
|
||||
else if (mode == 8) color = mode_hands3d(p, U);
|
||||
else if (mode == 9) color = mode_openpos(p, U);
|
||||
else color = mode_storm(p, U);
|
||||
|
||||
// Flash global + vignette
|
||||
color += float3(U.lightning_flash * 1.2);
|
||||
color *= vignette(p);
|
||||
|
||||
// Tone mapping doux (Reinhard)
|
||||
color = color / (1.0 + color);
|
||||
// Gamma
|
||||
color = pow(color, float3(0.85));
|
||||
|
||||
// Alpha pour transparence quand pose active (webcam visible dessous)
|
||||
// Overlay vidéo : translucide même sans pose (la webcam doit rester
|
||||
// visible en fond). Pose active = encore plus translucide.
|
||||
float alpha = mix(0.55, 0.25, U.pose_alive);
|
||||
alpha = max(alpha, U.lightning_flash * 0.8);
|
||||
alpha = max(alpha, U.flare * 0.6);
|
||||
return float4(color, alpha);
|
||||
}
|
||||
|
||||
// ===== Skeleton overlay ============================================
|
||||
|
||||
struct SkelIn {
|
||||
float3 pos [[attribute(0)]]; // x,y dans NDC, z profondeur (~ -0.5..+0.5)
|
||||
float conf [[attribute(1)]];
|
||||
float pid [[attribute(2)]]; // person_id (0..9)
|
||||
};
|
||||
struct SkelOut {
|
||||
float4 position [[position]];
|
||||
float conf;
|
||||
float pid;
|
||||
float depth;
|
||||
};
|
||||
|
||||
// Projection perspective douce : eloigne avec z, garde NDC en x,y
|
||||
vertex SkelOut skel_vertex(SkelIn in [[stage_in]],
|
||||
constant SceneUniforms& U [[buffer(1)]]) {
|
||||
SkelOut o;
|
||||
float z = clamp(in.pos.z, -1.0, 1.0);
|
||||
// Perspective : plus z augmente, plus le point est loin → scale < 1
|
||||
// RMS pulse fait respirer la profondeur
|
||||
float pulse = 1.0 + U.rms * 0.25;
|
||||
float persp = 1.0 / (1.0 + z * 0.8);
|
||||
float2 xy = in.pos.xy * persp * pulse;
|
||||
o.position = float4(xy, 0.0, 1.0);
|
||||
o.conf = in.conf;
|
||||
o.pid = in.pid;
|
||||
o.depth = z;
|
||||
return o;
|
||||
}
|
||||
|
||||
// Palette 6 couleurs par personne (turquoise, magenta, jaune, ambre, lilas, vert)
|
||||
constant float3 PERSON_COLORS[6] = {
|
||||
float3(0.0, 1.0, 0.85), // 0 turquoise
|
||||
float3(1.0, 0.3, 0.7), // 1 magenta
|
||||
float3(1.0, 0.9, 0.2), // 2 jaune
|
||||
float3(1.0, 0.55, 0.1), // 3 ambre
|
||||
float3(0.7, 0.5, 1.0), // 4 lilas
|
||||
float3(0.4, 1.0, 0.3), // 5+ vert (mains)
|
||||
};
|
||||
|
||||
// ===== Mesh overlay (triangles face/hand/body) =====================
|
||||
// Reuse meme layout que skel : pos.xyz + conf + pid.
|
||||
|
||||
vertex SkelOut mesh_vertex(SkelIn in [[stage_in]],
|
||||
constant SceneUniforms& U [[buffer(1)]]) {
|
||||
SkelOut o;
|
||||
float z = clamp(in.pos.z, -1.0, 1.0);
|
||||
float pulse = 1.0 + U.rms * 0.25;
|
||||
float persp = 1.0 / (1.0 + z * 0.8);
|
||||
float2 xy = in.pos.xy * persp * pulse;
|
||||
o.position = float4(xy, 0.0, 1.0);
|
||||
o.conf = in.conf;
|
||||
o.pid = in.pid;
|
||||
o.depth = z;
|
||||
return o;
|
||||
}
|
||||
|
||||
fragment float4 mesh_fragment(SkelOut in [[stage_in]]) {
|
||||
int pid = int(in.pid + 0.5);
|
||||
pid = ((pid % 6) + 6) % 6;
|
||||
float3 col = PERSON_COLORS[pid];
|
||||
float c = saturate(in.conf);
|
||||
// Saturation boost : couleurs vives quand pose detectee
|
||||
col = mix(col, col * 1.6, c);
|
||||
// Fog par profondeur (proche = plus lumineux)
|
||||
float depth_fog = 1.0 - clamp(in.depth + 0.5, 0.0, 1.0) * 0.5;
|
||||
col *= depth_fog;
|
||||
// Alpha TRES VISIBLE quand confiance haute : 0.85 sur skin, 0.3 fade
|
||||
return float4(col, mix(0.3, 0.85, c));
|
||||
}
|
||||
|
||||
fragment float4 skel_fragment(SkelOut in [[stage_in]]) {
|
||||
// Skeleton ULTRA visible quand pose detectee : couleur vive + opaque
|
||||
int pid = int(in.pid + 0.5);
|
||||
pid = ((pid % 6) + 6) % 6; // modulo positif
|
||||
float3 col = PERSON_COLORS[pid] * 1.4; // saturation boost
|
||||
float c = saturate(in.conf);
|
||||
// Depth fog : eclaircit ce qui est proche, eteint ce qui est loin
|
||||
float depth_fog = 1.0 - clamp(in.depth + 0.5, 0.0, 1.0) * 0.6;
|
||||
col *= depth_fog * (0.5 + 0.5 * c);
|
||||
// Alpha plein-opaque quand confiance haute (= squelette ultra net)
|
||||
return float4(col, mix(0.5, 1.0, c));
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
# SMPLer-X + RealityKit — pivot pour mesh whole-body anime temps reel
|
||||
|
||||
## Pourquoi pivoter
|
||||
|
||||
État actuel :
|
||||
- Apple Vision body pose : **OK** (13 joints ARKit, marche)
|
||||
- Apple Vision face landmarks : **bloqué** (`PyObjCPointer` non castable ctypes en Python 3.14)
|
||||
- Apple Vision hand pose : même blocage probable
|
||||
|
||||
Pour avoir un **mesh humain complet animé** (corps + visage + mains), la
|
||||
direction recommandée 2025 est **SMPLer-X** (ECCV 2024) qui produit des
|
||||
paramètres SMPL-X complets, décodables en mesh **10475 vertices** animé
|
||||
en temps réel.
|
||||
|
||||
## SMPLer-X
|
||||
|
||||
Repo : <https://github.com/caizhongang/SMPLer-X>
|
||||
Paper : [arXiv:2309.17448](https://arxiv.org/abs/2309.17448)
|
||||
|
||||
Output par personne (par frame) :
|
||||
- 144 paramètres SMPL-X (β shape + θ pose + expression + jaw + eye)
|
||||
- Décodable via `smplx` Python library en :
|
||||
- **10475 vertices** 3D
|
||||
- **127 joints** (corps + mains + visage)
|
||||
- **20908 triangles** (topologie standard SMPL-X)
|
||||
|
||||
Modèles :
|
||||
- `SMPLer-X-S` : ViT-S, 64 ms M5
|
||||
- `SMPLer-X-B` : ViT-B, 110 ms M5
|
||||
- `SMPLer-X-L` : ViT-L, 220 ms M5
|
||||
|
||||
Cible temps réel : **S** ou **B**, 10-15 fps acceptables avec frame skip.
|
||||
|
||||
## RealityKit pour le rendu
|
||||
|
||||
RealityKit (Apple) est l'API moderne pour scènes 3D :
|
||||
- Mesh skinning natif (animation de vertices par rig)
|
||||
- Render Metal sous le capot, ANE pour Object Detection
|
||||
- API Swift/Obj-C disponible via pyobjc-framework-RealityKit
|
||||
- Affiche les meshes animés en temps réel, multi-personne
|
||||
|
||||
## Architecture proposée
|
||||
|
||||
```
|
||||
data_only_viz/
|
||||
├── apple_vision_pose.py # actuel : body pose (~13 joints), gardé
|
||||
├── smpler_x_worker.py # NOUVEAU : SMPLer-X transformer → SMPL-X params
|
||||
├── smplx_mesh.py # NOUVEAU : décode params → vertices 3D
|
||||
└── reality_kit_view.py # NOUVEAU : affiche mesh dans RKView Swift bridge
|
||||
```
|
||||
|
||||
State extension :
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class SMPLXPerson:
|
||||
pid: int
|
||||
vertices_3d: list[tuple[float, float, float]] # 10475 verts (mètres)
|
||||
joints_3d: list[tuple[float, float, float]] # 127 joints
|
||||
faces: list[tuple[int, int, int]] # 20908 triangles (statique)
|
||||
cam_t: tuple[float, float, float] # position cam → personne
|
||||
|
||||
persons_smplx: list[SMPLXPerson]
|
||||
```
|
||||
|
||||
## Procédure d'installation
|
||||
|
||||
```bash
|
||||
# 1. SMPLer-X
|
||||
git clone https://github.com/caizhongang/SMPLer-X ~/.cache/av-live-smplerx
|
||||
cd ~/.cache/av-live-smplerx
|
||||
uv pip install --python /path/to/.venv/bin/python \
|
||||
torch torchvision smplx \
|
||||
mmcv-full mmdet mmpose mmtrack \
|
||||
"numpy<2"
|
||||
# Note : mm* libraries peuvent etre compliquees sur ARM macOS
|
||||
|
||||
# 2. SMPL-X model (academic license required)
|
||||
# Register at https://smpl-x.is.tue.mpg.de/
|
||||
# Download SMPLX_NEUTRAL.npz, place in models/smplx/
|
||||
|
||||
# 3. SMPLer-X checkpoint
|
||||
wget https://github.com/caizhongang/SMPLer-X/releases/download/v0.1.0/smpler_x_s32.pth.tar \
|
||||
-O checkpoints/smpler_x_s32.pth.tar
|
||||
|
||||
# 4. pyobjc-framework-RealityKit (peut etre absent du PyPI Python 3.14)
|
||||
uv pip install pyobjc-framework-RealityKit # ou via loadBundle
|
||||
```
|
||||
|
||||
## Phases d'integration
|
||||
|
||||
1. **Phase 1** : SMPLer-X seul → mesh dans state, rendu Metal triangles
|
||||
plein (pas RealityKit) — 1-2 jours de travail
|
||||
2. **Phase 2** : RealityKit Bridge → mesh skinné dans RKView superposé
|
||||
au MTKView — 3-4 jours, requires Swift bindings
|
||||
3. **Phase 3** : streaming USDZ depuis Python → RealityKit consomme —
|
||||
1 jour, plus simple si Swift bindings difficiles
|
||||
|
||||
## Trade-offs vs Apple Vision actuel
|
||||
|
||||
| Critere | Apple Vision (actuel) | SMPLer-X + RealityKit |
|
||||
|---|---|---|
|
||||
| Latence M5 | 5 ms (ANE) | 65 ms (ViT-S MPS) |
|
||||
| Body joints | 13 (ARKit) | 127 |
|
||||
| Face mesh | 0 (bloqué) | OUI (10475 verts dont 4716 face) |
|
||||
| Hands | 0 (bloqué) | OUI (15 joints × 2) |
|
||||
| 3D monde | Non | OUI (position 3D + camera) |
|
||||
| Multi-personne | OUI | OUI |
|
||||
| Install complexity | 0 (system) | Elevee (SMPL-X + mmlib) |
|
||||
|
||||
## Recommandation
|
||||
|
||||
À court terme :
|
||||
- **Garder Apple Vision body pose** (rapide, marche)
|
||||
- **Ajouter SMPLer-X** en worker complémentaire, rendu mesh basse cadence
|
||||
- **Skipper RealityKit** dans un premier temps : rendu Metal triangles
|
||||
remplis suffit pour démarrer
|
||||
|
||||
À moyen terme :
|
||||
- Migration RealityKit si SMPLer-X marche bien et qu'on veut le mesh
|
||||
skinné avec éclairage 3D Apple natif
|
||||
|
||||
## Alternatives à considérer
|
||||
|
||||
- **PIXIE** (Yale 2021) : SMPL-X plus rapide mais moins précis
|
||||
- **OSX** (CVPR 2023) : architecture transformer pour SMPL-X
|
||||
- **Hand4Whole** (CVPR 2023) : SMPL-X focus mains/visage
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Wrapper minimal autour de smplx.SMPLXLayer pour decoder les params
|
||||
de Multi-HMR (betas + thetas + expression) en vertices 3D."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
LOG = logging.getLogger("smplx_decoder")
|
||||
|
||||
|
||||
def _require_torch():
|
||||
"""Return the torch module, raising a clear error if not installed."""
|
||||
try:
|
||||
import torch
|
||||
return torch
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"smplx_decoder requires the 'multihmr' extra: "
|
||||
"uv sync --extra multihmr"
|
||||
) from e
|
||||
|
||||
|
||||
class SMPLXDecoder:
|
||||
"""Charge SMPL-X NEUTRAL et expose decode(params) -> (verts, joints)."""
|
||||
|
||||
def __init__(self, model_path: str, device: str = "mps") -> None:
|
||||
torch = _require_torch()
|
||||
# Demote unsupported devices to CPU (mirrors MultiHMRWorker pattern)
|
||||
if device == "mps" and not torch.backends.mps.is_available():
|
||||
device = "cpu"
|
||||
elif device.startswith("cuda") and not torch.cuda.is_available():
|
||||
device = "cpu"
|
||||
self.device = device
|
||||
import smplx
|
||||
model_path_p = Path(model_path)
|
||||
if model_path_p.is_file():
|
||||
# smplx.SMPLXLayer attend le dossier contenant SMPLX_<GENDER>.<ext>
|
||||
model_folder = str(model_path_p.parent)
|
||||
ext = "npz" if model_path_p.suffix == ".npz" else "pkl"
|
||||
else:
|
||||
model_folder = str(model_path_p)
|
||||
ext = "npz"
|
||||
self.layer = smplx.SMPLXLayer(
|
||||
model_path=model_folder,
|
||||
gender="neutral",
|
||||
num_betas=10,
|
||||
num_expression_coeffs=10,
|
||||
ext=ext,
|
||||
).to(self.device).eval()
|
||||
LOG.info("SMPL-X loaded from %s (device=%s)", model_folder, self.device)
|
||||
|
||||
def decode(
|
||||
self,
|
||||
betas: "torch.Tensor",
|
||||
body_pose: "torch.Tensor",
|
||||
global_orient: "torch.Tensor",
|
||||
left_hand_pose: "torch.Tensor",
|
||||
right_hand_pose: "torch.Tensor",
|
||||
jaw_pose: "torch.Tensor",
|
||||
expression: "torch.Tensor",
|
||||
transl: "torch.Tensor",
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
torch = _require_torch()
|
||||
with torch.no_grad():
|
||||
out = self.layer(
|
||||
betas=betas, body_pose=body_pose, global_orient=global_orient,
|
||||
left_hand_pose=left_hand_pose, right_hand_pose=right_hand_pose,
|
||||
jaw_pose=jaw_pose, expression=expression, transl=transl,
|
||||
return_verts=True,
|
||||
)
|
||||
return out.vertices.cpu().numpy(), out.joints.cpu().numpy()
|
||||
|
||||
def decode_neutral(self) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""T-pose neutre. Les poses sont des matrices de rotation : on
|
||||
utilise l'identite (pas zeros, qui collapserait le mesh)."""
|
||||
torch = _require_torch()
|
||||
d = self.device
|
||||
B = 1
|
||||
|
||||
def eye(n: int) -> "torch.Tensor":
|
||||
return torch.eye(3, device=d).expand(B, n, 3, 3).contiguous()
|
||||
|
||||
with torch.no_grad():
|
||||
out = self.layer(
|
||||
betas=torch.zeros((B, 10), device=d),
|
||||
body_pose=eye(21),
|
||||
global_orient=eye(1),
|
||||
left_hand_pose=eye(15),
|
||||
right_hand_pose=eye(15),
|
||||
jaw_pose=eye(1),
|
||||
expression=torch.zeros((B, 10), device=d),
|
||||
transl=torch.zeros((B, 3), device=d),
|
||||
)
|
||||
return (out.vertices[0].cpu().numpy(),
|
||||
out.joints[0].cpu().numpy())
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Envoie les vertices SMPL-X de chaque personne via TCP sur :57130.
|
||||
|
||||
UDP ne suffit pas : 10475 verts x 3 floats x 4 = 125 700 octets par
|
||||
personne, soit > MTU 1500. TCP fragmente proprement.
|
||||
|
||||
Protocole binaire little-endian, par frame :
|
||||
[4: longueur payload (uint32)]
|
||||
[4: magic 'SMPX']
|
||||
[4: n_persons (int32)]
|
||||
Pour chaque personne :
|
||||
[4: pid int32][4: confidence float32]
|
||||
[12: translation (3 float32)]
|
||||
[10*4: betas (10 float32)]
|
||||
[10*4: expression (10 float32)]
|
||||
[10475*3*4 = 125700: vertices (float32 LE)]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import socket
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
from typing import Sequence
|
||||
|
||||
import numpy as np
|
||||
|
||||
import os
|
||||
|
||||
from .mesh_rigger import MeshRigger
|
||||
from .state import SMPLXPerson, State
|
||||
|
||||
try:
|
||||
from .dino_reid import DinoReid
|
||||
except Exception: # noqa: BLE001
|
||||
DinoReid = None # type: ignore[assignment]
|
||||
|
||||
LOG = logging.getLogger("smplx_tcp")
|
||||
|
||||
MAGIC = b"SMPX"
|
||||
PORT = 57130
|
||||
|
||||
|
||||
class SMPLXTCPSender:
|
||||
def __init__(self, state: State, host: str = "127.0.0.1",
|
||||
port: int = PORT, target_fps: float = 30.0,
|
||||
enable_rigging: bool = True) -> None:
|
||||
import os as _os
|
||||
self.state = state
|
||||
self.host = _os.environ.get("AVBODY_HOST", host)
|
||||
self.port = port
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._sock: socket.socket | None = None
|
||||
# Hybrid keyframe rigging : entre deux keyframes Multi-HMR (~3 fps),
|
||||
# on translate le mesh via le delta pelvis Apple Vision (30 fps).
|
||||
# MULTIHMR_REID: 'dino' (try DINOv2 + IoU fusion, fallback IoU) /
|
||||
# 'iou' (pure IoU). Default: 'dino' if mlpackage exists.
|
||||
reid_mode = os.environ.get("MULTIHMR_REID", "dino").lower()
|
||||
dino = None
|
||||
if enable_rigging and reid_mode == "dino" and DinoReid is not None:
|
||||
try:
|
||||
if DinoReid.is_available():
|
||||
dino = DinoReid()
|
||||
LOG.info("MeshRigger: DINOv2 reid enabled")
|
||||
else:
|
||||
LOG.info(
|
||||
"MeshRigger: dino mlpackage absent, IoU only")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("MeshRigger: dino load failed (%s), IoU only", e)
|
||||
dino = None
|
||||
dino_weight = float(os.environ.get("MULTIHMR_REID_ALPHA", "0.5"))
|
||||
self._rigger = MeshRigger(
|
||||
state, dino_weight=dino_weight,
|
||||
dino_reid=dino) if enable_rigging else None
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name="smplx_tcp", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._close()
|
||||
|
||||
def _ensure_connected(self) -> bool:
|
||||
if self._sock is not None:
|
||||
return True
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(0.5)
|
||||
s.connect((self.host, self.port))
|
||||
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
s.settimeout(1.0) # 1 s write timeout — must be > worst-case frame transit
|
||||
self._sock = s
|
||||
LOG.info("connected to %s:%d", self.host, self.port)
|
||||
return True
|
||||
except (socket.error, ConnectionRefusedError):
|
||||
return False
|
||||
|
||||
def _send_or_close(self, payload: bytes) -> bool:
|
||||
"""Send *payload* (already length-prefixed) over _sock.
|
||||
|
||||
Returns True on success, False on any socket error (socket is closed).
|
||||
"""
|
||||
try:
|
||||
self._sock.sendall(payload)
|
||||
return True
|
||||
except socket.timeout:
|
||||
LOG.warning("smplx_tcp: send timeout — receiver stalled, dropping connection")
|
||||
self._close()
|
||||
return False
|
||||
except (BrokenPipeError, ConnectionResetError, OSError) as e:
|
||||
LOG.warning("smplx_tcp: send failed (%s) — reconnecting", e)
|
||||
self._close()
|
||||
return False
|
||||
|
||||
def _close(self) -> None:
|
||||
if self._sock is not None:
|
||||
try:
|
||||
self._sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
self._sock = None
|
||||
|
||||
@staticmethod
|
||||
def _pad10(arr: "np.ndarray") -> "np.ndarray":
|
||||
"""Return a contiguous float32 array of length 10, zero-padded if shorter."""
|
||||
flat = np.ascontiguousarray(arr, dtype="<f4").ravel()
|
||||
if len(flat) >= 10:
|
||||
return flat[:10]
|
||||
out = np.zeros(10, dtype="<f4")
|
||||
out[:len(flat)] = flat
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _serialize_persons(persons: Sequence[SMPLXPerson]) -> bytes:
|
||||
buf = bytearray()
|
||||
buf += MAGIC
|
||||
buf += struct.pack("<i", len(persons))
|
||||
for p in persons:
|
||||
buf += struct.pack("<i", p.pid)
|
||||
buf += struct.pack("<f", float(p.confidence))
|
||||
trans = np.ascontiguousarray(p.translation, dtype="<f4").ravel()
|
||||
if len(trans) < 3:
|
||||
t3 = np.zeros(3, dtype="<f4")
|
||||
t3[:len(trans)] = trans
|
||||
trans = t3
|
||||
buf += trans[:3].tobytes()
|
||||
buf += SMPLXTCPSender._pad10(p.betas).tobytes()
|
||||
buf += SMPLXTCPSender._pad10(p.expression).tobytes()
|
||||
buf += np.ascontiguousarray(p.vertices_3d, dtype="<f4").tobytes()
|
||||
return bytes(buf)
|
||||
|
||||
def _run(self) -> None:
|
||||
last_warn = 0.0
|
||||
n_sent = 0
|
||||
n_rigged = 0
|
||||
next_hb = time.monotonic() + 5.0
|
||||
while not self._stop.is_set():
|
||||
t0 = time.monotonic()
|
||||
if not self._ensure_connected():
|
||||
if t0 - last_warn > 5.0:
|
||||
LOG.warning("RealityKit app pas connectee (%s:%d)",
|
||||
self.host, self.port)
|
||||
last_warn = t0
|
||||
time.sleep(1.0)
|
||||
continue
|
||||
|
||||
with self.state.lock():
|
||||
persons = list(self.state.persons_smplx)
|
||||
body_kp = list(self.state.persons_body) if hasattr(
|
||||
self.state, "persons_body") else []
|
||||
body_ids = list(self.state.persons_body_ids) if hasattr(
|
||||
self.state, "persons_body_ids") else (
|
||||
list(range(len(body_kp))) if body_kp else [])
|
||||
|
||||
if persons and self._rigger is not None:
|
||||
rigged = self._rigger.apply(
|
||||
persons, body_kp, body_ids, t0)
|
||||
if rigged is not persons:
|
||||
n_rigged += 1
|
||||
persons = rigged
|
||||
|
||||
if t0 >= next_hb:
|
||||
fps = n_sent / 5.0
|
||||
rig_pct = (n_rigged / n_sent * 100.0) if n_sent else 0.0
|
||||
LOG.info("hb: %.1f fps tcp, %.0f%% rigged",
|
||||
fps, rig_pct)
|
||||
n_sent = 0
|
||||
n_rigged = 0
|
||||
next_hb = t0 + 5.0
|
||||
|
||||
if persons:
|
||||
n_sent += 1
|
||||
t_ser_start = time.monotonic()
|
||||
payload = self._serialize_persons(persons)
|
||||
t_send_start = time.monotonic()
|
||||
if not self._send_or_close(
|
||||
struct.pack("<I", len(payload)) + payload):
|
||||
continue
|
||||
t_send_end = time.monotonic()
|
||||
dt_tcp = (t_send_end - t_ser_start) * 1e3
|
||||
if LOG.isEnabledFor(logging.DEBUG) or dt_tcp > 20.0:
|
||||
LOG.log(
|
||||
logging.DEBUG if dt_tcp <= 20.0 else logging.WARNING,
|
||||
"tcp: ser=%.1f send=%.1fms",
|
||||
(t_send_start - t_ser_start) * 1e3,
|
||||
(t_send_end - t_send_start) * 1e3,
|
||||
)
|
||||
|
||||
dt = time.monotonic() - t0
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Thread-safe state container for the Metal visualizer.
|
||||
|
||||
Le listener OSC ecrit ; le renderer Metal lit a 60 fps. Tous les acces
|
||||
sont proteges par un Lock — la contention est negligeable (lectures
|
||||
courtes, ecritures rares).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass
|
||||
class PoseKp:
|
||||
x: float = 0.0
|
||||
y: float = 0.0
|
||||
z: float = 0.0 # profondeur (mediapipe world_landmarks, metres ; 0 par defaut)
|
||||
c: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kp3D:
|
||||
"""3D keypoint in metric coordinates relative to hip-center.
|
||||
Used for MediaPipe pose_world_landmarks (xyz in meters)."""
|
||||
x: float = 0.0
|
||||
y: float = 0.0
|
||||
z: float = 0.0
|
||||
c: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SMPLXPerson:
|
||||
"""Resultats Multi-HMR pour une personne : params SMPL-X + vertices
|
||||
decodes en metres. Vertices en repere camera (z > 0 devant)."""
|
||||
pid: int = -1
|
||||
vertices_3d: np.ndarray = field(default_factory=lambda: np.empty((0, 3), dtype=np.float32)) # (10475, 3)
|
||||
translation: np.ndarray = field(default_factory=lambda: np.zeros(3, dtype=np.float32)) # (3,)
|
||||
confidence: float = 0.0
|
||||
betas: np.ndarray = field(default_factory=lambda: np.zeros(10, dtype=np.float32)) # (10,)
|
||||
expression: np.ndarray = field(default_factory=lambda: np.zeros(10, dtype=np.float32)) # (10,)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NLFPerson:
|
||||
"""Resultats NLF pour une personne : vertices 3D SMPL (6890) en metres,
|
||||
coordonnees camera (z > 0 devant). Le path nonparametrique fournit les
|
||||
vertices directement sans decodage SMPL explicite."""
|
||||
pid: int = -1
|
||||
vertices_3d: tuple = field(default_factory=tuple) # ((x,y,z),) x 6890
|
||||
joints_3d: tuple = field(default_factory=tuple) # ((x,y,z),) x 24 (SMPL)
|
||||
translation: tuple = (0.0, 0.0, 0.0)
|
||||
confidence: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class State:
|
||||
# Audio sync
|
||||
bpm: float = 120.0
|
||||
beat: int = 0
|
||||
rms: float = 0.0
|
||||
amps: dict[str, float] = field(default_factory=dict)
|
||||
album: str = ""
|
||||
|
||||
# Data feeds
|
||||
bridge_alive: bool = False
|
||||
last_heartbeat: float = 0.0
|
||||
swpc_kp: float = 2.0
|
||||
swpc_flare_norm: float = 0.0
|
||||
swpc_wind_speed: float = 400.0
|
||||
swpc_bz: float = 0.0
|
||||
netz_dev: float = 0.0
|
||||
lightning_rate_min: float = 0.0
|
||||
last_lightning: tuple[float, float, float] = (0.0, 0.0, 999.0) # lat, lon, age
|
||||
last_lightning_t: float = 0.0
|
||||
usgs_last_mag: float = 0.0
|
||||
usgs_last_mag_t: float = 0.0
|
||||
aviation_count: int = 0
|
||||
social_rate: float = 0.0
|
||||
pose_count: int = 0
|
||||
pose_kp: list[PoseKp] = field(default_factory=lambda: [PoseKp() for _ in range(17)]) # YOLO COCO legacy
|
||||
pose_last_t: float = 0.0
|
||||
# MediaPipe : compat single-person (holistic legacy, fallback)
|
||||
body_kp: list[PoseKp] = field(
|
||||
default_factory=lambda: [PoseKp() for _ in range(33)])
|
||||
face_kp: list[PoseKp] = field(
|
||||
default_factory=lambda: [PoseKp() for _ in range(478)])
|
||||
left_hand_kp: list[PoseKp] = field(
|
||||
default_factory=lambda: [PoseKp() for _ in range(21)])
|
||||
right_hand_kp: list[PoseKp] = field(
|
||||
default_factory=lambda: [PoseKp() for _ in range(21)])
|
||||
body_present: bool = False
|
||||
face_present: bool = False
|
||||
hands_present: bool = False
|
||||
|
||||
# MediaPipe multi-personne : 3 workers paralleles, jusqu'a 4 sujets.
|
||||
# Chaque entree = liste de landmarks d'UNE personne. Les listes sont
|
||||
# independantes (pas d'association inter-personne — assemblees par
|
||||
# proximite si besoin dans le renderer).
|
||||
persons_body: list[list[PoseKp]] = field(default_factory=list)
|
||||
persons_face: list[list[PoseKp]] = field(default_factory=list)
|
||||
persons_hands: list[list[PoseKp]] = field(default_factory=list)
|
||||
# MediaPipe pose_world_landmarks per person : 33 keypoints in meters,
|
||||
# relative to the hip-center. Optional companion of persons_body
|
||||
# (image-space xy). Empty if no detection or backend doesn't emit it.
|
||||
persons_body3d: list[list[Kp3D]] = field(default_factory=list)
|
||||
# IDs persistants entre frames (ByteTrack-like via Hungarian IoU).
|
||||
# Couleur du skeleton dans le shader Metal = ID % palette_size.
|
||||
persons_body_ids: list[int] = field(default_factory=list)
|
||||
persons_face_ids: list[int] = field(default_factory=list)
|
||||
persons_hands_ids: list[int] = field(default_factory=list)
|
||||
|
||||
# NLF (SMPL 6890 verts x N personnes, path nonparametrique)
|
||||
persons_nlf: list = field(default_factory=list) # list[NLFPerson]
|
||||
nlf_last_t: float = 0.0
|
||||
|
||||
# Multi-HMR (SMPL-X 10475 verts x N personnes)
|
||||
persons_smplx: list = field(default_factory=list) # list[SMPLXPerson]
|
||||
smplx_last_t: float = 0.0
|
||||
|
||||
# Renderer
|
||||
width: int = 1280
|
||||
height: int = 720
|
||||
start_t: float = field(default_factory=time.monotonic)
|
||||
# Mode visuel 0..7 (cf scene.metal::bg_fragment dispatcher)
|
||||
viz_mode: int = 0
|
||||
viz_mode_names: tuple = (
|
||||
"storm", "tunnel", "plasma", "kaleido",
|
||||
"voronoi", "metaballs", "starfield", "bars",
|
||||
"hands3d", # mode 8 : voyage 3D pilote par les mains
|
||||
"openpos", # mode 9 : skeleton multi-personne sur fond minimal
|
||||
)
|
||||
# Preset open-data actif (USGS, Blitz, Wind, Kp/Bz, X-ray, OpenSky,
|
||||
# Bsky, Pose, Cosmos) — affiche dans le HUD.
|
||||
active_preset: str = ""
|
||||
# Scene audio active (envoyee par le clavier qsdfghjklm).
|
||||
active_scene: str = ""
|
||||
# Derniere frame webcam au format JPEG bytes (pour NSImageView overlay).
|
||||
# Le pose worker la met a jour ; le HUD timer lit et l'affiche.
|
||||
last_webcam_jpeg: bytes | None = None
|
||||
# Last full RGB frame fed to Multi-HMR (uint8 HxWx3, typ. 672x672).
|
||||
# Updated by multi_hmr_worker right before inference. Read by
|
||||
# MeshRigger for DINOv2-based person re-id. None when absent.
|
||||
last_frame_rgb: np.ndarray | None = None
|
||||
last_frame_rgb_t: float = 0.0
|
||||
|
||||
_lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
|
||||
|
||||
def elapsed(self) -> float:
|
||||
return time.monotonic() - self.start_t
|
||||
|
||||
def lock(self):
|
||||
return self._lock
|
||||
|
||||
def pose_alive(self, timeout: float = 1.5) -> bool:
|
||||
return (time.monotonic() - self.pose_last_t) < timeout
|
||||
|
||||
|
||||
# Mappings clavier AZERTY pour les 3 dimensions :
|
||||
# azertyuiop = video (viz mode, 8 + 2 libres)
|
||||
# qsdfghjklm = audio (scene SC)
|
||||
# wxcvbn = data source (focus HUD + signal a SC)
|
||||
|
||||
KEYMAP_VIDEO: tuple[tuple[str, str], ...] = (
|
||||
("a", "storm"),
|
||||
("z", "tunnel"),
|
||||
("e", "plasma"),
|
||||
("r", "kaleido"),
|
||||
("t", "voronoi"),
|
||||
("y", "metaballs"),
|
||||
("u", "starfield"),
|
||||
("i", "bars"),
|
||||
("o", "hands3d"), # voyage 3D pilote par les mains MediaPipe
|
||||
("p", "openpos"), # skeleton multi-personne 3D-stylise
|
||||
)
|
||||
|
||||
KEYMAP_AUDIO: tuple[tuple[str, str], ...] = (
|
||||
("q", "cavity"),
|
||||
("s", "geo"),
|
||||
("d", "body"),
|
||||
("f", "weather"),
|
||||
("g", "flight"),
|
||||
("h", "pulse"),
|
||||
("j", "quiet"),
|
||||
("k", "all"),
|
||||
("l", "full"),
|
||||
("m", "stop"),
|
||||
)
|
||||
|
||||
# Bundle preset = (source, scene SC, viz mode Metal).
|
||||
# Selectionner une source applique les 3 dimensions d'un coup : focus HUD,
|
||||
# scene audio dediee, mode visuel correspondant.
|
||||
SourceBundle = tuple[str, str, str, str] # (key, source, scene, viz)
|
||||
|
||||
KEYMAP_SOURCE: tuple[SourceBundle, ...] = (
|
||||
("w", "USGS", "geo", "voronoi"),
|
||||
("x", "Blitz", "pulse", "storm"),
|
||||
("c", "SWPC", "weather", "tunnel"),
|
||||
("v", "OpenSky", "flight", "kaleido"),
|
||||
("b", "Bsky", "pulse", "bars"),
|
||||
("n", "Pose", "body", "metaballs"),
|
||||
)
|
||||
|
||||
# 10 sources distinctes via touches 0-9 (granularite fine sur SWPC).
|
||||
KEYMAP_SOURCE_NUM: tuple[SourceBundle, ...] = (
|
||||
("0", "Cosmos", "full", "starfield"), # toutes sources
|
||||
("1", "USGS", "geo", "voronoi"), # earthquakes
|
||||
("2", "Blitz", "pulse", "storm"), # lightning
|
||||
("3", "Wind", "weather", "tunnel"), # SWPC solar wind speed
|
||||
("4", "Kp/Bz", "geo", "plasma"), # SWPC geomagnetic
|
||||
("5", "X-ray", "weather", "bars"), # SWPC solar flare
|
||||
("6", "OpenSky", "flight", "kaleido"), # aviation
|
||||
("7", "Bsky", "pulse", "bars"), # social firehose
|
||||
("8", "Pose", "body", "metaballs"), # body YOLO
|
||||
("9", "Grid", "weather", "plasma"), # netzfrequenz (futur)
|
||||
)
|
||||
@@ -0,0 +1,9 @@
|
||||
"""pytest configuration: ensure package root is on sys.path for test imports."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add the parent of data_only_viz/ to sys.path so that
|
||||
# "from data_only_viz.xxx import ..." works at module-level in test files.
|
||||
_parent = str(Path(__file__).resolve().parent.parent.parent)
|
||||
if _parent not in sys.path:
|
||||
sys.path.insert(0, _parent)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Unit tests for ActionHead feature extraction and buffers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
|
||||
def test_module_imports() -> None:
|
||||
from data_only_viz import action_head
|
||||
assert hasattr(action_head, "FeatureExtractor")
|
||||
assert hasattr(action_head, "PerPersonBuffer")
|
||||
assert hasattr(action_head, "ActionHead")
|
||||
assert action_head.WINDOW_LEN == 16
|
||||
assert action_head.J3D_JOINTS == 32
|
||||
assert action_head.FEATURE_DIM == 428
|
||||
assert action_head.HANDS_KP_TOTAL == 42
|
||||
assert action_head.HANDS_KP_FLAT == 126
|
||||
assert action_head.NUM_CLASSES == 3
|
||||
assert action_head.LABELS == ("debout", "assise", "danse")
|
||||
|
||||
|
||||
def _rand_j3d(seed: int = 0) -> np.ndarray:
|
||||
rng = np.random.default_rng(seed)
|
||||
return rng.normal(size=(32, 3)).astype(np.float32)
|
||||
|
||||
|
||||
def test_buffer_starts_empty() -> None:
|
||||
from data_only_viz.action_head import PerPersonBuffer
|
||||
buf = PerPersonBuffer()
|
||||
assert len(buf) == 0
|
||||
assert buf.frames_for(7) == []
|
||||
|
||||
|
||||
def test_buffer_append_grows_per_pid() -> None:
|
||||
from data_only_viz.action_head import PerPersonBuffer
|
||||
buf = PerPersonBuffer()
|
||||
buf.append(pid=1, j3d=_rand_j3d(1))
|
||||
buf.append(pid=1, j3d=_rand_j3d(2))
|
||||
buf.append(pid=2, j3d=_rand_j3d(3))
|
||||
assert len(buf.frames_for(1)) == 2
|
||||
assert len(buf.frames_for(2)) == 1
|
||||
|
||||
|
||||
def test_buffer_max_len_16() -> None:
|
||||
from data_only_viz.action_head import PerPersonBuffer, WINDOW_LEN
|
||||
buf = PerPersonBuffer()
|
||||
for i in range(WINDOW_LEN + 5):
|
||||
buf.append(pid=1, j3d=_rand_j3d(i))
|
||||
assert len(buf.frames_for(1)) == WINDOW_LEN
|
||||
|
||||
|
||||
def test_buffer_forget_releases_pid() -> None:
|
||||
from data_only_viz.action_head import PerPersonBuffer
|
||||
buf = PerPersonBuffer()
|
||||
buf.append(pid=1, j3d=_rand_j3d(0))
|
||||
buf.forget(1)
|
||||
assert buf.frames_for(1) == []
|
||||
assert len(buf) == 0
|
||||
|
||||
|
||||
def test_buffer_rejects_bad_shape() -> None:
|
||||
from data_only_viz.action_head import PerPersonBuffer
|
||||
buf = PerPersonBuffer()
|
||||
with pytest.raises(ValueError, match="32"):
|
||||
buf.append(pid=1, j3d=np.zeros((22, 3), dtype=np.float32))
|
||||
|
||||
|
||||
def test_feature_extractor_shape_full_buffer() -> None:
|
||||
from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, FEATURE_DIM
|
||||
frames = [_rand_j3d(i) for i in range(WINDOW_LEN)]
|
||||
feat = FeatureExtractor.from_buffer(frames)
|
||||
assert feat.shape == (428,)
|
||||
assert feat.dtype == np.float32
|
||||
assert not np.isnan(feat).any()
|
||||
|
||||
|
||||
def test_feature_extractor_short_buffer_pads() -> None:
|
||||
from data_only_viz.action_head import FeatureExtractor, FEATURE_DIM
|
||||
frames = [_rand_j3d(0), _rand_j3d(1), _rand_j3d(2)]
|
||||
feat = FeatureExtractor.from_buffer(frames)
|
||||
assert feat.shape == (FEATURE_DIM,)
|
||||
|
||||
|
||||
def test_feature_extractor_static_buffer_zero_velocity() -> None:
|
||||
from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, J3D_JOINTS
|
||||
static = _rand_j3d(42)
|
||||
frames = [static.copy() for _ in range(WINDOW_LEN)]
|
||||
feat = FeatureExtractor.from_buffer(frames)
|
||||
vel_block = feat[J3D_JOINTS * 3 : J3D_JOINTS * 3 * 2]
|
||||
assert np.allclose(vel_block, 0.0, atol=1e-6)
|
||||
|
||||
|
||||
def test_feature_extractor_kinetics_speed_and_accel() -> None:
|
||||
from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN
|
||||
frames = []
|
||||
for t in range(WINDOW_LEN):
|
||||
f = np.zeros((32, 3), dtype=np.float32)
|
||||
f[0, 0] = 0.1 * t
|
||||
frames.append(f)
|
||||
kin = FeatureExtractor.kinetics(frames)
|
||||
assert kin.shape == (3,)
|
||||
assert kin[0] > 0
|
||||
assert abs(kin[0] - 0.1 / 32) < 1e-4
|
||||
assert abs(kin[1]) < 1e-4
|
||||
|
||||
|
||||
def test_feature_extractor_symmetry_sign() -> None:
|
||||
from data_only_viz.action_head import FeatureExtractor, WINDOW_LEN, WRIST_LEFT, WRIST_RIGHT
|
||||
frames = []
|
||||
for t in range(WINDOW_LEN):
|
||||
f = np.zeros((32, 3), dtype=np.float32)
|
||||
f[WRIST_LEFT, 0] = 0.05 * t
|
||||
f[WRIST_RIGHT, 0] = -0.05 * t
|
||||
frames.append(f)
|
||||
kin = FeatureExtractor.kinetics(frames)
|
||||
assert kin[2] > 0.9
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Tests for ActionHead model (forward, step, checkpoint roundtrip)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
|
||||
def _rand_j3d(seed: int = 0) -> np.ndarray:
|
||||
rng = np.random.default_rng(seed)
|
||||
return rng.normal(size=(32, 3)).astype(np.float32)
|
||||
|
||||
|
||||
def test_model_forward_shape() -> None:
|
||||
from data_only_viz.action_head import ActionHeadModel, FEATURE_DIM, NUM_CLASSES
|
||||
model = ActionHeadModel()
|
||||
x = torch.zeros(1, FEATURE_DIM)
|
||||
h = model.init_hidden(batch=1)
|
||||
logits, h_new = model(x, h)
|
||||
assert logits.shape == (1, NUM_CLASSES)
|
||||
assert h_new.shape == h.shape
|
||||
|
||||
|
||||
def test_model_param_count_under_100k() -> None:
|
||||
from data_only_viz.action_head import ActionHeadModel
|
||||
model = ActionHeadModel()
|
||||
n = sum(p.numel() for p in model.parameters())
|
||||
assert n < 100_000, f"too many params: {n}"
|
||||
|
||||
|
||||
def test_action_head_step_warmup_returns_debout() -> None:
|
||||
from data_only_viz.action_head import ActionHead, LABELS
|
||||
head = ActionHead(ckpt_path=None)
|
||||
label, probs, kin = head.step(pid=1, j3d=_rand_j3d(0))
|
||||
assert label == LABELS[0]
|
||||
assert probs.shape == (3,)
|
||||
assert pytest.approx(float(probs[0]), abs=1e-6) == 1.0
|
||||
assert kin.shape == (3,)
|
||||
assert float(kin[0]) == 0.0
|
||||
|
||||
|
||||
def test_action_head_step_after_warmup_returns_some_label() -> None:
|
||||
from data_only_viz.action_head import ActionHead, LABELS
|
||||
head = ActionHead(ckpt_path=None)
|
||||
for i in range(5):
|
||||
label, probs, kin = head.step(pid=1, j3d=_rand_j3d(i))
|
||||
assert label in LABELS
|
||||
assert abs(float(probs.sum()) - 1.0) < 1e-5
|
||||
|
||||
|
||||
def test_action_head_forget_resets_hidden_state(tmp_path: Path) -> None:
|
||||
from data_only_viz.action_head import ActionHead
|
||||
head = ActionHead(ckpt_path=None)
|
||||
for i in range(5):
|
||||
head.step(pid=1, j3d=_rand_j3d(i))
|
||||
assert 1 in head._hidden
|
||||
head.forget(1)
|
||||
assert 1 not in head._hidden
|
||||
assert head._buffers.frames_for(1) == []
|
||||
|
||||
|
||||
def test_action_head_checkpoint_roundtrip(tmp_path: Path) -> None:
|
||||
from data_only_viz.action_head import ActionHead, ActionHeadModel
|
||||
model = ActionHeadModel()
|
||||
ckpt = tmp_path / "ah.pt"
|
||||
torch.save({"model_state_dict": model.state_dict(),
|
||||
"version": 1}, ckpt)
|
||||
head = ActionHead(ckpt_path=ckpt)
|
||||
for k, v in head._model.state_dict().items():
|
||||
assert torch.allclose(v, model.state_dict()[k])
|
||||
|
||||
|
||||
def test_action_head_step_handles_nan() -> None:
|
||||
from data_only_viz.action_head import ActionHead, LABELS
|
||||
head = ActionHead(ckpt_path=None)
|
||||
j = _rand_j3d(0)
|
||||
j[5, 1] = float("nan")
|
||||
label, probs, _kin = head.step(pid=1, j3d=j)
|
||||
assert label in LABELS
|
||||
assert not np.isnan(probs).any()
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Tests for ActionHeadPublisher."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
|
||||
class _FakeState:
|
||||
def __init__(self) -> None:
|
||||
self.persons_smplx = []
|
||||
self.smplx_last_t = 0.0
|
||||
self.persons_body3d = []
|
||||
self.persons_body_ids = []
|
||||
self.pose_last_t = 0.0
|
||||
self.persons_hands = []
|
||||
self.persons_hands_ids = []
|
||||
self.persons_face = []
|
||||
self.persons_face_ids = []
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def lock(self):
|
||||
return self._lock
|
||||
|
||||
|
||||
def _make_smplx_person(pid: int, seed: int = 0) -> dict:
|
||||
rng = np.random.default_rng(seed)
|
||||
return {"pid": pid, "v3d": rng.normal(size=(10475, 3)).astype(np.float32)}
|
||||
|
||||
|
||||
def test_publisher_smplx_source_emits_osc() -> None:
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
state.persons_smplx = [_make_smplx_person(7)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
actions = [c for c in bridge.send_action.call_args_list]
|
||||
assert len(actions) == 1
|
||||
assert actions[0].kwargs.get("pid", actions[0].args[0]) == 7
|
||||
bridge.send_enter.assert_called_with(pid=7)
|
||||
|
||||
|
||||
def test_publisher_falls_back_to_mediapipe_body3d() -> None:
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
state.persons_body3d = [[(0.1 * i, 0.2 * i, 0.3 * i) for i in range(33)]]
|
||||
state.persons_body_ids = [42]
|
||||
state.pose_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
bridge.send_action.assert_called_once()
|
||||
bridge.send_enter.assert_called_with(pid=42)
|
||||
|
||||
|
||||
def test_publisher_purges_lost_pid() -> None:
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
state.persons_smplx = [_make_smplx_person(1)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
bridge.reset_mock()
|
||||
state.persons_smplx = []
|
||||
state.smplx_last_t = 2.0
|
||||
state.persons_body3d = []
|
||||
pub._tick(t_now=1.0)
|
||||
bridge.send_leave.assert_called_with(pid=1)
|
||||
|
||||
|
||||
def test_publisher_no_double_emit_same_timestamp() -> None:
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
state.persons_smplx = [_make_smplx_person(1)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
bridge.reset_mock()
|
||||
pub._tick(t_now=1.0) # same smplx_last_t
|
||||
bridge.send_action.assert_not_called()
|
||||
|
||||
|
||||
def test_publisher_uses_face_lips_for_mouth_open() -> None:
|
||||
"""mouth_open from MediaPipe lip landmarks (idx 13 and 14) must be ~1.0."""
|
||||
from unittest.mock import patch
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher, MEDIAPIPE_LIP_UPPER_INNER, MEDIAPIPE_LIP_LOWER_INNER
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
|
||||
# Build a fake face landmark list: at least 15 landmarks.
|
||||
# idx 13 = upper inner (y=0), idx 14 = lower inner (y=1), rest zeros.
|
||||
face_kps = [(0.0, 0.0, 0.0)] * 15
|
||||
face_kps[MEDIAPIPE_LIP_UPPER_INNER] = (0.0, 0.0, 0.0)
|
||||
face_kps[MEDIAPIPE_LIP_LOWER_INNER] = (1.0, 0.0, 0.0) # 1m apart in x
|
||||
state.persons_face = [face_kps]
|
||||
state.persons_face_ids = [0]
|
||||
|
||||
captured_mouth: list[float] = []
|
||||
original_step = pub.head.step
|
||||
|
||||
def spy_step(pid, j3d, expr=None, mouth_open=0.0, hands_kp=None):
|
||||
captured_mouth.append(mouth_open)
|
||||
return original_step(pid, j3d, expr=expr, mouth_open=mouth_open, hands_kp=hands_kp)
|
||||
|
||||
pub.head.step = spy_step # type: ignore[method-assign]
|
||||
|
||||
state.persons_smplx = [_make_smplx_person(0)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
|
||||
assert len(captured_mouth) == 1
|
||||
assert abs(captured_mouth[0] - 1.0) < 1e-5
|
||||
|
||||
|
||||
def test_publisher_passes_hands_kp_to_step() -> None:
|
||||
"""hands_kp of shape (42, 3) must be passed to head.step."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
|
||||
# Two 21-kp hand arrays (left + right) for pid=0.
|
||||
rng = np.random.default_rng(7)
|
||||
left_kps = rng.normal(size=(21, 3)).astype(np.float32)
|
||||
right_kps = rng.normal(size=(21, 3)).astype(np.float32)
|
||||
# persons_hands flat list: [left, right], ids both 0 (same pid).
|
||||
state.persons_hands = [left_kps, right_kps]
|
||||
state.persons_hands_ids = [0, 0]
|
||||
|
||||
captured_hands: list = []
|
||||
original_step = pub.head.step
|
||||
|
||||
def spy_step(pid, j3d, expr=None, mouth_open=0.0, hands_kp=None):
|
||||
captured_hands.append(hands_kp)
|
||||
return original_step(pid, j3d, expr=expr, mouth_open=mouth_open, hands_kp=hands_kp)
|
||||
|
||||
pub.head.step = spy_step # type: ignore[method-assign]
|
||||
|
||||
state.persons_smplx = [_make_smplx_person(0)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
|
||||
assert len(captured_hands) == 1
|
||||
assert captured_hands[0] is not None
|
||||
assert captured_hands[0].shape == (42, 3)
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Tests for j3d augmentations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
WINDOW_LEN = 16
|
||||
|
||||
|
||||
def _sample_stack(seed: int = 0) -> np.ndarray:
|
||||
rng = np.random.default_rng(seed)
|
||||
return rng.normal(size=(WINDOW_LEN, 32, 3)).astype(np.float32)
|
||||
|
||||
|
||||
def test_mirror_swap_left_right_joints() -> None:
|
||||
from data_only_viz.training.augment import mirror_x, MIRROR_MAP
|
||||
x = _sample_stack(0)
|
||||
y = mirror_x(x)
|
||||
# Check output shape
|
||||
assert y.shape == (WINDOW_LEN, 32, 3)
|
||||
# x-coords are negated after reindexing
|
||||
assert np.allclose(y[..., 0], -x[:, list(MIRROR_MAP), :][:, :, 0], atol=1e-6)
|
||||
|
||||
|
||||
def test_noise_within_sigma() -> None:
|
||||
from data_only_viz.training.augment import add_noise
|
||||
rng = np.random.default_rng(0)
|
||||
x = _sample_stack(0)
|
||||
y = add_noise(x, sigma=0.01, rng=rng)
|
||||
diff = y - x
|
||||
assert np.allclose(diff.std(), 0.01, atol=2e-3)
|
||||
|
||||
|
||||
def test_time_stretch_keeps_shape() -> None:
|
||||
from data_only_viz.training.augment import time_stretch
|
||||
x = _sample_stack(0)
|
||||
y = time_stretch(x, factor=0.9, rng=None)
|
||||
assert y.shape == x.shape
|
||||
|
||||
|
||||
def test_rotate_y_preserves_distances() -> None:
|
||||
from data_only_viz.training.augment import rotate_y
|
||||
x = _sample_stack(0)
|
||||
y = rotate_y(x, angle_rad=0.3)
|
||||
d_x = np.linalg.norm(x[0, 0] - x[0, 1])
|
||||
d_y = np.linalg.norm(y[0, 0] - y[0, 1])
|
||||
assert abs(d_x - d_y) < 1e-5
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Tests for rule-based auto-labeler."""
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from data_only_viz.action_head import WINDOW_LEN
|
||||
|
||||
|
||||
def _static_seated(frame_count: int = WINDOW_LEN) -> list[np.ndarray]:
|
||||
"""Hip low (y small), knee bent ~80°."""
|
||||
frames = []
|
||||
for _ in range(frame_count):
|
||||
f = np.zeros((32, 3), dtype=np.float32)
|
||||
f[1] = [-0.1, 0.4, 0.0]
|
||||
f[2] = [0.1, 0.4, 0.0]
|
||||
f[4] = [-0.1, 0.4, 0.3]
|
||||
f[5] = [0.1, 0.4, 0.3]
|
||||
f[7] = [-0.1, 0.1, 0.3]
|
||||
f[8] = [0.1, 0.1, 0.3]
|
||||
frames.append(f)
|
||||
return frames
|
||||
|
||||
|
||||
def _static_standing(frame_count: int = WINDOW_LEN) -> list[np.ndarray]:
|
||||
"""Hip high, knees ~180°."""
|
||||
frames = []
|
||||
for _ in range(frame_count):
|
||||
f = np.zeros((32, 3), dtype=np.float32)
|
||||
f[1] = [-0.1, 0.9, 0.0]
|
||||
f[2] = [0.1, 0.9, 0.0]
|
||||
f[4] = [-0.1, 0.5, 0.0]
|
||||
f[5] = [0.1, 0.5, 0.0]
|
||||
f[7] = [-0.1, 0.1, 0.0]
|
||||
f[8] = [0.1, 0.1, 0.0]
|
||||
frames.append(f)
|
||||
return frames
|
||||
|
||||
|
||||
def _dancing(frame_count: int = WINDOW_LEN) -> list[np.ndarray]:
|
||||
"""Standing pose with high wrist velocity."""
|
||||
base = _static_standing(1)[0]
|
||||
frames = []
|
||||
for t in range(frame_count):
|
||||
f = base.copy()
|
||||
phase = 2 * np.pi * t * 0.125 # 0.125 = 1/8, slower oscillation
|
||||
f[20] = base[20] + np.array([np.sin(phase) * 0.5, np.cos(phase) * 0.5, 0])
|
||||
f[21] = base[21] + np.array(
|
||||
[-np.sin(phase) * 0.5, np.cos(phase) * 0.5, 0]
|
||||
)
|
||||
frames.append(f.astype(np.float32))
|
||||
return frames
|
||||
|
||||
|
||||
def test_autolabel_static_standing_is_debout() -> None:
|
||||
from data_only_viz.training.autolabel import autolabel_window
|
||||
|
||||
label, conf = autolabel_window(_static_standing())
|
||||
assert label == "debout"
|
||||
assert conf >= 0.5
|
||||
|
||||
|
||||
def test_autolabel_static_seated_is_assise() -> None:
|
||||
from data_only_viz.training.autolabel import autolabel_window
|
||||
|
||||
label, conf = autolabel_window(_static_seated())
|
||||
assert label == "assise"
|
||||
assert conf >= 0.5
|
||||
|
||||
|
||||
def test_autolabel_dancing_is_danse() -> None:
|
||||
from data_only_viz.training.autolabel import autolabel_window
|
||||
|
||||
label, conf = autolabel_window(_dancing())
|
||||
assert label == "danse"
|
||||
assert conf >= 0.5
|
||||
|
||||
|
||||
def test_autolabel_ambiguous_is_none() -> None:
|
||||
from data_only_viz.training.autolabel import autolabel_window
|
||||
|
||||
base = _static_standing(WINDOW_LEN)
|
||||
for t, f in enumerate(base):
|
||||
f[20, 0] += 0.01 * np.sin(t)
|
||||
label, _conf = autolabel_window(base)
|
||||
assert label in ("debout", None)
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Tests for dataset jsonl IO + sliding windows + split."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
|
||||
def _make_session_jsonl(path: Path, n_frames: int = 64) -> None:
|
||||
rng = np.random.default_rng(0)
|
||||
with path.open("w") as f:
|
||||
for t in range(n_frames):
|
||||
row = {"ts": t / 30.0,
|
||||
"session": "sess01",
|
||||
"pid": 1,
|
||||
"j3d": rng.normal(size=(32, 3)).tolist()}
|
||||
f.write(json.dumps(row) + "\n")
|
||||
|
||||
|
||||
def test_load_frames_jsonl(tmp_path: Path) -> None:
|
||||
from data_only_viz.training.dataset import load_frames_jsonl
|
||||
p = tmp_path / "raw.jsonl"
|
||||
_make_session_jsonl(p)
|
||||
frames = load_frames_jsonl(p)
|
||||
assert len(frames) == 64
|
||||
assert frames[0].j3d.shape == (32, 3)
|
||||
assert frames[0].pid == 1
|
||||
assert frames[0].session == "sess01"
|
||||
|
||||
|
||||
def test_sliding_windows(tmp_path: Path) -> None:
|
||||
from data_only_viz.training.dataset import (
|
||||
load_frames_jsonl,
|
||||
sliding_windows,
|
||||
)
|
||||
p = tmp_path / "raw.jsonl"
|
||||
_make_session_jsonl(p, n_frames=64)
|
||||
frames = load_frames_jsonl(p)
|
||||
windows = list(sliding_windows(frames, window_len=16, stride=4))
|
||||
assert len(windows) == 13
|
||||
assert windows[0].j3d_stack.shape == (16, 32, 3)
|
||||
assert windows[0].session == "sess01"
|
||||
|
||||
|
||||
def test_write_and_load_dataset_jsonl(tmp_path: Path) -> None:
|
||||
from data_only_viz.training.dataset import (
|
||||
DatasetRow,
|
||||
load_dataset_jsonl,
|
||||
write_dataset_jsonl,
|
||||
)
|
||||
rng = np.random.default_rng(0)
|
||||
rows = [
|
||||
DatasetRow(
|
||||
window_id=f"sess01_pid1_w{i:04d}",
|
||||
label="debout" if i % 2 == 0 else "danse",
|
||||
j3d_stack=rng.normal(size=(16, 32, 3)).astype(np.float32),
|
||||
session="sess01",
|
||||
pid_local=1,
|
||||
auto_label_confidence=0.8,
|
||||
manually_validated=False,
|
||||
)
|
||||
for i in range(5)
|
||||
]
|
||||
out = tmp_path / "ds.jsonl"
|
||||
write_dataset_jsonl(rows, out)
|
||||
loaded = load_dataset_jsonl(out)
|
||||
assert len(loaded) == 5
|
||||
assert loaded[0].label == "debout"
|
||||
assert loaded[0].j3d_stack.shape == (16, 32, 3)
|
||||
assert np.allclose(loaded[0].j3d_stack, rows[0].j3d_stack, atol=1e-6)
|
||||
|
||||
|
||||
def test_write_and_load_dataset_jsonl_with_hands_kp(tmp_path: Path) -> None:
|
||||
from data_only_viz.training.dataset import (
|
||||
DatasetRow,
|
||||
load_dataset_jsonl,
|
||||
write_dataset_jsonl,
|
||||
)
|
||||
rng = np.random.default_rng(1)
|
||||
hands_kp = rng.normal(size=(16, 42, 3)).astype(np.float32)
|
||||
row = DatasetRow(
|
||||
window_id="sess01_pid1_w0000",
|
||||
label="danse",
|
||||
j3d_stack=rng.normal(size=(16, 32, 3)).astype(np.float32),
|
||||
session="sess01",
|
||||
pid_local=1,
|
||||
auto_label_confidence=0.9,
|
||||
manually_validated=True,
|
||||
hands_kp_stack=hands_kp,
|
||||
)
|
||||
out = tmp_path / "with_hands.jsonl"
|
||||
write_dataset_jsonl([row], out)
|
||||
loaded = load_dataset_jsonl(out)
|
||||
assert loaded[0].hands_kp_stack is not None
|
||||
assert loaded[0].hands_kp_stack.shape == (16, 42, 3)
|
||||
assert np.allclose(loaded[0].hands_kp_stack, hands_kp, atol=1e-6)
|
||||
|
||||
|
||||
def test_load_dataset_jsonl_without_hands_kp_is_ok(tmp_path: Path) -> None:
|
||||
"""Legacy v2 rows without hands_kp field should load with hands_kp_stack=None."""
|
||||
import json
|
||||
from data_only_viz.training.dataset import load_dataset_jsonl
|
||||
rng = np.random.default_rng(2)
|
||||
row = {
|
||||
"window_id": "sess01_pid1_w0000",
|
||||
"label": "debout",
|
||||
"j3d": rng.normal(size=(16, 32, 3)).tolist(),
|
||||
"session": "sess01",
|
||||
"pid_local": 1,
|
||||
"auto_label_confidence": 0.8,
|
||||
"manually_validated": False,
|
||||
}
|
||||
out = tmp_path / "legacy.jsonl"
|
||||
out.write_text(json.dumps(row) + "\n")
|
||||
loaded = load_dataset_jsonl(out)
|
||||
assert len(loaded) == 1
|
||||
assert loaded[0].hands_kp_stack is None
|
||||
|
||||
|
||||
def test_split_by_session(tmp_path: Path) -> None:
|
||||
from data_only_viz.training.dataset import DatasetRow, split_by_session
|
||||
rng = np.random.default_rng(0)
|
||||
rows = []
|
||||
for sess in ("s01", "s02", "s03", "s04", "s05", "s06", "s07"):
|
||||
rows.append(DatasetRow(
|
||||
window_id=f"{sess}_w0", label="debout",
|
||||
j3d_stack=rng.normal(size=(16, 32, 3)).astype(np.float32),
|
||||
session=sess, pid_local=1, auto_label_confidence=0.7,
|
||||
manually_validated=False,
|
||||
))
|
||||
train, val, test = split_by_session(rows, ratios=(0.7, 0.15, 0.15), seed=0)
|
||||
all_sessions = {r.session for r in train + val + test}
|
||||
assert all_sessions == {"s01","s02","s03","s04","s05","s06","s07"}
|
||||
train_s = {r.session for r in train}
|
||||
val_s = {r.session for r in val}
|
||||
test_s = {r.session for r in test}
|
||||
assert train_s.isdisjoint(val_s)
|
||||
assert train_s.isdisjoint(test_s)
|
||||
assert val_s.isdisjoint(test_s)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""DETRPose model size flows from CLI through to the worker."""
|
||||
|
||||
import pytest
|
||||
|
||||
from data_only_viz.detrpose import DETRPoseWorker, DEFAULT_MODEL_SIZE
|
||||
|
||||
|
||||
def test_default_model_size_unchanged():
|
||||
assert DEFAULT_MODEL_SIZE == "n"
|
||||
|
||||
|
||||
def test_worker_accepts_model_size_kwarg():
|
||||
# Should not raise; we don't load the model (that requires the extra)
|
||||
worker = DETRPoseWorker.__new__(DETRPoseWorker)
|
||||
worker.model_size = None
|
||||
worker._configure_model_size("s")
|
||||
assert worker.model_size == "s"
|
||||
|
||||
|
||||
def test_worker_rejects_invalid_model_size():
|
||||
worker = DETRPoseWorker.__new__(DETRPoseWorker)
|
||||
worker.model_size = None
|
||||
with pytest.raises(ValueError):
|
||||
worker._configure_model_size("xxxl")
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Tests for the DINOv2 reid backend.
|
||||
|
||||
These tests are skipped automatically if the .mlpackage is not present
|
||||
(`scripts/convert_dinov2.py` was never run) or pyobjc is unavailable.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from data_only_viz.dino_reid import DEFAULT_MLPACKAGE, EMBED_DIM, DinoReid
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not DEFAULT_MLPACKAGE.exists(),
|
||||
reason=f"DINOv2 mlpackage missing at {DEFAULT_MLPACKAGE}; "
|
||||
"run scripts/convert_dinov2.py first",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def reid() -> DinoReid:
|
||||
return DinoReid()
|
||||
|
||||
|
||||
def test_is_available() -> None:
|
||||
assert DinoReid.is_available() is True
|
||||
|
||||
|
||||
def test_load(reid: DinoReid) -> None:
|
||||
assert reid is not None
|
||||
assert reid._out_name
|
||||
|
||||
|
||||
def test_embed_random_crops_different(reid: DinoReid) -> None:
|
||||
# Two crops with very different visual content. DINOv2 CLS tokens
|
||||
# for two iid noise patches are surprisingly close (~0.98), so we
|
||||
# build crops that are visually distinct: one is mostly red, the
|
||||
# other is mostly green with a striped pattern.
|
||||
a = np.zeros((224, 224, 3), dtype=np.uint8)
|
||||
a[..., 0] = 220 # red
|
||||
a[40:80, 40:180] = (240, 30, 30)
|
||||
b = np.zeros((224, 224, 3), dtype=np.uint8)
|
||||
b[..., 1] = 200 # green
|
||||
for i in range(0, 224, 16):
|
||||
b[i:i + 8] = (10, 30, 220) # blue stripes
|
||||
embs = reid.embed_crops([a, b])
|
||||
assert embs.shape == (2, EMBED_DIM)
|
||||
norms = np.linalg.norm(embs, axis=1)
|
||||
assert np.allclose(norms, 1.0, atol=1e-3)
|
||||
cos = float(np.dot(embs[0], embs[1]))
|
||||
assert cos < 0.95, f"distinct crops too similar: cos={cos:.3f}"
|
||||
|
||||
|
||||
def test_embed_identical_crops_same(reid: DinoReid) -> None:
|
||||
rng = np.random.default_rng(7)
|
||||
a = rng.integers(0, 255, size=(224, 224, 3), dtype=np.uint8)
|
||||
embs = reid.embed_crops([a, a.copy()])
|
||||
assert embs.shape == (2, EMBED_DIM)
|
||||
cos = float(np.dot(embs[0], embs[1]))
|
||||
assert cos > 0.999, f"identical crops cos={cos:.4f} (expected ~1.0)"
|
||||
|
||||
|
||||
def test_latency_batch4(reid: DinoReid) -> None:
|
||||
rng = np.random.default_rng(0)
|
||||
crops = [rng.integers(0, 255, size=(180, 90, 3), dtype=np.uint8)
|
||||
for _ in range(4)]
|
||||
# warmup
|
||||
reid.embed_crops(crops)
|
||||
t0 = time.perf_counter()
|
||||
reid.embed_crops(crops)
|
||||
dt_ms = (time.perf_counter() - t0) * 1e3
|
||||
# Spec target: < 30 ms for batch=4 on M5.
|
||||
assert dt_ms < 80.0, f"batch=4 too slow: {dt_ms:.1f} ms"
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Sanity tests for MediaPipe offline extractor (no MediaPipe runtime -- we
|
||||
mock the landmarker and feed synthetic landmarks)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
|
||||
def test_build_j3d32_combines_body_and_fingertips() -> None:
|
||||
from data_only_viz.scripts.extract_mediapipe_offline import _build_j3d32
|
||||
from data_only_viz.action_head import J3D_JOINTS
|
||||
body3d = np.linspace(0, 1, 33 * 3).reshape(33, 3).astype(np.float32)
|
||||
hands_kp42 = np.linspace(2, 3, 42 * 3).reshape(42, 3).astype(np.float32)
|
||||
j3d = _build_j3d32(body3d, hands_kp42)
|
||||
assert j3d is not None
|
||||
assert j3d.shape == (J3D_JOINTS, 3)
|
||||
# The body22 portion comes from body3d via MEDIAPIPE_TO_22.
|
||||
# The fingertip portion (indices 22..31) comes from hands_kp at idx 4,8,12,16,20.
|
||||
assert np.allclose(j3d[22], hands_kp42[4])
|
||||
assert np.allclose(j3d[26], hands_kp42[20])
|
||||
assert np.allclose(j3d[27], hands_kp42[21 + 4])
|
||||
assert np.allclose(j3d[31], hands_kp42[21 + 20])
|
||||
|
||||
|
||||
def test_build_j3d32_returns_none_when_no_body() -> None:
|
||||
from data_only_viz.scripts.extract_mediapipe_offline import _build_j3d32
|
||||
j3d = _build_j3d32(None, np.zeros((42, 3), dtype=np.float32))
|
||||
assert j3d is None
|
||||
|
||||
|
||||
def test_hands_kp42_combines_left_right_sides() -> None:
|
||||
from data_only_viz.scripts.extract_mediapipe_offline import _hands_kp42
|
||||
left = np.linspace(0, 1, 21 * 3).reshape(21, 3).astype(np.float32)
|
||||
right = np.linspace(2, 3, 21 * 3).reshape(21, 3).astype(np.float32)
|
||||
out = _hands_kp42(left, right)
|
||||
assert out.shape == (42, 3)
|
||||
assert np.allclose(out[:21], left)
|
||||
assert np.allclose(out[21:], right)
|
||||
|
||||
|
||||
def test_hands_kp42_zero_pads_when_missing() -> None:
|
||||
from data_only_viz.scripts.extract_mediapipe_offline import _hands_kp42
|
||||
left = np.ones((21, 3), dtype=np.float32)
|
||||
out = _hands_kp42(left, None)
|
||||
assert np.allclose(out[:21], left)
|
||||
assert np.allclose(out[21:], 0.0)
|
||||
|
||||
|
||||
def test_mouth_open_from_face_lips() -> None:
|
||||
from data_only_viz.scripts.extract_mediapipe_offline import _mouth_open
|
||||
# MediaPipe FaceMesh has 478 landmarks. Build a sparse array : zero
|
||||
# everywhere except idx 13 (upper inner) and idx 14 (lower inner),
|
||||
# 1 metre apart on the y axis.
|
||||
face = np.zeros((478, 3), dtype=np.float32)
|
||||
face[13] = [0.0, 1.0, 0.0]
|
||||
face[14] = [0.0, 0.0, 0.0]
|
||||
assert abs(_mouth_open(face) - 1.0) < 1e-6
|
||||
|
||||
|
||||
def test_mouth_open_returns_zero_on_empty_face() -> None:
|
||||
from data_only_viz.scripts.extract_mediapipe_offline import _mouth_open
|
||||
assert _mouth_open(None) == 0.0
|
||||
assert _mouth_open(np.zeros((10, 3), dtype=np.float32)) == 0.0
|
||||
|
||||
|
||||
def test_lmk_list_to_array_round_trip() -> None:
|
||||
from data_only_viz.scripts.extract_mediapipe_offline import _lmk_list_to_array
|
||||
class _Lmk:
|
||||
def __init__(self, x: float, y: float, z: float) -> None:
|
||||
self.x = x; self.y = y; self.z = z
|
||||
lmks = [_Lmk(i, 2 * i, 3 * i) for i in range(5)]
|
||||
arr = _lmk_list_to_array(lmks)
|
||||
assert arr is not None
|
||||
assert arr.shape == (5, 3)
|
||||
assert np.allclose(arr[2], [2.0, 4.0, 6.0])
|
||||
|
||||
|
||||
def test_lmk_list_to_array_none_input() -> None:
|
||||
from data_only_viz.scripts.extract_mediapipe_offline import _lmk_list_to_array
|
||||
assert _lmk_list_to_array(None) is None
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Tests for FaceFilterChain, HandFilterChain, and multi.py discrimination."""
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from data_only_viz.pose_filter import (
|
||||
FaceFilterChain,
|
||||
HandFilterChain,
|
||||
PoseFilterChain,
|
||||
)
|
||||
from data_only_viz.state import Kp3D, PoseKp
|
||||
|
||||
|
||||
def _jitter_face(n_pts: int, base_x: float, base_y: float,
|
||||
amp: float, rng: random.Random) -> list[PoseKp]:
|
||||
return [
|
||||
PoseKp(
|
||||
x=base_x + rng.uniform(-amp, amp),
|
||||
y=base_y + rng.uniform(-amp, amp),
|
||||
z=rng.uniform(-amp, amp),
|
||||
c=1.0,
|
||||
)
|
||||
for _ in range(n_pts)
|
||||
]
|
||||
|
||||
|
||||
def test_face_filter_reduces_jitter() -> None:
|
||||
chain = FaceFilterChain()
|
||||
rng = random.Random(42)
|
||||
n_pts = 68
|
||||
base_x, base_y = 0.5, 0.5
|
||||
amp = 0.01
|
||||
outputs: list[list[PoseKp]] = []
|
||||
t = 0.0
|
||||
for k in range(8):
|
||||
t += 1.0 / 30.0
|
||||
faces = [_jitter_face(n_pts, base_x, base_y, amp, rng)]
|
||||
out = chain.apply(faces, [0], t)
|
||||
outputs.append(out[0])
|
||||
# Compute variance on x of joint 0 across the last 5 frames.
|
||||
last = outputs[-5:]
|
||||
xs = [f[0].x for f in last]
|
||||
mean = sum(xs) / len(xs)
|
||||
var = sum((v - mean) ** 2 for v in xs) / len(xs)
|
||||
assert var < 0.005, f"face filter variance too high: {var}"
|
||||
|
||||
|
||||
def test_hand_filter_left_right_independent() -> None:
|
||||
chain = HandFilterChain()
|
||||
rng = random.Random(7)
|
||||
n_pts = 21
|
||||
t = 0.0
|
||||
last_l: list[PoseKp] = []
|
||||
last_r: list[PoseKp] = []
|
||||
for k in range(6):
|
||||
t += 1.0 / 30.0
|
||||
left_hand = _jitter_face(n_pts, 0.2, 0.5, 0.008, rng)
|
||||
right_hand = _jitter_face(n_pts, 0.8, 0.5, 0.008, rng)
|
||||
out = chain.apply([left_hand, right_hand], [0, 0],
|
||||
["Left", "Right"], t)
|
||||
last_l, last_r = out[0], out[1]
|
||||
# Left and right hands keep distinct positions despite same pid.
|
||||
assert abs(last_l[0].x - last_r[0].x) > 0.4
|
||||
# Filter reduced jitter on each side.
|
||||
assert 0.1 < last_l[0].x < 0.35
|
||||
assert 0.65 < last_r[0].x < 0.9
|
||||
|
||||
|
||||
def test_hand_filter_chain_wrapper_smoke() -> None:
|
||||
chain = PoseFilterChain()
|
||||
rng = random.Random(0)
|
||||
hands = [_jitter_face(21, 0.5, 0.5, 0.01, rng) for _ in range(2)]
|
||||
out = chain.apply_hand(hands, [0, 1], ["Left", "Right"], t_now=0.1)
|
||||
assert len(out) == 2
|
||||
assert len(out[0]) == 21
|
||||
|
||||
|
||||
def test_face_filter_disabled_passthrough() -> None:
|
||||
chain = FaceFilterChain(enabled_stages=())
|
||||
faces = [[PoseKp(x=0.5, y=0.5, z=0.0, c=1.0) for _ in range(68)]]
|
||||
out = chain.apply(faces, [0], t_now=0.0)
|
||||
assert out[0][0].x == 0.5
|
||||
|
||||
|
||||
def test_face_hand_latency_under_5ms() -> None:
|
||||
"""Full chain (body 33 + face 68 + hand 21x2) < 5 ms per frame."""
|
||||
body_chain = PoseFilterChain(
|
||||
enabled_stages=("median", "kalman", "lookahead", "ik"))
|
||||
face_chain = FaceFilterChain()
|
||||
hand_chain = HandFilterChain()
|
||||
rng = random.Random(0)
|
||||
body = [Kp3D(x=i * 0.01, y=i * 0.02, z=i * 0.03, c=1.0)
|
||||
for i in range(33)]
|
||||
face = _jitter_face(68, 0.5, 0.5, 0.01, rng)
|
||||
hand_l = _jitter_face(21, 0.2, 0.5, 0.01, rng)
|
||||
hand_r = _jitter_face(21, 0.8, 0.5, 0.01, rng)
|
||||
# Warm-up
|
||||
for k in range(5):
|
||||
t = k * 0.033
|
||||
body_chain.apply([body], [0], t)
|
||||
face_chain.apply([face], [0], t)
|
||||
hand_chain.apply([hand_l, hand_r], [0, 0], ["Left", "Right"], t)
|
||||
# Measure
|
||||
durs: list[float] = []
|
||||
for k in range(30):
|
||||
t = (k + 5) * 0.033
|
||||
t0 = time.perf_counter()
|
||||
body_chain.apply([body], [0], t)
|
||||
face_chain.apply([face], [0], t)
|
||||
hand_chain.apply([hand_l, hand_r], [0, 0], ["Left", "Right"], t)
|
||||
durs.append((time.perf_counter() - t0) * 1000.0)
|
||||
avg = sum(durs) / len(durs)
|
||||
# CI margin : actual M-class target is < 5 ms ; allow 25 ms in tests.
|
||||
assert avg < 25.0, f"chain too slow: {avg:.2f} ms"
|
||||
|
||||
|
||||
# ----------------------- multi.py discrimination ---------------------------
|
||||
|
||||
|
||||
def _make_body(n_visible: int) -> list[PoseKp]:
|
||||
"""Make a 33-joint body with `n_visible` high-conf joints, rest low."""
|
||||
out: list[PoseKp] = []
|
||||
for i in range(33):
|
||||
c = 1.0 if i < n_visible else 0.05
|
||||
# Spread across both x and y so the bbox has non-zero area.
|
||||
out.append(PoseKp(x=0.1 + i * 0.01, y=0.2 + i * 0.005, z=0.0, c=c))
|
||||
return out
|
||||
|
||||
|
||||
def _make_body3d(n: int = 33) -> list[Kp3D]:
|
||||
return [Kp3D(x=0.0, y=0.0, z=0.0, c=1.0) for _ in range(n)]
|
||||
|
||||
|
||||
def _instantiate_worker():
|
||||
"""Build a MultiWorker without starting the thread (skip if cv2 missing)."""
|
||||
pytest.importorskip("cv2", reason="opencv not installed")
|
||||
from data_only_viz.multi import MultiWorker
|
||||
from data_only_viz.state import State
|
||||
return MultiWorker(state=State(), camera_index=-1)
|
||||
|
||||
|
||||
def test_ghost_rejection_drops_low_visibility_body() -> None:
|
||||
w = _instantiate_worker()
|
||||
bodies = [_make_body(n_visible=5), _make_body(n_visible=25)]
|
||||
b3d = [_make_body3d(), _make_body3d()]
|
||||
ids = [0, 1]
|
||||
new_bodies, new_b3d, new_ids = w._reject_ghosts_and_nms(bodies, b3d, ids)
|
||||
assert len(new_bodies) == 1
|
||||
assert len(new_b3d) == 1
|
||||
assert new_ids == [1]
|
||||
assert w._n_ghost_dropped == 1
|
||||
|
||||
|
||||
def test_nms_keeps_best_score() -> None:
|
||||
w = _instantiate_worker()
|
||||
# Two heavily overlapping bodies, second has higher mean confidence.
|
||||
b1 = _make_body(n_visible=20)
|
||||
b2 = _make_body(n_visible=33)
|
||||
new_bodies, _, new_ids = w._reject_ghosts_and_nms([b1, b2], [], [0, 1])
|
||||
# IoU of identical bbox => one dropped, the higher-score one kept.
|
||||
assert len(new_bodies) == 1
|
||||
assert new_ids == [1]
|
||||
|
||||
|
||||
def test_pid_persistence_through_short_absence() -> None:
|
||||
w = _instantiate_worker()
|
||||
body = _make_body(n_visible=30)
|
||||
# Frame 1..30 : pid 0 present.
|
||||
for _ in range(30):
|
||||
new_ids = w._apply_pid_hysteresis([body], [0])
|
||||
assert new_ids == [0]
|
||||
# Frames 31..35 : pid 0 absent (no detection).
|
||||
for _ in range(5):
|
||||
w._apply_pid_hysteresis([], [])
|
||||
# Frame 36 : a NEW pid 9 appears at the same bbox -> should be remapped.
|
||||
new_ids = w._apply_pid_hysteresis([body], [9])
|
||||
assert new_ids == [0], f"expected hysteresis remap to 0, got {new_ids}"
|
||||
|
||||
|
||||
def test_drop_low_visibility_face() -> None:
|
||||
w = _instantiate_worker()
|
||||
# 30 valid (non-zero) + 38 zeros.
|
||||
face_bad = [
|
||||
PoseKp(x=(0.1 if i < 30 else 0.0),
|
||||
y=(0.1 if i < 30 else 0.0), z=0.0, c=1.0)
|
||||
for i in range(68)
|
||||
]
|
||||
face_ok = [
|
||||
PoseKp(x=0.1 + i * 0.001, y=0.2, z=0.0, c=1.0)
|
||||
for i in range(68)
|
||||
]
|
||||
kept, ids = w._drop_low_visibility(
|
||||
[face_bad, face_ok], [0, 1], min_visible=50, which="face")
|
||||
assert len(kept) == 1
|
||||
assert ids == [1]
|
||||
assert w._n_face_dropped == 1
|
||||
|
||||
|
||||
def test_drop_low_visibility_hand() -> None:
|
||||
w = _instantiate_worker()
|
||||
hand_bad = [PoseKp(x=0.0, y=0.0, z=0.0, c=1.0) for _ in range(21)]
|
||||
# Only 10 visible (others are zero) -> drop.
|
||||
for i in range(10):
|
||||
hand_bad[i] = PoseKp(x=0.5, y=0.5, z=0.0, c=1.0)
|
||||
hand_ok = [PoseKp(x=0.1 + i * 0.01, y=0.2, z=0.0, c=1.0)
|
||||
for i in range(21)]
|
||||
kept, ids = w._drop_low_visibility(
|
||||
[hand_bad, hand_ok], [0, 1], min_visible=15, which="hand")
|
||||
assert len(kept) == 1
|
||||
assert ids == [1]
|
||||
assert w._n_hand_dropped == 1
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user