Compare commits
595 Commits
v0.2.0-rc1
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
 L'électron rare
|
8cfa0694a7 | ||
|
L'électron rare
|
1fe186bdb8 | ||
|
L'électron rare
|
d058858b23 | ||
|
L'électron rare
|
144d55a7fa | ||
|
L'électron rare
|
0bc15e7000 | ||
|
L'électron rare
|
0e5c4df3d7 | ||
|
L'électron rare
|
e71ae814f7 | ||
|
L'électron rare
|
ba5db3237e | ||
|
L'électron rare
|
c8318cecc2 | ||
|
L'électron rare
|
876afc0cf9 | ||
|
L'électron rare
|
a6556e809a | ||
|
L'électron rare
|
51a3412158 | ||
|
L'électron rare
|
69c560dfe6 | ||
|
L'électron rare
|
7b039fbb78 | ||
|
L'électron rare
|
d970f4cbe9 | ||
|
L'électron rare
|
8d6d42ed3c | ||
|
L'électron rare
|
fbc6d4efa3 | ||
|
L'électron rare
|
1302ffd87a | ||
|
L'électron rare
|
720d566294 | ||
|
L'électron rare
|
1d59d68777 | ||
|
L'électron rare
|
8d93163766 | ||
|
L'électron rare
|
6d43120c0a | ||
|
L'électron rare
|
1c07211dd8 | ||
|
L'électron rare
|
6fe98c6b00 | ||
|
L'électron rare
|
32a722e281 | ||
|
L'électron rare
|
eb3803680b | ||
|
L'électron rare
|
bcf7ac180e | ||
|
L'électron rare
|
a569122f14 | ||
|
L'électron rare
|
feb32938fb | ||
|
L'électron rare
|
c878056733 | ||
|
L'électron rare
|
3199aa326d | ||
|
L'électron rare
|
c66c4271fa | ||
|
L'électron rare
|
ef1d557eff | ||
|
L'électron rare
|
3ffd81afc7 | ||
|
L'électron rare
|
65f13b4621 | ||
|
L'électron rare
|
b85a9c93a9 | ||
|
L'électron rare
|
651fd86d37 | ||
|
L'électron rare
|
d337ed9b29 | ||
|
L'électron rare
|
0126fc97a8 | ||
|
L'électron rare
|
4ea0bb2c2a | ||
|
L'électron rare
|
ba4a1bb103 | ||
|
L'électron rare
|
05d0fc1df1 | ||
|
L'électron rare
|
caa41fd7f1 | ||
|
L'électron rare
|
e10361c56d | ||
|
L'électron rare
|
a7a1563ef8 | ||
|
L'électron rare
|
5769af3d26 | ||
|
L'électron rare
|
f3ca0ac100 | ||
|
L'électron rare
|
4e33a03e1b | ||
|
L'électron rare
|
8b08282aa5 | ||
|
L'électron rare
|
91b3923431 | ||
|
L'électron rare
|
e2a41c0d8a | ||
|
L'électron rare
|
65bf3aad08 | ||
|
L'électron rare
|
c0cde337c9 | ||
|
L'électron rare
|
e7ce016688 | ||
|
L'électron rare
|
5d425b1fbd | ||
|
L'électron rare
|
32cdfa0d3e | ||
|
L'électron rare
|
e20a4d130a | ||
|
L'électron rare
|
366d916e9d | ||
|
L'électron rare
|
9ee7990dcd | ||
|
L'électron rare
|
8bc7acbd14 | ||
|
L'électron rare
|
0a4f51ebfb | ||
|
L'électron rare
|
039abaa4a8 | ||
|
L'électron rare
|
5bfc63f86d | ||
|
L'électron rare
|
ac6768d8a8 | ||
|
L'électron rare
|
287c4977a9 | ||
|
L'électron rare
|
b8ae1a668c | ||
|
L'électron rare
|
48b04084fa | ||
|
L'électron rare
|
06b868e377 | ||
|
L'électron rare
|
0a2865e136 | ||
|
L'électron rare
|
3a3e6b113f | ||
|
L'électron rare
|
ef719e76f3 | ||
|
L'électron rare
|
60640a881e | ||
|
L'électron rare
|
29c66c7151 | ||
|
L'électron rare
|
1f19a5db96 | ||
|
L'électron rare
|
ea42ba4bb3 | ||
|
L'électron rare
|
021c53f078 | ||
|
L'électron rare
|
50bc6f5d71 | ||
|
L'électron rare
|
1510a37059 | ||
|
clement
|
095fd51812 | ||
|
clement
|
1491aef30e | ||
|
clement
|
24ce960992 | ||
|
clement
|
08b7c72e49 | ||
|
clement
|
86c071d0c9 | ||
|
clement
|
04eb182fab | ||
|
clement
|
b00b6dfbda | ||
|
clement
|
fbc8b55044 | ||
|
clement
|
67f347e705 | ||
|
clement
|
6f941c52cb | ||
|
clement
|
b2dab64aae | ||
|
clement
|
bb90820988 | ||
|
clement
|
5adb8abdc8 | ||
|
L'électron rare
|
40c21a246c | ||
|
L'électron rare
|
2b7b4a81c6 | ||
|
L'électron rare
|
ef99c32cac | ||
|
L'électron rare
|
27d5ce9d98 | ||
|
L'électron rare
|
33de64f599 | ||
|
L'électron rare
|
5a9a6338a0 | ||
|
L'électron rare
|
c49de5c500 | ||
|
L'électron rare
|
660d493ad6 | ||
|
L'électron rare
|
bd9f9fca29 | ||
|
L'électron rare
|
7037ebcd82 | ||
|
L'électron rare
|
02f7ad42b2 | ||
|
L'électron rare
|
16cd6a3655 | ||
|
L'électron rare
|
4c0794a3db | ||
|
L'électron rare
|
9111da4a5c | ||
|
L'électron rare
|
65c06ed4c6 | ||
|
L'électron rare
|
24078111b4 | ||
|
L'électron rare
|
4cfd2f0a98 | ||
|
L'électron rare
|
f9f62e2951 | ||
|
L'électron rare
|
38dca5642e | ||
|
L'électron rare
|
2ced4e728f | ||
|
L'électron rare
|
0daa6d6bcd | ||
|
L'électron rare
|
b3ec348378 | ||
|
L'électron rare
|
2902f5c7cd | ||
|
L'électron rare
|
4bccbc1580 | ||
|
L'électron rare
|
50386efe8f | ||
|
L'électron rare
|
d31c6c52d0 | ||
|
L'électron rare
|
65fe76d5f0 | ||
|
L'électron rare
|
140953f3cb | ||
|
L'électron rare
|
9590615069 | ||
|
L'électron rare
|
cc485003f6 | ||
|
L'électron rare
|
2994778b02 | ||
|
L'électron rare
|
e04fb88e42 | ||
|
L'électron rare
|
afc220f83f | ||
|
L'électron rare
|
d4306f6252 | ||
|
L'électron rare
|
9dbf0bcbc9 | ||
|
L'électron rare
|
0bd8e46dd7 | ||
|
L'électron rare
|
d483a29d3d | ||
|
L'électron rare
|
64348aa90d | ||
|
L'électron rare
|
8bead14e23 | ||
|
L'électron rare
|
9f53f27317 | ||
|
L'électron rare
|
1d6f65bbb2 | ||
|
L'électron rare
|
cadbced958 | ||
|
L'électron rare
|
8ed0365309 | ||
|
L'électron rare
|
69176fb94d | ||
|
L'électron rare
|
50e9e79a22 | ||
|
L'électron rare
|
0cc64194df | ||
|
L'électron rare
|
b72056ecb3 | ||
|
L'électron rare
|
c0e13a0fa0 | ||
|
L'électron rare
|
2f9f43e664 | ||
|
L'électron rare
|
8dde561c34 | ||
|
L'électron rare
|
15e0b79083 | ||
|
L'électron rare
|
d8d6017443 | ||
|
L'électron rare
|
db8d7d465f | ||
|
L'électron rare
|
635343262e | ||
|
L'électron rare
|
d02a83d6a2 | ||
|
L'électron rare
|
5def521587 | ||
|
L'électron rare
|
6de2627f7e | ||
|
L'électron rare
|
fc268405d2 | ||
|
L'électron rare
|
ce00b90f43 | ||
|
L'électron rare
|
f635240df9 | ||
|
L'électron rare
|
e0c4405044 | ||
|
L'électron rare
|
68fa3c4e3b | ||
|
L'électron rare
|
af55f8a145 | ||
|
L'électron rare
|
f6b350a35c | ||
|
L'électron rare
|
0e6604cb80 | ||
|
L'électron rare
|
3a5cf80aa7 | ||
|
L'électron rare
|
4c7d960b45 | ||
|
L'électron rare
|
1090f2c223 | ||
|
L'électron rare
|
fc73be4851 | ||
|
L'électron rare
|
38d79f8510 | ||
|
L'électron rare
|
6acc302982 | ||
|
L'électron rare
|
b7fa138172 | ||
|
L'électron rare
|
3d366be50c | ||
|
L'électron rare
|
eb42da0f31 | ||
|
L'électron rare
|
b7d537b10d | ||
|
L'électron rare
|
6bb8e41af1 | ||
|
L'électron rare
|
d8f1cd9e71 | ||
|
L'électron rare
|
6f8903e303 | ||
|
L'électron rare
|
c7ba26100e | ||
|
L'électron rare
|
d0a66a5fd3 | ||
|
L'électron rare
|
39ac5d0ad1 | ||
|
L'électron rare
|
f8c65b91ba | ||
|
L'électron rare
|
9d2a414718 | ||
|
L'électron rare
|
e99fe1f585 | ||
|
L'électron rare
|
2532a9a7e5 | ||
|
L'électron rare
|
47fe6a46af | ||
|
L'électron rare
|
b254d7cf1d | ||
|
L'électron rare
|
8559cba14a | ||
|
L'électron rare
|
93bbfd8960 | ||
|
L'électron rare
|
ee4cd9b92c | ||
|
L'électron rare
|
7bd5a4f5fe | ||
|
L'électron rare
|
cd77189c12 | ||
|
L'électron rare
|
e9ca68102a | ||
|
L'électron rare
|
ae146ac48e | ||
|
L'électron rare
|
0c3d410e98 | ||
|
L'électron rare
|
e53ae8d94d | ||
|
L'électron rare
|
ece67bf1fb | ||
|
L'électron rare
|
09fec3aa36 | ||
|
L'électron rare
|
84de68cb12 | ||
|
L'électron rare
|
8daad4040b | ||
|
L'électron rare
|
5063e5929a | ||
|
L'électron rare
|
8db2c2a7a9 | ||
|
L'électron rare
|
8aa41e2aff | ||
|
L'électron rare
|
d3a2a591ae | ||
|
L'électron rare
|
89ade5524c | ||
|
L'électron rare
|
3ecef87f4b | ||
|
L'électron rare
|
244d5dd944 | ||
|
L'électron rare
|
21d5209047 | ||
|
L'électron rare
|
c82b149741 | ||
|
L'électron rare
|
b954032165 | ||
|
L'électron rare
|
0179e32837 | ||
|
L'électron rare
|
0c8ea8b104 | ||
|
L'électron rare
|
6964831036 | ||
|
L'électron rare
|
adeb98e569 | ||
|
L'électron rare
|
73af4890b1 | ||
|
L'électron rare
|
ab265e4473 | ||
|
L'électron rare
|
fa7797d259 | ||
|
L'électron rare
|
d646f4649d | ||
|
L'électron rare
|
71926dec84 | ||
|
L'électron rare
|
80aa33a19c | ||
|
L'électron rare
|
946b83d9ea | ||
|
L'électron rare
|
cde2882975 | ||
|
L'électron rare
|
c0a360ff58 | ||
|
L'électron rare
|
5aa9b31ec5 | ||
|
L'électron rare
|
1edeaf1e6e | ||
|
L'électron rare
|
e81a70de26 | ||
|
L'électron rare
|
5470ae9ed5 | ||
|
L'électron rare
|
bc661aea5a | ||
|
L'électron rare
|
aec2014b0a | ||
|
L'électron rare
|
8615472589 | ||
|
L'électron rare
|
b0357af715 | ||
|
L'électron rare
|
3bab0d5e42 | ||
|
L'électron rare
|
7b35c19017 | ||
|
L'électron rare
|
3098593dd0 | ||
|
L'électron rare
|
04ff9ceb22 | ||
|
L'électron rare
|
8ebd028039 | ||
|
clement
|
929fd9aaf0 | ||
|
clement
|
25bfdf0cb2 | ||
|
clement
|
f03b7bb3bd | ||
|
L'électron rare
|
2908dc4c28 | ||
|
clement
|
84054714d3 | ||
|
L'électron rare
|
042da7faa0 | ||
|
clement
|
4cb6ce14c0 | ||
|
clement
|
dcb35772ae | ||
|
L'électron rare
|
7b458b8b58 | ||
|
clement
|
df04d8d051 | ||
|
L'électron rare
|
dc766fe896 | ||
|
clement
|
25ce6e0139 | ||
|
clement
|
b18cb7c514 | ||
|
L'électron rare
|
c535367817 | ||
|
clement
|
6cf09c21be | ||
|
clement
|
a87044f698 | ||
|
clement
|
be02a71bde | ||
|
clement
|
74e43d6915 | ||
|
clement
|
8c7a1aee00 | ||
|
clement
|
2d64e8d6aa | ||
|
L'électron rare
|
68e5c5c8a8 | ||
|
clement
|
8e6434cf2a | ||
|
clement
|
25acb50e59 | ||
|
clement
|
d1c9485904 | ||
|
clement
|
74b1ccfa1a | ||
|
clement
|
07268dd8f3 | ||
|
clement
|
89d201b068 | ||
|
clement
|
3d38b1006e | ||
|
clement
|
296dc99435 | ||
|
clement
|
39ac9406a8 | ||
|
clement
|
8f6886f44e | ||
|
clement
|
9f7f8558d4 | ||
|
clement
|
3eb6426f61 | ||
|
clement
|
8a3e7fa5cb | ||
|
clement
|
162eab3d2e | ||
|
clement
|
2a03a40270 | ||
|
clement
|
0edaf7b2ba | ||
|
clement
|
05f16aa886 | ||
|
clement
|
d11addea4a | ||
|
clement
|
532b23735c | ||
|
L'électron rare
|
684ea5d4d3 | ||
|
clement
|
96ea70a645 | ||
|
clement
|
f5527369ca | ||
|
clement
|
b90d0ad802 | ||
|
clement
|
a95db5efd7 | ||
|
clement
|
a17e417fa3 | ||
|
clement
|
51f5bff1bf | ||
|
L'électron rare
|
ce46a27903 | ||
|
clement
|
b154d1cbd4 | ||
|
L'électron rare
|
7675c2ac11 | ||
|
clement
|
daeb1e6c1e | ||
|
clement
|
a6dce8e712 | ||
|
clement
|
1df8b89ac3 | ||
|
clement
|
210a224e68 | ||
|
clement
|
a01d09287b | ||
|
clement
|
8bb1b778ed | ||
|
L'électron rare
|
6a14b3b67d | ||
|
clement
|
fd60783047 | ||
|
L'électron rare
|
29e0df3378 | ||
|
clement
|
96bb1151a7 | ||
|
clement
|
97905fa2e6 | ||
|
clement
|
3ece3c9e8e | ||
|
clement
|
e33796c1b4 | ||
|
clement
|
1b922d446f | ||
|
clement
|
d2636d4a12 | ||
|
clement
|
38f1974455 | ||
|
clement
|
744ba6475a | ||
|
clement
|
528a9398b5 | ||
|
clement
|
2d767e8d98 | ||
|
L'électron rare
|
f9da19f547 | ||
|
clement
|
987df60b10 | ||
|
L'électron rare
|
529d6c5cd9 | ||
|
clement
|
3fb9e0eca0 | ||
|
clement
|
b8098de39e | ||
|
L'électron rare
|
46c88236f6 | ||
|
clement
|
2bfa846631 | ||
|
clement
|
128bc3f1b4 | ||
|
L'électron rare
|
6482baf22c | ||
|
clement
|
cff42d0863 | ||
|
clement
|
d5a1de0938 | ||
|
L'électron rare
|
16adac4bec | ||
|
clement
|
b65d16843e | ||
|
L'électron rare
|
b141127a13 | ||
|
L'électron rare
|
c81f3d8587 | ||
|
L'électron rare
|
2ed077bbf5 | ||
|
clement
|
9c8c271113 | ||
|
clement
|
b80ba66d4c | ||
|
clement
|
6c7cd45ecd | ||
|
clement
|
0069b82875 | ||
|
clement
|
c483977c34 | ||
|
clement
|
f9d64cd7a5 | ||
|
clement
|
df2bceca74 | ||
|
clement
|
a484ab943c | ||
|
clement
|
3ce4c38f1c | ||
|
clement
|
7ca9fd98ff | ||
|
L'électron rare
|
6843e5aafc | ||
|
clement
|
015374af38 | ||
|
L'électron rare
|
2d927107f1 | ||
|
L'électron rare
|
ee7f349359 | ||
|
L'électron rare
|
2250fe347f | ||
|
clement
|
53f9111ed8 | ||
|
clement
|
fe7b25e536 | ||
|
clement
|
8d95f3e455 | ||
|
clement
|
e9022fab7d | ||
|
clement
|
222c087610 | ||
|
clement
|
141ed0e122 | ||
|
L'électron rare
|
2f4a80ce2b | ||
|
L'électron rare
|
495c5680bf | ||
|
clement
|
90ded48a57 | ||
|
clement
|
62957f09d5 | ||
|
L'électron rare
|
7b90acf80e | ||
|
L'électron rare
|
1990fc692f | ||
|
clement
|
17d78127f6 | ||
|
L'électron rare
|
d7c1049661 | ||
|
L'électron rare
|
ab88b0f6df | ||
|
L'électron rare
|
ca21a4216d | ||
|
clement
|
562ac45d13 | ||
|
clement
|
2777243d5c | ||
|
clement
|
77249049c9 | ||
|
clement
|
fab35a5520 | ||
|
clement
|
2aac8f57f6 | ||
|
clement
|
9ad0ddda13 | ||
|
clement
|
bb11311720 | ||
|
L'électron rare
|
6ec864ed7d | ||
|
L'électron rare
|
16f68efe01 | ||
|
clement
|
57abc2ae0f | ||
|
clement
|
463bab7665 | ||
|
clement
|
b8f5cc0b2e | ||
|
L'électron rare
|
b071c31b86 | ||
|
L'électron rare
|
582d955522 | ||
|
L'électron rare
|
714c5988c2 | ||
|
L'électron rare
|
ebe15d0804 | ||
|
L'électron rare
|
0fa6cac2bf | ||
|
L'électron rare
|
91c4c5c599 | ||
|
clement
|
6a286b4f0e | ||
|
L'électron rare
|
6566e7e410 | ||
|
clement
|
154db324cf | ||
|
L'électron rare
|
8530f92c15 | ||
|
L'électron rare
|
2213b4e53d | ||
|
clement
|
217e41c632 | ||
|
L'électron rare
|
67fd253de5 | ||
|
clement
|
c1b567f116 | ||
|
L'électron rare
|
c90cbb103c | ||
|
L'électron rare
|
a74d3fb402 | ||
|
clement
|
5c0007f961 | ||
|
L'électron rare
|
1d62ff4347 | ||
|
L'électron rare
|
73c17d41f7 | ||
|
clement
|
e1cf2af339 | ||
|
clement
|
764bf02dc8 | ||
|
clement
|
4cabf12eac | ||
|
L'électron rare
|
427341b8a1 | ||
|
clement
|
858762fe39 | ||
|
clement
|
6b7cae277e | ||
|
L'électron rare
|
2ad47c705e | ||
|
L'électron rare
|
4414bbf168 | ||
|
clement
|
85379be922 | ||
|
clement
|
211ee2b323 | ||
|
clement
|
2b41516a20 | ||
|
L'électron rare
|
9e32e3afd8 | ||
|
L'électron rare
|
048403e92b | ||
|
clement
|
8ad309de33 | ||
|
clement
|
2e32dec51c | ||
|
L'électron rare
|
1101758719 | ||
|
L'électron rare
|
2d05ac9372 | ||
|
L'électron rare
|
7f44ee527c | ||
|
L'électron rare
|
82ac0181a3 | ||
|
L'électron rare
|
73432a0c96 | ||
|
L'électron rare
|
0f5a9cd098 | ||
|
L'électron rare
|
db2f331bd1 | ||
|
L'électron rare
|
c1b0cf07b6 | ||
|
L'électron rare
|
b145655892 | ||
|
L'électron rare
|
3ec306fdb2 | ||
|
L'électron rare
|
e23b97f048 | ||
|
L'électron rare
|
abfe950c25 | ||
|
L'électron rare
|
aa37b7832c | ||
|
L'électron rare
|
3d33719f2b | ||
|
L'électron rare
|
241b93046b | ||
|
L'électron rare
|
4adc118e2b | ||
|
L'électron rare
|
3b3b1fe51b | ||
|
L'électron rare
|
1a6503cb71 | ||
|
L'électron rare
|
5343913a7a | ||
|
L'électron rare
|
a98957ce46 | ||
|
L'électron rare
|
c7f98c24af | ||
|
L'électron rare
|
a88fc5e643 | ||
|
L'électron rare
|
24613cd809 | ||
|
L'électron rare
|
0d467da088 | ||
|
L'électron rare
|
c3fdb11e65 | ||
|
L'électron rare
|
cdee717d51 | ||
|
L'électron rare
|
031bb16e0f | ||
|
L'électron rare
|
abedc53a4c | ||
|
L'électron rare
|
245edacd4f | ||
|
L'électron rare
|
337f791b7a | ||
|
L'électron rare
|
a9ce3d58f5 | ||
|
L'électron rare
|
1660640d48 | ||
|
L'électron rare
|
114ebbd99e | ||
|
L'électron rare
|
6b6be7bff2 | ||
|
L'électron rare
|
46d912e526 | ||
|
L'électron rare
|
b487038f30 | ||
|
L'électron rare
|
15578ec60a | ||
|
L'électron rare
|
4285d160c3 | ||
|
L'électron rare
|
883ada7d61 | ||
|
L'électron rare
|
1cd8c2e259 | ||
|
L'électron rare
|
bcfb86ae53 | ||
|
L'électron rare
|
d8ea115a1b | ||
|
L'électron rare
|
66a16d263b | ||
|
L'électron rare
|
abccc1c7b3 | ||
|
L'électron rare
|
0a3798ea38 | ||
|
L'électron rare
|
79842c8bc0 | ||
|
L'électron rare
|
0080cc489f | ||
|
L'électron rare
|
0cfaa0451c | ||
|
L'électron rare
|
cc0d5270cb | ||
|
L'électron rare
|
c4e4511262 | ||
|
L'électron rare
|
5dcefb3420 | ||
|
L'électron rare
|
42460abf6f | ||
|
L'électron rare
|
04764b6d6f | ||
|
L'électron rare
|
0757450648 | ||
|
L'électron rare
|
3314b58b69 | ||
|
L'électron rare
|
57e3a200bc | ||
|
L'électron rare
|
8c4d9f4d7b | ||
|
L'électron rare
|
50772f3b9b | ||
|
L'électron rare
|
a79bc14dc3 | ||
|
L'électron rare
|
9e3cfc308a | ||
|
L'électron rare
|
b259f5437f | ||
|
L'électron rare
|
54d07aca27 | ||
|
L'électron rare
|
8d09ae555b | ||
|
L'électron rare
|
8954198648 | ||
|
L'électron rare
|
8c8de924db | ||
|
L'électron rare
|
2ec59571c9 | ||
|
L'électron rare
|
83ea2be4e6 | ||
|
L'électron rare
|
c2c9753c8c | ||
|
L'électron rare
|
2e484ed2ed | ||
|
L'électron rare
|
73f85aa276 | ||
|
L'électron rare
|
082bb19dd5 | ||
|
L'électron rare
|
83dc910624 | ||
|
L'électron rare
|
480aa79ef4 | ||
|
L'électron rare
|
8a4a0e688d | ||
|
L'électron rare
|
32a4ef2232 | ||
|
L'électron rare
|
f504ad4f41 | ||
|
L'électron rare
|
b40991f7a8 | ||
|
L'électron rare
|
48d019cc77 | ||
|
L'électron rare
|
b1e53da2bb | ||
|
L'électron rare
|
f9d5614d4d | ||
|
L'électron rare
|
0475ea0340 | ||
|
L'électron rare
|
28078dbeba | ||
|
L'électron rare
|
10873707e4 | ||
|
L'électron rare
|
131f6ed5a8 | ||
|
L'électron rare
|
8afc44b2ca | ||
|
L'électron rare
|
dac96481f2 | ||
|
L'électron rare
|
598f00ea31 | ||
|
L'électron rare
|
5629a7e859 | ||
|
L'électron rare
|
ca25732265 | ||
|
L'électron rare
|
5dfbfc7ae9 | ||
|
L'électron rare
|
e7b107a96e | ||
|
L'électron rare
|
b918eeea16 | ||
|
L'électron rare
|
01b0c56ea2 | ||
|
L'électron rare
|
b4b8139902 | ||
|
L'électron rare
|
242c8ec751 | ||
|
L'électron rare
|
1e9f711278 | ||
|
L'électron rare
|
c5d89e740c | ||
|
L'électron rare
|
cf8d24d629 | ||
|
L'électron rare
|
b01a35326b | ||
|
L'électron rare
|
5ed6aeb388 | ||
|
L'électron rare
|
a8c803403c | ||
|
L'électron rare
|
f106b35fa3 | ||
|
L'électron rare
|
fe69f99ba2 | ||
|
L'électron rare
|
cf6eb9c47e | ||
|
L'électron rare
|
ff2ef5dfb8 | ||
|
L'électron rare
|
886b90b84d | ||
|
L'électron rare
|
9ab238bd2d | ||
|
L'électron rare
|
2a3fdb1d38 | ||
|
L'électron rare
|
088b233a48 | ||
|
L'électron rare
|
de2748accb | ||
|
L'électron rare
|
ec2c387091 | ||
|
L'électron rare
|
7a4f14056a | ||
|
L'électron rare
|
71d0a47723 | ||
|
L'électron rare
|
52b902d3e3 | ||
|
L'électron rare
|
d7c8a3e687 | ||
|
L'électron rare
|
8657805eaf | ||
|
L'électron rare
|
c5941b9625 | ||
|
L'électron rare
|
df2b068c2b | ||
 electron
|
1b88da7053 | ||
|
Clément SAILLANT
|
5c61112826 | ||
|
L'électron rare
|
68a56057c4 | ||
|
L'électron rare
|
986d03c09d | ||
|
L'électron rare
|
85589f2627 | ||
|
L'électron rare
|
a22027edb7 | ||
|
L'électron rare
|
a46cbd471c | ||
|
L'électron rare
|
0ee7ddddcd | ||
|
L'électron rare
|
3b5f29e5c7 | ||
|
L'électron rare
|
9e1482e29d | ||
|
L'électron rare
|
ea9c96f716 | ||
|
L'électron rare
|
8cf3d90073 | ||
|
L'électron rare
|
6dab2bcf72 | ||
|
L'électron rare
|
6aea7e0230 | ||
|
L'électron rare
|
819c9688e7 | ||
|
L'électron rare
|
177214d913 | ||
|
L'électron rare
|
76985a2ae9 | ||
|
L'électron rare
|
010b5caaa9 | ||
|
L'électron rare
|
c0834ed7ab | ||
|
L'électron rare
|
87982efb0b | ||
|
L'électron rare
|
ddb4ee3e9d | ||
|
L'électron rare
|
db33e74c15 | ||
|
L'électron rare
|
d8aa727887 | ||
|
L'électron rare
|
950cb29628 | ||
|
L'électron rare
|
360a0745de | ||
|
L'électron rare
|
555efa1b4b | ||
|
L'électron rare
|
914e52deaf | ||
|
L'électron rare
|
7fc4f41f4e | ||
|
L'électron rare
|
e33375f7ac | ||
|
L'électron rare
|
af66ecaecd | ||
|
L'électron rare
|
73f8f75771 | ||
|
L'électron rare
|
8099b79ec8 | ||
|
L'électron rare
|
e06f53b11e | ||
|
L'électron rare
|
b6a85e24fa | ||
|
L'électron rare
|
3ec973d2ce | ||
|
L'électron rare
|
8c24db8e03 | ||
|
L'électron rare
|
131c93f885 | ||
|
L'électron rare
|
94124f974e | ||
|
L'électron rare
|
30fa7276c8 | ||
|
L'électron rare
|
a8be68adff | ||
|
L'électron rare
|
35b18f6ca5 | ||
|
L'électron rare
|
1af056dbf1 | ||
|
L'électron rare
|
9b82a90579 | ||
|
L'électron rare
|
39457dde41 | ||
|
L'électron rare
|
eeb5726103 | ||
|
L'électron rare
|
4cca03db9f | ||
|
L'électron rare
|
fddb31e3ee | ||
|
L'électron rare
|
5e58c0c4fe | ||
|
L'électron rare
|
4463ca0d43 | ||
|
L'électron rare
|
04fb49aae3 | ||
|
L'électron rare
|
c8139a2bea | ||
|
L'électron rare
|
921ae9cd42 | ||
|
L'électron rare
|
20d91c037f | ||
|
L'électron rare
|
4b6386ab5a | ||
|
L'électron rare
|
7211ca8393 | ||
|
L'électron rare
|
f79ce95f78 | ||
|
L'électron rare
|
9eff9daad8 | ||
|
L'électron rare
|
9fd856744d | ||
|
L'électron rare
|
549a6b53a6 | ||
|
L'électron rare
|
7949f50920 | ||
|
L'électron rare
|
ebed40c98a | ||
|
L'électron rare
|
b8d76ccb27 | ||
|
L'électron rare
|
e70595d81a | ||
|
L'électron rare
|
1848368bc2 | ||
|
L'électron rare
|
630b4aa6c0 | ||
|
L'électron rare
|
d7f3e3a2c1 | ||
|
L'électron rare
|
e7b3b63f5f | ||
|
L'électron rare
|
ec4be7bd1f | ||
|
L'électron rare
|
66ad49fe43 | ||
|
L'électron rare
|
0ab7c56036 | ||
|
L'électron rare
|
077a7f707f | ||
|
L'électron rare
|
b15dc75ad5 | ||
|
L'électron rare
|
d9fb4ac503 | ||
|
L'électron rare
|
bb06f349c7 | ||
|
L'électron rare
|
698fe9f7da | ||
|
L'électron rare
|
f951aacc9a | ||
|
L'électron rare
|
95c3bb5687 | ||
|
L'électron rare
|
a769bcd7ae | ||
|
L'électron rare
|
d122c93d19 | ||
|
L'électron rare
|
89e1be4803 | ||
|
L'électron rare
|
6848c798cc | ||
|
L'électron rare
|
d3e6297091 | ||
|
L'électron rare
|
7d0711b3f6 | ||
|
L'électron rare
|
b6cd8cf4c7 | ||
|
L'électron rare
|
bd46f6e052 | ||
|
L'électron rare
|
1828d7ca3e | ||
|
L'électron rare
|
67302e7ca2 | ||
|
L'électron rare
|
9838da3137 |
+33
@@ -10,6 +10,19 @@ launcher/**/xcuserdata/
|
||||
launcher/**/*.xcuserstate
|
||||
launcher/**/Package.resolved
|
||||
|
||||
# iphone-arbody — xcodeproj is generated by xcodegen, Local.xcconfig
|
||||
# carries personal Apple Developer Team ID. Both are local-only.
|
||||
iphone-arbody/ARBodyTracker.xcodeproj/
|
||||
iphone-arbody/Config/Local.xcconfig
|
||||
iphone-arbody/**/xcuserdata/
|
||||
iphone-arbody/**/*.xcuserstate
|
||||
iphone-arbody/build/
|
||||
iphone-arbody/DerivedData/
|
||||
|
||||
# SwiftPM build + editor artifacts
|
||||
.build/
|
||||
.swiftpm/
|
||||
|
||||
# openFrameworks — on garde les shaders + settings.json pour qu'ils
|
||||
# arrivent sur les autres machines, mais on ignore les binaires.
|
||||
oscope-of/bin/*
|
||||
@@ -44,3 +57,23 @@ sound_algo/*.log
|
||||
.vscode/
|
||||
*.swp
|
||||
*~
|
||||
|
||||
# tool session state (claude-mem / remember skill)
|
||||
.remember/
|
||||
.serena/
|
||||
.claude/agent-memory/
|
||||
|
||||
# Swift coverage artifacts
|
||||
*.profraw
|
||||
|
||||
# local python venvs (dirs or symlinks)
|
||||
**/.venv
|
||||
|
||||
# corrupted recovery backups
|
||||
AV-Live-corrupted-*/
|
||||
|
||||
# macOS / iCloud collision artifacts (auto-created on rename)
|
||||
*\ 2
|
||||
*\ 2.*
|
||||
*\ 3
|
||||
*\ 3.*
|
||||
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
[submodule "third_party/SMPLer-X"]
|
||||
path = third_party/SMPLer-X
|
||||
url = https://github.com/electron-rare/SMPLer-X.git
|
||||
url = https://git.saillant.cc/electron-rare/SMPLer-X.git
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for AI coding agents (Claude Code, Aider, Cursor, etc.) working in this repo.
|
||||
|
||||
## Project
|
||||
|
||||
`AV-Live` — live-coding audio-visual performance system: SuperCollider sound engine, openFrameworks visualiser driven by a Hantek 6022BL oscilloscope, and a SwiftUI menubar launcher orchestrating everything. Public, GPL-3. Repo `electron-rare/AV-Live`, branch `main`. Multi-host: GrosMac (source), macm1 (sink / Multi-HMR + Apple Vision ANE), iPhone 16 Pro (ARKit/LiDAR pub).
|
||||
|
||||
## Tech stack (per sub-project)
|
||||
|
||||
| Sub-project | Stack |
|
||||
|-------------|-------|
|
||||
| `sound_algo/` | SuperCollider (sclang + scsynth), 1099 SynthDefs, 345 tracks |
|
||||
| `oscope-of/` | openFrameworks C++, libusb (Hantek bulk), GLSL 150 / GL 3.2 core |
|
||||
| `launcher/` | SwiftUI menubar app, Swift Package Manager |
|
||||
| `data_only_viz/` | Python 3.11+ via `uv`, native Metal (pyobjc), multi-backend pose |
|
||||
| `data_feeds/` | Python data ingestion |
|
||||
| `web_realart/` | Node.js, Express, OSC bridge |
|
||||
| `avlivebody-mac/` | SwiftUI body-tracking client (ARKit/SMPL-X mesh, ad-hoc signed for local dev) |
|
||||
| `iphone-arbody/` | iOS app, ARBodyTracker, publishes `/body3d/kp` via OSC |
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Python sub-projects (uv only)
|
||||
cd data_only_viz && uv sync && uv run python -m data_only_viz
|
||||
cd data_feeds && uv sync
|
||||
|
||||
# openFrameworks
|
||||
cd oscope-of && make -j
|
||||
|
||||
# Web bridge
|
||||
cd web_realart && npm install && npm start
|
||||
|
||||
# Swift
|
||||
open launcher/Package.swift # or xcodebuild from CLI
|
||||
open avlivebody-mac/avlivebody.xcodeproj
|
||||
```
|
||||
|
||||
## Conventions
|
||||
|
||||
- Commits: subject ≤ 50 chars, body ≤ 72, no underscore in scope, no AI attribution, never `--no-verify` (hooks enforce).
|
||||
- Branches: `feat/<name>`, `fix/<name>`, `docs/<name>`, `refactor/<name>`, `chore/<name>`.
|
||||
- Language: French to the user, English in code/comments/commits.
|
||||
- No emojis in code/docs/commits unless explicitly requested.
|
||||
- Python: **always `uv`** (never pip/poetry/conda directly).
|
||||
- `.gitignore` already excludes `*.pt`, `*.ckpt`, `*.safetensors`, `*.mlpackage` at root — don't commit weights.
|
||||
- License: GPL-3 (whole repo) — keep new files under a compatible license header when adding third-party code.
|
||||
|
||||
## File layout
|
||||
|
||||
- `sound_algo/` — SC sound engine (own `CLAUDE.md`)
|
||||
- `oscope-of/` — visualiser
|
||||
- `launcher/` — macOS menubar
|
||||
- `data_only_viz/` — pose / mesh / body tracking pipeline (Metal)
|
||||
- `data_feeds/` — data ingestion
|
||||
- `web_realart/` — web UI + OSC bridge
|
||||
- `avlivebody-mac/`, `iphone-arbody/` — body-tracking clients
|
||||
- `shared/` — cross-sub-project assets
|
||||
- `third_party/` — vendored deps (CHECK before adding to root deps)
|
||||
- `tools/` — helper scripts
|
||||
- `docs/superpowers/plans/` — in-flight plans/specs
|
||||
- `AV-Live-corrupted-20260514/` — quarantined corrupted snapshot, do not touch
|
||||
|
||||
## Domain-specific gotchas
|
||||
|
||||
- **mDNS hostnames are required** (`grosmac.local`, `supra-m1.local`) for `AVBODY_HOST` / `MULTIHMR_REMOTE_HOST`. They resist DHCP changes (iPhone hotspot reassigns 172.20.10.x routinely).
|
||||
- **`POSE_FILTER` chain ordering is load-bearing**: default is `median+kalman+lookahead+ik`. Extras must be inserted at the right stage — `one_euro_joints` BEFORE kalman, `one_euro_bones` AFTER SMPL-X fusion in `multi.py`. `arkit_fuse` overrides 14 body slots with ARKit ARSkeleton3D from iOS app via `/body3d/kp` on `:57128` (always-on listener).
|
||||
- **`ICP_FUSION=1`** requires `ICP_LIDAR_HOST` (iPhone IP), `ICP_LIDAR_PORT` (default 5500, iPhone ARMesh TCP), and an extrinsic JSON at `~/.config/av-live/lidar_extrinsic.json`. See `docs/ICP_FUSION.md`.
|
||||
- **iPhone OSC port `57128`** is hardcoded as the publish target for `/body3d/kp` — don't reassign.
|
||||
- **`avlivebody-mac` requires ad-hoc signing for local dev** (fixed in `85589f2`). Don't strip the signing identity.
|
||||
- **`onVideoFrame` retain cycle in avlivebody** was fixed in `3b5f29e` — when adding new frame callbacks, mind the strong-self capture.
|
||||
- **AVLive-Body legacy** has been archived (`9e1482e`); the canonical client is `avlivebody-mac`. Don't reintroduce paths to the old project.
|
||||
- **macm1 = sink** (Multi-HMR CoreML + Apple Vision ANE + SMPL-X TCP); GrosMac = source. Mind the direction when wiring new OSC topics.
|
||||
- **Each major sub-project has its own `CLAUDE.md`** — closest wins. Put cross-cutting rules here, sub-project specifics in the nested file.
|
||||
|
||||
## When in doubt
|
||||
|
||||
- Read root `CLAUDE.md` and the nested `CLAUDE.md` of the sub-project you're editing.
|
||||
- Recent commits: `git log --oneline -20`.
|
||||
- Plans: `docs/superpowers/plans/`.
|
||||
- Cluster context: `~/CLAUDE.md` (GrosMac / macm1 / iPhone topology).
|
||||
- For sound: read `sound_algo/CLAUDE.md` before touching SynthDefs.
|
||||
@@ -1,6 +1,6 @@
|
||||
# AV-Live
|
||||
|
||||
Live coding audio-visual performance system : moteur SuperCollider, visualiseur openFrameworks piloté par un oscilloscope Hantek 6022BL, app menubar macOS qui orchestre le tout.
|
||||
Live coding audio-visual performance system : moteur SuperCollider, visualiseur openFrameworks piloté par un oscilloscope Hantek 6022BL, app menubar macOS qui orchestre le tout. **RC0.1+ (tag `v0.1.0-rc1`)** ajoute le pipeline Multi-HMR distribué M5 ↔ macm1 sur LAN gigabit + 10 scènes Metal pose-réactives + unified 3D armature wireframe (body+face+hands).
|
||||
|
||||
## Communication
|
||||
|
||||
@@ -26,6 +26,37 @@ Toujours répondre en français à l'utilisateur. Code, commentaires de code, co
|
||||
| Détection pose / mesh / body tracking | `data_only_viz/` |
|
||||
| Bridge web / UI de live coding | `web_realart/` |
|
||||
| Plans / specs en cours | `docs/superpowers/plans/` |
|
||||
| Multi-HMR remote server pyobjc | `data_only_viz/scripts/multihmr_server.py` (tourne sur macm1) |
|
||||
| Mesh dense rigger 27 fps perçu | `data_only_viz/mesh_rigger.py` |
|
||||
| 3D wireframe armature (body+face+hands) | `launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift` |
|
||||
| Pose → Metal scenes uniforms | `launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift` + `Resources/scene.metal` |
|
||||
| Filter chain (median + Kalman + lookahead + IK) | `data_only_viz/pose_filter.py` |
|
||||
| DINO re-id pid matching | `data_only_viz/dino_reid.py` |
|
||||
|
||||
## RC0.1+ environment variables
|
||||
|
||||
| Env | Default | Effect |
|
||||
|-----|---------|--------|
|
||||
| `MULTIHMR_BACKEND` | `pytorch` | `pytorch`, `coreml`, `remote` |
|
||||
| `MULTIHMR_REMOTE_HOST` | `127.0.0.1` | macm1 IP for remote inference |
|
||||
| `MULTIHMR_REMOTE_JPEG` | `1` | JPEG q=80 on the wire |
|
||||
| `MULTIHMR_REMOTE_ASYNC` | `1` | client double-buffer queue |
|
||||
| `MULTIHMR_SERVER_BACKEND` | `pyobjc` | server: `pyobjc` or `coremltools` |
|
||||
| `MULTIHMR_LOOP_FPS` | `30` | Python worker loop target_fps |
|
||||
| `AVBODY_HOST` | `127.0.0.1` | route TCP mesh + OSC to remote AVLiveBody |
|
||||
| `MEDIAPIPE_DELEGATE` | `gpu` | Metal SRGBA (faster); `cpu` fallback for stability |
|
||||
| `POSE_FILTER` | `median+kalman+lookahead+ik` | filter chain stages |
|
||||
| `MULTIHMR_REID` | `dino` | DINO cosine matching, `iou` fallback |
|
||||
|
||||
## Network topology
|
||||
|
||||
| Host | mDNS | IP (DHCP) | Role |
|
||||
|------|------|-----------|------|
|
||||
| GrosMac M5 | `grosmac.local` | LAN | Source + visualisation (AVLiveBody + data_only_viz + data_feeds) |
|
||||
| macm1 M1 Max | `supra-m1.local` | `192.168.0.175` | Sink (Multi-HMR CoreML + Apple Vision ANE + SMPL-X TCP) |
|
||||
| iPhone 16 Pro | (Personal Hotspot) | DHCP | ARKit/LiDAR pub via OSC `/body3d/kp` |
|
||||
|
||||
`AVBODY_HOST` / `MULTIHMR_REMOTE_HOST` accept mDNS hostnames — résiste aux changements DHCP (notamment iPhone hotspot 172.20.10.x).
|
||||
|
||||
## Environment variables
|
||||
|
||||
@@ -33,10 +64,35 @@ Toujours répondre en français à l'utilisateur. Code, commentaires de code, co
|
||||
|-----|---------|--------|
|
||||
| `POSE_FILTER` | `median+kalman+lookahead+ik` | filter chain stages — extra: `one_euro_joints` (joint-space CHI 2012 One Euro, inserted before kalman), `one_euro_bones` (bone-vector One Euro applied after SMPL-X fusion in multi.py), `arkit_fuse` (overrides 14 body slots with ARKit ARSkeleton3D from the iOS app, expects /body3d/kp on :57128) |
|
||||
| `IPHONE_OSC_PORT` | `57128` | UDP port the iPhone ARBodyTracker app pushes /body3d/kp to (always-on listener in data_only_viz) |
|
||||
| `VIDEO_ROTATE` | `none` | rotate the source video frame before MediaPipe + display: `none`/`ccw`/`cw`/`180`. Applied at multi.py read; pose detection + skeleton overlay stay aligned. ARKit 3D joints (gravity-aligned world) are unaffected |
|
||||
| `ICP_FUSION` | `0` | `1` to enable LiDAR↔SMPL-X ICP fusion (cf. `docs/ICP_FUSION.md`) |
|
||||
| `ICP_LIDAR_HOST` | _(unset)_ | iPhone ARBodyTracker IP when `ICP_FUSION=1` |
|
||||
| `ICP_LIDAR_PORT` | `5500` | iPhone ARMesh TCP port |
|
||||
| `ICP_LIDAR_EXTRINSIC` | `~/.config/av-live/lidar_extrinsic.json` | extrinsic JSON path |
|
||||
| `FINGER_PIANO` | `0` | `1` enables air-piano finger strike emission from data_only_viz |
|
||||
| `FINGER_STRIKE_VEL` | `0.02` | downward relative-velocity threshold for a strike (normalized image units/frame) |
|
||||
| `FINGER_STRIKE_REFRACTORY_MS` | `120` | min ms between strikes per finger |
|
||||
| `FINGER_SOURCE` | `auto` | hand source for finger gestures: `auto` (iPhone Vision hands if fresh, else MediaPipe), `iphone`, `mediapipe` |
|
||||
| `FINGER_DEBUG` | `0` | `1` logs each detected strike/pinch (hand, finger) — for threshold tuning |
|
||||
| `PINCH_RATIO_ON` | `0.50` | thumb-to-finger distance / hand size below which a pinch engages (0.50 validated live 2026-07-02 for ring/pinky reach) |
|
||||
| `PINCH_RATIO_OFF` | `0.65` | distance/size above which the pinch re-arms (hysteresis) |
|
||||
| `PINCH_REFRACTORY_MS` | `250` | min ms between pinches per finger |
|
||||
| `PINCH_MARGIN` | `0.05` | closest-wins margin between nearest and runner-up fingertip (0.05 validated live: 0.20 rejected middle/ring/pinky pinches) |
|
||||
| `PINCH_EXT_RATIO` | `1.35` | tip-to-wrist distance / hand size above which a finger counts as extended (open-hand pinch gate) |
|
||||
| `PINCH_EXT_MIN` | `1` | min extended non-pinching fingers required for a pinch to engage; `0` disables the open-hand gate (1 validated live: inner-finger pinches curl neighbours) |
|
||||
| `PINCH_DEBOUNCE_FRAMES` | `3` | consecutive qualifying frames before a pinch engage fires (release stays immediate); `1` disables |
|
||||
| `HAND_CONF_MIN` | `0.45` | min Vision hand confidence to draw (raised from 0.3; applies to both overlay and side panels) |
|
||||
| `HAND_PERSIST_FRAMES` | `3` | consecutive frames a hand track must match before being drawn (1 = off, all hands drawn immediately) |
|
||||
| `HAND_SWAP_LR` | `0` | Safety knob: invert the Vision chirality interpretation when routing hands (gestures + panels). Chirality validated correct live 2026-07-02; default `0`. Set `1` if a future iPhone app build delivers inverted chirality |
|
||||
| `HAND_NEAR_MIN` | `0.10` | Minimum wrist-to-middle-MCP distance (normalised) for a hand to trigger pinch/strike. Hands smaller than this are gated out. Display panels are unaffected |
|
||||
| `HAND_PERSIST_GRACE` | `2` | grace frames before dropping a hand track (0 = strict per-frame) |
|
||||
| `HAND_NEAR_OFF` | `0.08` | hand size hysteresis off-threshold for gesture slot stabilizer |
|
||||
| `HAND_HOLD_FRAMES` | `2` | gesture slot: frames to hold last hand on Vision miss |
|
||||
| `HAND_FACE_MIN` | `0.5` | palm-spread ratio threshold — gesture slot arms only when hand_facing >= this (palm-facing ~0.7-1.0, side-on < 0.4; 0.8× hysteresis for off) |
|
||||
| `ARKIT_BONE_MAX` | `0.5` | max ARKit body bone length in normalized units; longer bones are skipped (tracking-loss garbage filter) |
|
||||
| `PINCH_ENABLE` | `0` | enable pinch gesture detection |
|
||||
| `VIZ_HUD` | `0` | show debug HUD overlay |
|
||||
| `CONCERT_MIRROR` | `1` | mirror video horizontally (!=0 enables) |
|
||||
|
||||
## Conventions globales
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# AV-Live
|
||||
|
||||
**Release : `v0.1.0-rc1` (2026-05-14)** — distributed Multi-HMR (M5 ↔ macm1 LAN), unified 3D armature openpos, hybrid mesh rigger 27 fps perceived, 10 pose-reactive Metal scenes. See [RC0.1+ architecture](#rc01-distributed-multi-hmr-architecture) below.
|
||||
|
||||
> **Live coding audio-visual performance system** built around a SuperCollider sound engine, an openFrameworks oscilloscope visualizer driven by a real Hantek 6022BL USB scope, a macOS menubar launcher, and a Metal-native pose / body-mesh visualizer that listens to the same audio bus.
|
||||
>
|
||||
> 15 scripted demoparties · ~47 fullscreen visuals (26 3D parametric meshes + 18 procedural shaders + 3 dedicated C++ scenes) · 5 OS pixel-art shaders · 14 retro OS logos · 33 GLSL shader pairs (~65 files) · 1099 SynthDefs across 368 tracks (23 albums × 16) · **20 real-world data feeds** · 7 pose-estimation backends · **SMPL-X body mesh** (10 475 vertices via Multi-HMR + RealityKit) · **3 launch modes** (Full / Data-only / Body Mesh) — all reactive to the audio physically passing through the scope probes.
|
||||
@@ -101,7 +103,89 @@ Metal-native fullscreen visualizer (pyobjc, MTKView, ~60 fps) driven by webcam p
|
||||
- **Renderer** : Metal pipelines compiled at runtime (`shaders/*.metal`), `bg_pipeline` (full-screen FBM) + `skel_pipeline` (skeleton lines). SMPL face topology shipped as binary (`mesh_topology.py`) for RealityKit-compatible mesh rendering.
|
||||
- **Tracker** : One Euro Filter on keypoints + IoU multi-person association (`scipy.linear_sum_assignment`, ByteTrack-like).
|
||||
- **OSC out → sclang** `:57121` : `/pose/count`, `/pose/center`, `/pose/wrist`, `/pose/head`, `/pose/sho_span`, `/pose/limb_span`.
|
||||
- **Thread-safe state** : `state.py` exposes `State.lock()` ; dataclasses `PoseKp`, `NLFPerson` (vertices_3d, joints_3d), and a multi-person container.
|
||||
- **OSC out → AVLiveBody** `:57126` UDP (mode openpos, mode 9 / touche `p`) : `/pose/skel`, `/face/kp` (68 dlib landmarks), `/hand/kp` (21 × 2 hands), `/pose3d/kp` (33 MediaPipe pose_world_landmarks 3D meters).
|
||||
- **TCP out → AVLiveBody** `:57130` : SMPL-X dense mesh (10475 verts) frame-packed binary, 30 Hz rigged interpolation between Multi-HMR keyframes.
|
||||
- **Thread-safe state** : `state.py` exposes `State.lock()` ; dataclasses `PoseKp`, `Kp3D`, `SMPLXPerson`, multi-person container.
|
||||
|
||||
## RC0.1+ distributed Multi-HMR architecture
|
||||
|
||||
`feat/action-head` + tag `v0.1.0-rc1` (2026-05-14). The body-mesh pipeline is now **distributed across two Apple-silicon Macs over LAN gigabit** :
|
||||
|
||||
```
|
||||
M5 (capture host) macm1 (compute host)
|
||||
┌─────────────────────────────────┐ ┌──────────────────────────────────┐
|
||||
│ Caméra MacBook Pro │ │ Multi-HMR server :57140 (pyobjc) │
|
||||
│ data_only_viz/ │ JPEG q80 │ multihmr_full_672_s.mlpackage │
|
||||
│ ├─ multi_hmr_worker ├──TCP────▶│ ├─ Pyobjc direct CoreML.fwk │
|
||||
│ │ backend=remote (async) │ │ ├─ ~87 ms predict (M1 Max GPU) │
|
||||
│ ├─ MultiHMRRemoteBackend │◀──RSP───┤ └─ 6 outputs (v3d 10475, │
|
||||
│ │ queue maxsize 2/3 │ │ transl, scores, betas, │
|
||||
│ │ JPEG encode q80 │ │ expression, joints 127) │
|
||||
│ │ │ │ │
|
||||
│ ├─ MediaPipe Holistic │ │ AVLiveBody display (RealityKit) │
|
||||
│ │ Metal GPU delegate │ │ ├─ MeshRenderer (TCP :57130) │
|
||||
│ │ pose 33 + face 478 + │ /pose/* │ │ SMPL-X dense, low-level │
|
||||
│ │ hand 21×2 ├──UDP────▶│ │ mesh, 30 fps rigged interp │
|
||||
│ ├─ pose_bridge (UDP :57126) │ /face/* │ │ │
|
||||
│ ├─ smplx_tcp (TCP :57130, 30 Hz) │ /hand/* │ ├─ Skeleton3DRenderer (OSC) │
|
||||
│ ├─ MeshRigger (Hungarian + DINO │ /pose3d/│ │ 1 fused LowLevelMesh / │
|
||||
│ │ re-id, sticky pid 0.30/0.15) │ │ │ person : body 33 + face │
|
||||
│ └─ PoseFilterChain │ │ │ 68 + hand 21×2 = 143 vts │
|
||||
│ (median + Kalman CV + │ │ │ 288 line indices │
|
||||
│ lookahead + IK clamps) │ │ └─ 10 Metal scenes (storm, │
|
||||
│ │ │ tunnel, plasma, kaleido, │
|
||||
│ TCP loop fps : 25 │ │ voronoi, metaballs, │
|
||||
│ Multi-HMR fresh fps : 9 │ │ starfield, bars, hands3d, │
|
||||
│ MeshRigger perceived : 27 │ │ openpos), all consume │
|
||||
│ │ │ pose uniforms (mouth, │
|
||||
└─────────────────────────────────┘ │ velocity, head_tilt, │
|
||||
│ arm_spread, eye_open, │
|
||||
│ finger_pinch, body_xyz) │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Key technical wins (commits in `feat/action-head`)
|
||||
|
||||
| Commit | Win |
|
||||
|--------|-----|
|
||||
| `5800156` | Multi-HMR ViT-S/672 → CoreML mlpackage (FP32, 6 outputs) |
|
||||
| `52588b9` | roma.rotmat_to_rotvec → branchless atan2 (fixed all-NaN v3d/transl bug) |
|
||||
| `4e7101c` | NaN/Inf guard on v3d before TCP ship |
|
||||
| `2c8094c` | MeshRigger : 27 fps perceived via Hungarian pid match + Vision pelvis delta |
|
||||
| `1f623fe` | AVLiveBody mirror webcam preview (CATransform3D scale -1) |
|
||||
| `9838da3` | Remote inference protocol (TCP :57140) + multi-buffer async client |
|
||||
| `67302e7` | Pyobjc server (drops coremltools overhead) + MediaPipe Metal GPU SRGBA |
|
||||
| `1828d7c` | 12 new SceneUniforms (mouth, eye, head, finger, body) drive 5 scenes |
|
||||
| `bd46f6e` | Lift Python self-throttle 10 → 30 fps loop, fresh fps metric |
|
||||
| latest | 10 Metal scenes pose-reactive + SMPL-X 127 joints output (finger props) |
|
||||
|
||||
### Environment toggles
|
||||
|
||||
| Env var | Default | Effect |
|
||||
|---------|---------|--------|
|
||||
| `MULTIHMR_BACKEND` | `pytorch` | `pytorch`, `coreml`, `remote` |
|
||||
| `MULTIHMR_REMOTE_HOST` | `127.0.0.1` | macm1 IP (gigabit LAN) |
|
||||
| `MULTIHMR_REMOTE_JPEG` | `1` | JPEG q=80 compression on the wire |
|
||||
| `MULTIHMR_REMOTE_ASYNC` | `1` | client double-buffer queue (maxsize 2/3) |
|
||||
| `MULTIHMR_SERVER_BACKEND` | `pyobjc` | server : `pyobjc` or `coremltools` |
|
||||
| `MULTIHMR_LOOP_FPS` | `30` | Python loop target_fps (formerly capped at 10) |
|
||||
| `AVBODY_HOST` | `127.0.0.1` | route TCP mesh + OSC pose to a remote AVLiveBody |
|
||||
| `MEDIAPIPE_DELEGATE` | `cpu` | `gpu` Metal SRGBA (faster but IOSurface flake on M5) |
|
||||
| `POSE_FILTER` | `median+kalman+lookahead+ik` | toggle filter chain stages |
|
||||
| `MULTIHMR_REID` | `dino` if mlpackage present | `dino` (cosine match) or `iou` |
|
||||
| `MULTIHMR_REID_ALPHA` | `0.5` | IoU vs cosine weight (0=DINO only, 1=IoU only) |
|
||||
|
||||
### Hardware ceilings observed (M5 + M1 Max 32c GPU, LAN gigabit)
|
||||
|
||||
| Path | Predict | Live loop | Mesh rigged perceived |
|
||||
|------|---------|-----------|------------------------|
|
||||
| M5 local (PyTorch MPS) | 270 ms | 3.5 fps | 27 fps |
|
||||
| M5 local (CoreML FP32) | 139 ms | 6.8 fps | 27 fps |
|
||||
| Remote macm1 (idle GPU) | 53 ms | ~18 fps | 27 fps |
|
||||
| Remote macm1 (under MLX contention) | 87 ms | 25 fps loop / 9 fps fresh | 27 fps |
|
||||
| Studio M3 Ultra (80c GPU, *training only*) | est. 35 ms (~30 fps) | — | — |
|
||||
|
||||
Hard ceiling on macm1 ≈ 18 fps fresh predict (unified memory bandwidth + CoreML sync overhead) ; further gains require moving MLX servers off macm1 or quantising the model.
|
||||
|
||||
### 🎬 Demoparties — 15 narrative demos
|
||||
Each demoparty is a multi-act scripted show with unique scroller text, background sequence, scroller style, and SuperCollider album track :
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
*.xcodeproj/
|
||||
Config/Local.xcconfig
|
||||
.build/
|
||||
.swiftpm/
|
||||
@@ -0,0 +1,3 @@
|
||||
// Copy to Config/Local.xcconfig and set your Apple Developer Team ID.
|
||||
// Config/Local.xcconfig is gitignored.
|
||||
DEVELOPMENT_TEAM = YOUR_TEAM_ID
|
||||
@@ -0,0 +1,8 @@
|
||||
#include? "Local.xcconfig"
|
||||
|
||||
MACOSX_DEPLOYMENT_TARGET = 15.0
|
||||
SWIFT_VERSION = 5.10
|
||||
// Manual ad-hoc signing for local dev (no Apple Mac Development cert
|
||||
// required). Override here or via target settings for distribution.
|
||||
CODE_SIGN_STYLE = Manual
|
||||
CODE_SIGN_IDENTITY = -
|
||||
@@ -0,0 +1,62 @@
|
||||
# AVLiveBody (macOS)
|
||||
|
||||
Native macOS Xcode app that renders SMPL-X body meshes in RealityKit
|
||||
from the USB iPhone body-tracking pipeline (ARBodyTracker -> Multi-HMR
|
||||
worker -> AVLiveBody scene).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- macOS 15+
|
||||
- Xcode 16+
|
||||
- `xcodegen` — `brew install xcodegen`
|
||||
|
||||
## First-time setup
|
||||
|
||||
1. Copy the CoreML model into the app resources (required, gitignored,
|
||||
~195 MB). Without it the app degrades to skeleton-only rendering:
|
||||
|
||||
```
|
||||
cp -R ~/.cache/av-live-multihmr/multihmr_full_672_s.mlpackage \
|
||||
avlivebody-mac/Sources/AVLiveBody/Resources/
|
||||
```
|
||||
|
||||
2. Create your local xcconfig and set your signing team:
|
||||
|
||||
```
|
||||
cp Config/Local.xcconfig.example Config/Local.xcconfig
|
||||
# Edit Config/Local.xcconfig:
|
||||
# DEVELOPMENT_TEAM = <your Apple Developer Team ID>
|
||||
```
|
||||
|
||||
## Build
|
||||
|
||||
Generate the Xcode project (run after every `project.yml` change) then
|
||||
open or build from the CLI:
|
||||
|
||||
```
|
||||
cd avlivebody-mac
|
||||
xcodegen generate
|
||||
open AVLiveBody.xcodeproj
|
||||
```
|
||||
|
||||
CLI build / test:
|
||||
|
||||
```
|
||||
xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \
|
||||
-destination 'platform=macOS' build
|
||||
|
||||
xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \
|
||||
-destination 'platform=macOS' test
|
||||
```
|
||||
|
||||
## Runtime requirements
|
||||
|
||||
A tethered iPhone running the matching `ARBodyTracker` iOS app over USB
|
||||
is required for body input. See `iphone-arbody/` for the iOS side.
|
||||
|
||||
## Architecture
|
||||
|
||||
- Design spec:
|
||||
`docs/superpowers/specs/2026-05-18-avlivebody-macos-rewrite-design.md`
|
||||
- Implementation plan:
|
||||
`docs/superpowers/plans/2026-05-18-avlivebody-macos-rewrite.md`
|
||||
@@ -0,0 +1,139 @@
|
||||
import Cocoa
|
||||
import CoreVideo
|
||||
import SwiftUI
|
||||
|
||||
/// Forces a regular, keyboard-focusable foreground app.
|
||||
final class AppDelegate: NSObject, NSApplicationDelegate {
|
||||
func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
NSApp.setActivationPolicy(.regular)
|
||||
NSApp.activate()
|
||||
}
|
||||
}
|
||||
|
||||
@main
|
||||
struct AVLiveBodyApp: App {
|
||||
@NSApplicationDelegateAdaptor(AppDelegate.self)
|
||||
private var appDelegate
|
||||
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
ContentView()
|
||||
.frame(minWidth: 900, minHeight: 600)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
struct ContentView: View {
|
||||
@StateObject private var consumer = USBSkeletonConsumer()
|
||||
@StateObject private var settings = RenderSettings()
|
||||
@State private var prevPelvis: SIMD3<Float>?
|
||||
private let controller = SceneController()
|
||||
private let renderer = SceneRenderer.make()
|
||||
/// Multi-HMR mesh inference (CoreML, ~150-300 ms/frame) is OFF by
|
||||
/// default: it would saturate the main thread and beachball the UI.
|
||||
/// Skeleton-only rendering (iPhone 91-joint ARKit stream) otherwise.
|
||||
/// Enable on a capable, properly-threaded host with AVBODY_MULTIHMR=1.
|
||||
private let multiHMR: MultiHMRCoreML? = {
|
||||
guard ProcessInfo.processInfo.environment["AVBODY_MULTIHMR"] == "1"
|
||||
else { return nil }
|
||||
return MultiHMRCoreML()
|
||||
}()
|
||||
/// Placeholder intrinsics until a `.meta` frame supplies real ones.
|
||||
private let cameraK: [Float] = [
|
||||
672, 0, 336, 0, 672, 336, 0, 0, 1,
|
||||
]
|
||||
|
||||
var body: some View {
|
||||
ZStack(alignment: .topTrailing) {
|
||||
LayeredSceneView(controller: controller, renderer: renderer,
|
||||
showScene: settings.showScene)
|
||||
StatusBar(consumer: consumer)
|
||||
.frame(maxWidth: .infinity, alignment: .top)
|
||||
HStack(alignment: .top) {
|
||||
Spacer()
|
||||
if settings.showPanel {
|
||||
SettingsPanel(settings: settings)
|
||||
.transition(.move(edge: .trailing))
|
||||
}
|
||||
Button {
|
||||
withAnimation { settings.showPanel.toggle() }
|
||||
} label: {
|
||||
Image(systemName: "slider.horizontal.3")
|
||||
.padding(8)
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
.padding(8)
|
||||
}
|
||||
}
|
||||
.onAppear { wire(); applyAll() }
|
||||
.onDisappear { consumer.stop() }
|
||||
.onReceive(consumer.$skeletons) { skeletons in
|
||||
controller.updateSkeleton(skeletons)
|
||||
pushUniforms()
|
||||
}
|
||||
.onReceive(consumer.$hands) { _ in pushUniforms() }
|
||||
.onReceive(consumer.$face) { _ in pushUniforms() }
|
||||
.onReceive(settings.objectWillChange) { _ in
|
||||
// objectWillChange fires before the value updates; apply on
|
||||
// the next runloop tick so we read the new values.
|
||||
DispatchQueue.main.async { applyAll() }
|
||||
}
|
||||
}
|
||||
|
||||
private func applyAll() {
|
||||
controller.setSkeletonVisible(settings.showSkeleton)
|
||||
controller.setMeshVisible(settings.showMesh)
|
||||
controller.setVideoVisible(settings.showVideo)
|
||||
controller.setVideoOpacity(settings.videoOpacity)
|
||||
controller.updateMeshMaterial(metallic: settings.meshMetallic,
|
||||
roughness: settings.meshRoughness)
|
||||
controller.setLightIntensities(key: settings.keyIntensity,
|
||||
fill: settings.fillIntensity,
|
||||
rim: settings.rimIntensity)
|
||||
controller.setFieldOfView(settings.fieldOfView)
|
||||
if let r = renderer {
|
||||
r.uniforms.viz_mode = Float(settings.vizMode)
|
||||
}
|
||||
}
|
||||
|
||||
private func pushUniforms() {
|
||||
guard let r = renderer else { return }
|
||||
r.uniforms.viz_mode = Float(settings.vizMode)
|
||||
SceneUniformBuilder.fill(&r.uniforms,
|
||||
skeletons: consumer.skeletons,
|
||||
hands: consumer.hands,
|
||||
face: consumer.face,
|
||||
prevPelvis: &prevPelvis)
|
||||
controller.updateHandFace(hands: consumer.hands,
|
||||
face: consumer.face,
|
||||
depth: Float(-(prevPelvis?.z ?? 0)))
|
||||
}
|
||||
|
||||
private func wire() {
|
||||
let controller = self.controller
|
||||
let multiHMR = self.multiHMR
|
||||
let cameraK = self.cameraK
|
||||
consumer.onVideoFrame = { [weak consumer] pixelBuffer in
|
||||
MainActor.assumeIsolated {
|
||||
controller.updateVideo(pixelBuffer)
|
||||
guard let consumer, let hmr = multiHMR else { return }
|
||||
// Snapshot skeletons on main; run the heavy CoreML
|
||||
// prediction off-main (drops frames if busy), then hop
|
||||
// back to main for the light fuse + mesh UI update.
|
||||
let skeletons = consumer.skeletons
|
||||
hmr.inferAsync(pixelBuffer, cameraK: cameraK) { raw in
|
||||
DispatchQueue.main.async {
|
||||
MainActor.assumeIsolated {
|
||||
let fused = BodyFusion.fuse(
|
||||
persons: raw,
|
||||
skeletons: skeletons)
|
||||
controller.updateMesh(fused)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
consumer.start()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import Foundation
|
||||
import simd
|
||||
|
||||
/// ARKit/Multi-HMR world coords (y up, z back) -> RealityKit world
|
||||
/// coords (y up, z forward). Apply to every vertex/translation that
|
||||
/// crosses from source pipeline space into the scene.
|
||||
@inline(__always)
|
||||
func arkitToRealityKit(_ v: SIMD3<Float>) -> SIMD3<Float> {
|
||||
SIMD3<Float>(v.x, -v.y, -v.z)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// avlivebody-mac/Sources/AVLiveBody/HandFaceSkeleton.swift
|
||||
import AVLiveWire
|
||||
import AppKit
|
||||
import RealityKit
|
||||
import simd
|
||||
|
||||
@MainActor
|
||||
final class HandFaceSkeleton {
|
||||
let root = Entity()
|
||||
private static let r: Float = 0.008
|
||||
private static let spanX: Float = 1.2 // maps [0,1] -> [-0.6,0.6] m
|
||||
private static let spanY: Float = 0.9
|
||||
|
||||
private let mesh = MeshResource.generateSphere(radius: r)
|
||||
private let handMat = SimpleMaterial(color: NSColor.cyan, isMetallic: false)
|
||||
private let faceMat = SimpleMaterial(color: NSColor.magenta, isMetallic: false)
|
||||
private var handPool: [ModelEntity] = []
|
||||
private var facePool: [ModelEntity] = []
|
||||
private var depth: Float = 0
|
||||
|
||||
func setDepth(_ z: Float) { depth = z }
|
||||
|
||||
func update(hands: HandsPayload?, face: FacePayload?) {
|
||||
let hpts = (hands?.hands ?? []).flatMap { $0.points }
|
||||
layout(&handPool, count: hpts.count, mat: handMat)
|
||||
for (i, p) in hpts.enumerated() { place(handPool[i], p.x, p.y) }
|
||||
for i in hpts.count..<handPool.count { handPool[i].isEnabled = false }
|
||||
|
||||
let fpts = face?.points ?? []
|
||||
layout(&facePool, count: fpts.count, mat: faceMat)
|
||||
for (i, p) in fpts.enumerated() { place(facePool[i], p.x, p.y) }
|
||||
for i in fpts.count..<facePool.count { facePool[i].isEnabled = false }
|
||||
}
|
||||
|
||||
private func place(_ e: ModelEntity, _ nx: Float, _ ny: Float) {
|
||||
// normalized top-left [0,1] -> centered meters, y up
|
||||
e.transform.translation = SIMD3<Float>(
|
||||
(nx - 0.5) * Self.spanX,
|
||||
(0.5 - ny) * Self.spanY,
|
||||
depth)
|
||||
e.isEnabled = true
|
||||
}
|
||||
|
||||
private func layout(_ pool: inout [ModelEntity], count: Int,
|
||||
mat: SimpleMaterial) {
|
||||
while pool.count < count {
|
||||
let e = ModelEntity(mesh: mesh, materials: [mat])
|
||||
root.addChild(e); pool.append(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleName</key><string>AVLiveBody</string>
|
||||
<key>CFBundleIdentifier</key><string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleExecutable</key><string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundlePackageType</key><string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key><string>1.0</string>
|
||||
<key>CFBundleVersion</key><string>1</string>
|
||||
<key>LSMinimumSystemVersion</key><string>15.0</string>
|
||||
<key>NSCameraUsageDescription</key>
|
||||
<string>Receives the tethered iPhone camera over USB.</string>
|
||||
<key>NSLocalNetworkUsageDescription</key>
|
||||
<string>Connects to the tethered iPhone over USB (usbmuxd).</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,51 @@
|
||||
// avlivebody-mac/Sources/AVLiveBody/LayeredSceneView.swift
|
||||
import AppKit
|
||||
import MetalKit
|
||||
import RealityKit
|
||||
import SwiftUI
|
||||
|
||||
/// Layered container: MTKView (Metal shader bg, transparent) behind the
|
||||
/// SceneController's transparent ARView. The owner holds the SceneRenderer
|
||||
/// so it can push uniforms each frame.
|
||||
struct LayeredSceneView: NSViewRepresentable {
|
||||
let controller: SceneController
|
||||
let renderer: SceneRenderer?
|
||||
let showScene: Bool
|
||||
|
||||
func makeNSView(context: Context) -> NSView {
|
||||
controller.setUp()
|
||||
let container = NSView(frame: .zero)
|
||||
container.wantsLayer = true
|
||||
container.layer?.backgroundColor = NSColor.black.cgColor
|
||||
|
||||
if let renderer {
|
||||
let mtk = MTKView(frame: container.bounds,
|
||||
device: MTLCreateSystemDefaultDevice())
|
||||
mtk.delegate = renderer
|
||||
mtk.colorPixelFormat = .bgra8Unorm
|
||||
mtk.framebufferOnly = false
|
||||
mtk.layer?.isOpaque = true
|
||||
mtk.clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 1)
|
||||
mtk.preferredFramesPerSecond = 60
|
||||
mtk.autoresizingMask = [.width, .height]
|
||||
container.addSubview(mtk)
|
||||
context.coordinator.mtk = mtk
|
||||
}
|
||||
|
||||
let arView = controller.arView
|
||||
arView.frame = container.bounds
|
||||
arView.autoresizingMask = [.width, .height]
|
||||
arView.wantsLayer = true
|
||||
arView.layer?.isOpaque = false
|
||||
arView.layer?.backgroundColor = NSColor.clear.cgColor
|
||||
container.addSubview(arView) // front
|
||||
return container
|
||||
}
|
||||
|
||||
func updateNSView(_ nsView: NSView, context: Context) {
|
||||
context.coordinator.mtk?.isHidden = !showScene
|
||||
}
|
||||
|
||||
func makeCoordinator() -> Coordinator { Coordinator() }
|
||||
final class Coordinator { var mtk: MTKView? }
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import AppKit
|
||||
import Foundation
|
||||
import RealityKit
|
||||
import simd
|
||||
|
||||
/// Renders SMPL-X dense body meshes (10475 vertices) from Multi-HMR.
|
||||
/// Triangle indices come from the bundled `smplx_faces.bin`
|
||||
/// (flat UInt32 triplets).
|
||||
@MainActor
|
||||
final class MeshEntity {
|
||||
let root = Entity()
|
||||
|
||||
private static let vertexCount = 10475
|
||||
private let faces: [UInt32]
|
||||
private var pools: [Int: ModelEntity] = [:]
|
||||
private var material = SimpleMaterial(
|
||||
color: NSColor(white: 0.8, alpha: 1.0),
|
||||
roughness: 0.5, isMetallic: false)
|
||||
|
||||
init() {
|
||||
faces = MeshEntity.loadFaces()
|
||||
}
|
||||
|
||||
func update(_ persons: [MultiHMRPerson]) {
|
||||
for (idx, person) in persons.enumerated() {
|
||||
let entity = pools[idx] ?? {
|
||||
let e = ModelEntity()
|
||||
root.addChild(e)
|
||||
pools[idx] = e
|
||||
return e
|
||||
}()
|
||||
guard let mesh = buildMesh(person.vertices) else { continue }
|
||||
entity.model = ModelComponent(mesh: mesh,
|
||||
materials: [material])
|
||||
let t = person.translation
|
||||
entity.transform.translation = arkitToRealityKit(t)
|
||||
entity.isEnabled = true
|
||||
}
|
||||
for idx in pools.keys where idx >= persons.count {
|
||||
pools[idx]?.isEnabled = false
|
||||
}
|
||||
}
|
||||
|
||||
func setVisible(_ visible: Bool) { root.isEnabled = visible }
|
||||
|
||||
func setMaterial(metallic: Bool, roughness: Double) {
|
||||
material = SimpleMaterial(
|
||||
color: NSColor(white: 0.8, alpha: 1.0),
|
||||
roughness: .float(Float(roughness)), isMetallic: metallic)
|
||||
for entity in pools.values {
|
||||
entity.model?.materials = [material]
|
||||
}
|
||||
}
|
||||
|
||||
private func buildMesh(_ verts: [SIMD3<Float>])
|
||||
-> MeshResource? {
|
||||
guard verts.count == Self.vertexCount, !faces.isEmpty else {
|
||||
NSLog("MeshEntity: vertex count mismatch %d (expected %d), faces=%d",
|
||||
verts.count, Self.vertexCount, faces.count)
|
||||
return nil
|
||||
}
|
||||
var descriptor = MeshDescriptor(name: "smplx")
|
||||
descriptor.positions = MeshBuffer(verts.map(arkitToRealityKit))
|
||||
descriptor.primitives = .triangles(faces)
|
||||
return try? MeshResource.generate(from: [descriptor])
|
||||
}
|
||||
|
||||
private static func loadFaces() -> [UInt32] {
|
||||
guard let url = Bundle.main.url(
|
||||
forResource: "smplx_faces", withExtension: "bin"),
|
||||
let data = try? Data(contentsOf: url) else {
|
||||
NSLog("MeshEntity: smplx_faces.bin missing")
|
||||
return []
|
||||
}
|
||||
return data.withUnsafeBytes { raw in
|
||||
Array(raw.bindMemory(to: UInt32.self))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// avlivebody-mac/Sources/AVLiveBody/RenderSettings.swift
|
||||
import SwiftUI
|
||||
|
||||
/// Live visual settings for AVLiveBody. ContentView observes this and
|
||||
/// pushes changes to SceneController. Defaults chosen so the scene is
|
||||
/// lit and everything visible out of the box.
|
||||
@MainActor
|
||||
final class RenderSettings: ObservableObject {
|
||||
// Layer visibility
|
||||
@Published var showSkeleton: Bool = true
|
||||
@Published var showMesh: Bool = true
|
||||
@Published var showVideo: Bool = true
|
||||
|
||||
// Mesh material
|
||||
@Published var meshMetallic: Bool = false
|
||||
@Published var meshRoughness: Double = 0.6
|
||||
|
||||
// Video quad
|
||||
@Published var videoOpacity: Double = 1.0
|
||||
|
||||
// Lights (RealityKit DirectionalLight intensities)
|
||||
@Published var keyIntensity: Double = 4000
|
||||
@Published var fillIntensity: Double = 1500
|
||||
@Published var rimIntensity: Double = 2000
|
||||
|
||||
// Camera
|
||||
@Published var fieldOfView: Double = 55
|
||||
|
||||
// Panel visibility
|
||||
@Published var showPanel: Bool = false
|
||||
|
||||
// Metal shader background (Phase 2)
|
||||
@Published var showScene: Bool = true
|
||||
@Published var vizMode: Int = 0 // 0..9
|
||||
}
|
||||
+56
-13
@@ -34,8 +34,34 @@ struct SceneUniforms {
|
||||
float hand_l_y;
|
||||
float hand_r_x;
|
||||
float hand_r_y;
|
||||
// ---- 2026-05-14 face / hand / body3d derivatives ----
|
||||
// mouth_open : |lip51.y - lip57.y| normalized 0..~0.15
|
||||
// eye_open_l/r : H/W ratio of eye bbox, ~0.15 (closed) .. 0.45 (open)
|
||||
// head_tilt : atan2(eye_R.y-eye_L.y, eye_R.x-eye_L.x) rad
|
||||
// head_yaw : nose.y - eyes_mid.y (proxy pitch, normalized)
|
||||
// finger_pinch_l/r: simd_distance(thumb_tip[4], index_tip[8]) px-norm
|
||||
// body_x/y/z : pelvis world meters (MediaPipe hip-relative)
|
||||
// body_height : |hip.y - head.y| world meters
|
||||
// arm_spread : |wristL.x - wristR.x| world meters
|
||||
// pose_velocity : EMA(|pelvis(t) - pelvis(t-1)|), alpha=0.3
|
||||
float mouth_open;
|
||||
float eye_open_l;
|
||||
float eye_open_r;
|
||||
float head_tilt;
|
||||
float head_yaw;
|
||||
float finger_pinch_l;
|
||||
float finger_pinch_r;
|
||||
float body_x;
|
||||
float body_y;
|
||||
float body_z;
|
||||
float body_height;
|
||||
float arm_spread;
|
||||
float pose_velocity;
|
||||
float _pad0;
|
||||
float _pad1;
|
||||
float _pad2;
|
||||
float _pad3;
|
||||
float _pad4;
|
||||
};
|
||||
|
||||
struct VsOut {
|
||||
@@ -120,21 +146,25 @@ float vignette(float2 p) {
|
||||
// ===== Modes =======================================================
|
||||
|
||||
// ---- 0 storm : tissu fbm reactif + bloom-fake ----
|
||||
// Pose mods : pose_velocity boost intensity, head_tilt shifts hue.
|
||||
float3 mode_storm(float2 p, constant SceneUniforms& U) {
|
||||
float storm = saturate(U.kp_norm * 1.0 + max(-U.bz_norm, 0.0) * 0.5);
|
||||
float speed = 0.08 + U.wind_norm * 1.5;
|
||||
float storm = saturate(U.kp_norm * 1.0 + max(-U.bz_norm, 0.0) * 0.5
|
||||
+ U.pose_velocity * 2.0);
|
||||
float speed = 0.08 + U.wind_norm * 1.5 + U.pose_velocity * 3.0;
|
||||
float zoom = 1.8 - U.rms * 1.2;
|
||||
float n = fbm(p * zoom + float2(U.time * speed, U.time * speed * 0.7));
|
||||
n = pow(n, 1.2 - U.rms * 0.5);
|
||||
float netz = sin(U.time * 50.0 + U.netz_dev * 800.0) * 0.06;
|
||||
float3 base = palIQ(n + storm * 0.5,
|
||||
float hue_shift = U.head_tilt * 0.25;
|
||||
float3 base = palIQ(n + storm * 0.5 + hue_shift,
|
||||
float3(0.10, 0.05, 0.20),
|
||||
float3(0.40, 0.30, 0.55),
|
||||
float3(1.0, 1.0, 1.0),
|
||||
float3(0.0, 0.33, 0.67));
|
||||
float bloom = smoothstep(0.7, 1.0, n);
|
||||
return base * (n * 1.4 + 0.3) + netz + U.rms * 1.2
|
||||
+ bloom * 0.7
|
||||
float velocity_boost = 1.0 + U.pose_velocity * 1.5;
|
||||
return (base * (n * 1.4 + 0.3) + netz + U.rms * 1.2
|
||||
+ bloom * 0.7) * velocity_boost
|
||||
+ float3(1.0, 0.55, 0.1) * U.flare * 1.4
|
||||
+ float3(U.lightning_flash * 0.7);
|
||||
}
|
||||
@@ -167,34 +197,40 @@ float3 mode_tunnel(float2 p, constant SceneUniforms& U) {
|
||||
}
|
||||
|
||||
// ---- 2 plasma : volumetric noise palette IQ ----
|
||||
// Pose mods : mouth_open modulates spatial frequency; head_yaw shifts hue.
|
||||
float3 mode_plasma(float2 p, constant SceneUniforms& U) {
|
||||
float t = U.time * (0.5 + U.rms * 1.5);
|
||||
float freq_boost = 1.0 + U.mouth_open * 12.0;
|
||||
// 3 octaves de sin/cos en composition
|
||||
float v = sin(p.x * 4.0 + t)
|
||||
+ sin(p.y * 5.0 - t * 1.2)
|
||||
float v = sin(p.x * 4.0 * freq_boost + t)
|
||||
+ sin(p.y * 5.0 * freq_boost - t * 1.2)
|
||||
+ sin((p.x + p.y) * 3.5 + t * 0.7)
|
||||
+ sin(length(p) * (8.0 + U.kp_norm * 4.0) - t * 1.8);
|
||||
v = v * 0.25 + 0.5;
|
||||
// Fake volumetric "depth" : repeat layers
|
||||
float layer2 = sin(p.x * 2.0 - t * 0.5) * sin(p.y * 2.5 + t * 0.7);
|
||||
v = mix(v, v * 0.5 + 0.5 * (layer2 + 1.0) * 0.5, 0.35);
|
||||
float3 col = palIQ(v,
|
||||
float hue_offset = U.head_yaw * 0.5;
|
||||
float3 col = palIQ(v + hue_offset,
|
||||
float3(0.5),
|
||||
float3(0.5),
|
||||
float3(1.0, 1.0, 1.0),
|
||||
float3(0.0, 0.33, 0.67));
|
||||
col *= 0.8 + U.kp_norm * 0.7 + U.social_rate * 0.5;
|
||||
col *= 0.8 + U.kp_norm * 0.7 + U.social_rate * 0.5
|
||||
+ U.mouth_open * 1.0;
|
||||
return col + float3(0.6, 0.3, 1.0) * U.lightning_flash * 0.5;
|
||||
}
|
||||
|
||||
// ---- 3 kaleido : KIFS fractal 6-fold avec rot 3D fake ----
|
||||
// Pose mods : arm_spread drives segment count (4..16).
|
||||
float3 mode_kaleido(float2 p, constant SceneUniforms& U) {
|
||||
float ang = U.time * 0.15 + U.flare * 2.0;
|
||||
float c = cos(ang), s = sin(ang);
|
||||
p = float2(c * p.x - s * p.y, s * p.x + c * p.y);
|
||||
float r = length(p);
|
||||
float a = atan2(p.y, p.x);
|
||||
float seg = 6.28318 / 6.0;
|
||||
float seg_count = clamp(ceil(4.0 + U.arm_spread * 8.0), 3.0, 16.0);
|
||||
float seg = 6.28318 / seg_count;
|
||||
a = abs(fmod(a + seg * 0.5, seg) - seg * 0.5);
|
||||
float2 q = float2(cos(a), sin(a)) * r;
|
||||
// Iteration KIFS-like
|
||||
@@ -303,14 +339,18 @@ float3 mode_metaballs(float2 p, constant SceneUniforms& U) {
|
||||
}
|
||||
|
||||
// ---- 6 starfield : galaxy spiral + parallax ----
|
||||
// Pose mods : finger_pinch L+R drives star density per layer.
|
||||
float3 mode_starfield(float2 p, constant SceneUniforms& U) {
|
||||
float warp = U.time * (1.5 + U.wind_norm * 6.0);
|
||||
float pinch = saturate((U.finger_pinch_l + U.finger_pinch_r) * 2.0);
|
||||
int stars_per_layer = 30 + int(pinch * 70.0); // 30..100
|
||||
// 3 layers of stars at different speeds
|
||||
float3 col = float3(0);
|
||||
for (int L = 0; L < 3; ++L) {
|
||||
float speed = (1.0 + float(L) * 0.5);
|
||||
float scale = 6.0 + float(L) * 4.0;
|
||||
for (int k = 0; k < 50; ++k) {
|
||||
for (int k = 0; k < 100; ++k) {
|
||||
if (k >= stars_per_layer) break;
|
||||
float fk = float(k + L * 50);
|
||||
float r0 = hash21(float2(fk, 7.0 + float(L)));
|
||||
float a0 = hash21(float2(fk, 17.0 + float(L))) * 6.28;
|
||||
@@ -352,8 +392,11 @@ float3 mode_bars(float2 p, constant SceneUniforms& U) {
|
||||
// Hauteur barre depend du bin "i" via hash + RMS
|
||||
float h0 = hash21(float2(float(i), 0.0));
|
||||
float h = sin(t * (0.5 + h0 * 4.0) + float(i)) * 0.5 + 0.5;
|
||||
h = h * (0.3 + U.rms * 1.5 + U.social_rate * 0.4);
|
||||
h = clamp(h, 0.02, 0.85);
|
||||
// body_height (0..~1.8 m) + eye_open avg modulate bar height
|
||||
float eyes = (U.eye_open_l + U.eye_open_r) * 0.5;
|
||||
h = h * (0.3 + U.rms * 1.5 + U.social_rate * 0.4
|
||||
+ U.body_height * 0.6 + eyes * 0.8);
|
||||
h = clamp(h, 0.02, 0.95);
|
||||
float bar_top = y_base + h * perspective * 0.3;
|
||||
// Largeur = 1 / nbars perspective
|
||||
float bx = (fi - 0.5) * perspective * 1.5;
|
||||
@@ -0,0 +1,172 @@
|
||||
import AppKit
|
||||
import Foundation
|
||||
import CoreVideo
|
||||
import RealityKit
|
||||
import simd
|
||||
import AVLiveWire
|
||||
|
||||
/// Owns the single RealityKit scene: the video quad, the body root,
|
||||
/// and an orbital camera. The app calls `updateVideo/updateSkeleton/
|
||||
/// updateMesh` from the main queue.
|
||||
@MainActor
|
||||
final class SceneController {
|
||||
let arView = ARView(frame: .zero)
|
||||
|
||||
private let cameraAnchor = AnchorEntity(world: .zero)
|
||||
private let camera = PerspectiveCamera()
|
||||
private let worldAnchor = AnchorEntity(world: .zero)
|
||||
|
||||
private let keyLight = DirectionalLight()
|
||||
private let fillLight = DirectionalLight()
|
||||
private let rimLight = DirectionalLight()
|
||||
|
||||
private(set) var videoQuad: VideoQuad?
|
||||
private(set) var skeleton: SkeletonEntity?
|
||||
private(set) var mesh: MeshEntity?
|
||||
private(set) var handFace: HandFaceSkeleton?
|
||||
|
||||
/// Orbital camera state.
|
||||
private var orbitYaw: Float = 0
|
||||
private var orbitPitch: Float = 0
|
||||
private var orbitRadius: Float = 3.0
|
||||
|
||||
private var didSetUp = false
|
||||
|
||||
func setUp() {
|
||||
guard !didSetUp else { return }
|
||||
didSetUp = true
|
||||
arView.environment.background = .color(.clear)
|
||||
arView.layer?.isOpaque = false
|
||||
arView.layer?.backgroundColor = NSColor.clear.cgColor
|
||||
arView.scene.addAnchor(worldAnchor)
|
||||
|
||||
keyLight.light.intensity = 4000
|
||||
keyLight.orientation = simd_quatf(angle: .pi / 6,
|
||||
axis: SIMD3(1, 0, 0))
|
||||
let keyA = AnchorEntity(world: SIMD3<Float>(1, 2, -1))
|
||||
keyA.addChild(keyLight)
|
||||
arView.scene.addAnchor(keyA)
|
||||
|
||||
fillLight.light.intensity = 1500
|
||||
fillLight.light.color = NSColor(red: 0.7, green: 0.8,
|
||||
blue: 1.0, alpha: 1.0)
|
||||
let fillA = AnchorEntity(world: SIMD3<Float>(-2, 1, -2))
|
||||
fillA.addChild(fillLight)
|
||||
arView.scene.addAnchor(fillA)
|
||||
|
||||
rimLight.light.intensity = 2000
|
||||
let rimA = AnchorEntity(world: SIMD3<Float>(0, 1, -5))
|
||||
rimA.addChild(rimLight)
|
||||
arView.scene.addAnchor(rimA)
|
||||
|
||||
camera.camera.fieldOfViewInDegrees = 55
|
||||
cameraAnchor.addChild(camera)
|
||||
arView.scene.addAnchor(cameraAnchor)
|
||||
applyCamera()
|
||||
|
||||
let q = VideoQuad()
|
||||
worldAnchor.addChild(q.entity)
|
||||
videoQuad = q
|
||||
|
||||
let s = SkeletonEntity()
|
||||
worldAnchor.addChild(s.root)
|
||||
skeleton = s
|
||||
|
||||
let m = MeshEntity()
|
||||
worldAnchor.addChild(m.root)
|
||||
mesh = m
|
||||
|
||||
let hf = HandFaceSkeleton()
|
||||
worldAnchor.addChild(hf.root)
|
||||
handFace = hf
|
||||
|
||||
installOrbitGestures()
|
||||
}
|
||||
|
||||
func updateVideo(_ pixelBuffer: CVPixelBuffer) {
|
||||
videoQuad?.update(pixelBuffer)
|
||||
}
|
||||
|
||||
func updateSkeleton(_ skeletons: [Int: SkeletonPayload]) {
|
||||
skeleton?.update(skeletons)
|
||||
}
|
||||
|
||||
func updateMesh(_ persons: [MultiHMRPerson]) {
|
||||
mesh?.update(persons)
|
||||
}
|
||||
|
||||
func updateHandFace(hands: HandsPayload?, face: FacePayload?,
|
||||
depth: Float) {
|
||||
handFace?.setDepth(depth)
|
||||
handFace?.update(hands: hands, face: face)
|
||||
}
|
||||
|
||||
func setSkeletonVisible(_ v: Bool) { skeleton?.setVisible(v) }
|
||||
func setMeshVisible(_ v: Bool) { mesh?.setVisible(v) }
|
||||
func setVideoVisible(_ v: Bool) { videoQuad?.setVisible(v) }
|
||||
func setVideoOpacity(_ v: Double) { videoQuad?.setOpacity(v) }
|
||||
|
||||
func updateMeshMaterial(metallic: Bool, roughness: Double) {
|
||||
mesh?.setMaterial(metallic: metallic, roughness: roughness)
|
||||
}
|
||||
|
||||
func setLightIntensities(key: Double, fill: Double, rim: Double) {
|
||||
keyLight.light.intensity = Float(key)
|
||||
fillLight.light.intensity = Float(fill)
|
||||
rimLight.light.intensity = Float(rim)
|
||||
}
|
||||
|
||||
func setFieldOfView(_ deg: Double) {
|
||||
camera.camera.fieldOfViewInDegrees = Float(deg)
|
||||
}
|
||||
|
||||
// MARK: - Orbital camera
|
||||
|
||||
private func applyCamera() {
|
||||
let cy = cos(orbitYaw), sy = sin(orbitYaw)
|
||||
let cp = cos(orbitPitch), sp = sin(orbitPitch)
|
||||
let pos = SIMD3<Float>(orbitRadius * cp * sy,
|
||||
orbitRadius * sp,
|
||||
orbitRadius * cp * cy)
|
||||
cameraAnchor.transform.translation = pos
|
||||
camera.look(at: .zero, from: pos, relativeTo: nil)
|
||||
}
|
||||
|
||||
private func installOrbitGestures() {
|
||||
let pan = NSPanGestureRecognizer(
|
||||
target: OrbitTarget.shared, action: #selector(
|
||||
OrbitTarget.handlePan(_:)))
|
||||
OrbitTarget.shared.controller = self
|
||||
arView.addGestureRecognizer(pan)
|
||||
}
|
||||
|
||||
fileprivate func orbit(dx: Float, dy: Float) {
|
||||
orbitYaw += dx * 0.01
|
||||
orbitPitch = max(-1.4, min(1.4, orbitPitch + dy * 0.01))
|
||||
applyCamera()
|
||||
}
|
||||
}
|
||||
|
||||
/// Bridges the AppKit pan gesture to `SceneController.orbit`.
|
||||
final class OrbitTarget: NSObject {
|
||||
static let shared = OrbitTarget()
|
||||
weak var controller: SceneController?
|
||||
private var last: CGPoint = .zero
|
||||
|
||||
@objc func handlePan(_ g: NSPanGestureRecognizer) {
|
||||
switch g.state {
|
||||
case .began:
|
||||
last = g.translation(in: g.view)
|
||||
case .changed:
|
||||
let p = g.translation(in: g.view)
|
||||
let dx = Float(p.x - last.x)
|
||||
let dy = Float(p.y - last.y)
|
||||
last = p
|
||||
MainActor.assumeIsolated {
|
||||
self.controller?.orbit(dx: dx, dy: -dy)
|
||||
}
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import Foundation
|
||||
import Metal
|
||||
import MetalKit
|
||||
|
||||
/// Metal renderer for the 10 background viz modes (storm, tunnel,
|
||||
/// plasma, kaleido, voronoi, metaballs, starfield, bars, hands3d,
|
||||
/// openpos). Compiles the bundled scene.metal at runtime; sits as the
|
||||
/// backing layer under the transparent ARView.
|
||||
final class SceneRenderer: NSObject, MTKViewDelegate {
|
||||
/// Mirror of scene.metal SceneUniforms (36 floats, 144 B). Field
|
||||
/// order MUST match the shader exactly.
|
||||
struct SceneUniforms {
|
||||
var time: Float = 0
|
||||
var rms: Float = 0
|
||||
var kp_norm: Float = 0
|
||||
var netz_dev: Float = 0
|
||||
var lightning_flash: Float = 0
|
||||
var flare: Float = 0
|
||||
var wind_norm: Float = 0
|
||||
var bz_norm: Float = 0
|
||||
var social_rate: Float = 0
|
||||
var pose_alive: Float = 0
|
||||
var pose_count: Float = 0
|
||||
var width: Float = 1280
|
||||
var height: Float = 720
|
||||
var viz_mode: Float = 0
|
||||
var hand_l_x: Float = 0
|
||||
var hand_l_y: Float = 0
|
||||
var hand_r_x: Float = 0
|
||||
var hand_r_y: Float = 0
|
||||
var mouth_open: Float = 0
|
||||
var eye_open_l: Float = 0
|
||||
var eye_open_r: Float = 0
|
||||
var head_tilt: Float = 0
|
||||
var head_yaw: Float = 0
|
||||
var finger_pinch_l: Float = 0
|
||||
var finger_pinch_r: Float = 0
|
||||
var body_x: Float = 0
|
||||
var body_y: Float = 0
|
||||
var body_z: Float = 0
|
||||
var body_height: Float = 0
|
||||
var arm_spread: Float = 0
|
||||
var pose_velocity: Float = 0
|
||||
var _pad0: Float = 0
|
||||
var _pad1: Float = 0
|
||||
var _pad2: Float = 0
|
||||
var _pad3: Float = 0
|
||||
var _pad4: Float = 0
|
||||
}
|
||||
|
||||
private let device: MTLDevice
|
||||
private let commandQueue: MTLCommandQueue
|
||||
private let bgPipeline: MTLRenderPipelineState
|
||||
private let uniformsBuffer: MTLBuffer
|
||||
private var startTime: CFTimeInterval = CACurrentMediaTime()
|
||||
|
||||
var uniforms = SceneUniforms()
|
||||
|
||||
static func make() -> SceneRenderer? { SceneRenderer(failable: ()) }
|
||||
|
||||
private init?(failable: Void) {
|
||||
guard let dev = MTLCreateSystemDefaultDevice(),
|
||||
let queue = dev.makeCommandQueue() else { return nil }
|
||||
self.device = dev
|
||||
self.commandQueue = queue
|
||||
guard let url = Bundle.main.url(forResource: "scene",
|
||||
withExtension: "metal"),
|
||||
let source = try? String(contentsOf: url, encoding: .utf8) else {
|
||||
NSLog("SceneRenderer: scene.metal missing from bundle")
|
||||
return nil
|
||||
}
|
||||
let lib: MTLLibrary
|
||||
do { lib = try dev.makeLibrary(source: source,
|
||||
options: MTLCompileOptions()) }
|
||||
catch { NSLog("SceneRenderer: compile error %@",
|
||||
String(describing: error)); return nil }
|
||||
guard let vfn = lib.makeFunction(name: "bg_vertex"),
|
||||
let ffn = lib.makeFunction(name: "bg_fragment") else {
|
||||
NSLog("SceneRenderer: bg_vertex/bg_fragment missing")
|
||||
return nil
|
||||
}
|
||||
let pd = MTLRenderPipelineDescriptor()
|
||||
pd.vertexFunction = vfn
|
||||
pd.fragmentFunction = ffn
|
||||
pd.colorAttachments[0].pixelFormat = .bgra8Unorm
|
||||
do { self.bgPipeline =
|
||||
try dev.makeRenderPipelineState(descriptor: pd) }
|
||||
catch { NSLog("SceneRenderer: pipeline failed %@",
|
||||
String(describing: error)); return nil }
|
||||
guard let buf = dev.makeBuffer(
|
||||
length: MemoryLayout<SceneUniforms>.stride,
|
||||
options: .storageModeShared) else { return nil }
|
||||
self.uniformsBuffer = buf
|
||||
super.init()
|
||||
}
|
||||
|
||||
func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
|
||||
uniforms.width = Float(size.width)
|
||||
uniforms.height = Float(size.height)
|
||||
}
|
||||
|
||||
func draw(in view: MTKView) {
|
||||
uniforms.time = Float(CACurrentMediaTime() - startTime)
|
||||
uniformsBuffer.contents().bindMemory(
|
||||
to: SceneUniforms.self, capacity: 1).pointee = uniforms
|
||||
guard let rpd = view.currentRenderPassDescriptor,
|
||||
let drawable = view.currentDrawable,
|
||||
let cb = commandQueue.makeCommandBuffer(),
|
||||
let enc = cb.makeRenderCommandEncoder(descriptor: rpd)
|
||||
else { return }
|
||||
enc.setRenderPipelineState(bgPipeline)
|
||||
enc.setFragmentBuffer(uniformsBuffer, offset: 0, index: 0)
|
||||
enc.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3)
|
||||
enc.endEncoding()
|
||||
cb.present(drawable)
|
||||
cb.commit()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// avlivebody-mac/Sources/AVLiveBody/SceneUniformBuilder.swift
|
||||
import AVLiveWire
|
||||
import simd
|
||||
|
||||
enum SceneUniformBuilder {
|
||||
// ARKit defaultBody3D indices (best-effort constants — ARSkeletonDefinition
|
||||
// is iOS-only and cannot be queried here; verify indices against the
|
||||
// iPhone's 91-joint order if channels read zero unexpectedly; the j()
|
||||
// guard degrades any out-of-range or invalid joint to 0 gracefully).
|
||||
private static let hipsIdx = 0
|
||||
private static let headIdx = 51
|
||||
private static let lWristIdx = 32
|
||||
private static let rWristIdx = 7
|
||||
|
||||
static func fill(_ u: inout SceneRenderer.SceneUniforms,
|
||||
skeletons: [Int: SkeletonPayload],
|
||||
hands: HandsPayload?,
|
||||
face: FacePayload?,
|
||||
prevPelvis: inout SIMD3<Float>?) {
|
||||
u.pose_count = Float(skeletons.count)
|
||||
u.pose_alive = skeletons.isEmpty ? 0 : 1
|
||||
|
||||
if let body = skeletons.values.first {
|
||||
func j(_ i: Int) -> SIMD3<Float>? {
|
||||
(i < body.joints.count && i < body.valid.count
|
||||
&& body.valid[i]) ? body.joints[i] : nil
|
||||
}
|
||||
if let hips = j(hipsIdx) {
|
||||
u.body_x = hips.x; u.body_y = hips.y; u.body_z = hips.z
|
||||
if let prev = prevPelvis {
|
||||
let v = simd_length(hips - prev)
|
||||
u.pose_velocity = u.pose_velocity * 0.7 + v * 0.3
|
||||
}
|
||||
prevPelvis = hips
|
||||
if let head = j(headIdx) {
|
||||
u.body_height = abs(head.y - hips.y)
|
||||
}
|
||||
}
|
||||
if let lw = j(lWristIdx), let rw = j(rWristIdx) {
|
||||
u.arm_spread = abs(lw.x - rw.x)
|
||||
}
|
||||
}
|
||||
|
||||
// Hands (Phase-0 Vision, normalized [0,1]).
|
||||
u.hand_l_x = 0; u.hand_l_y = 0; u.hand_r_x = 0; u.hand_r_y = 0
|
||||
u.finger_pinch_l = 0; u.finger_pinch_r = 0
|
||||
for hand in hands?.hands ?? [] {
|
||||
let c = centroid(hand.points) // (x,y)
|
||||
let pinch = dist(hand.points, 4, 8) // thumb_tip..index_tip
|
||||
if hand.isRight {
|
||||
u.hand_r_x = c.x; u.hand_r_y = c.y; u.finger_pinch_r = pinch
|
||||
} else {
|
||||
u.hand_l_x = c.x; u.hand_l_y = c.y; u.finger_pinch_l = pinch
|
||||
}
|
||||
}
|
||||
|
||||
// Face (Phase-0 Vision allPoints). Region-based derivation is
|
||||
// deferred to v2 — the Vision face-region index mapping is not yet
|
||||
// established. Leave all face scalars at 0 intentionally for v1.
|
||||
if let f = face, f.points.count > 8 {
|
||||
let ys = f.points.map { $0.y }, xs = f.points.map { $0.x }
|
||||
let h = (ys.max() ?? 0) - (ys.min() ?? 0)
|
||||
let w = (xs.max() ?? 0) - (xs.min() ?? 0)
|
||||
u.mouth_open = 0 // set from outer/inner-lip points at impl time
|
||||
u.eye_open_l = 0; u.eye_open_r = 0
|
||||
u.head_tilt = 0; u.head_yaw = 0
|
||||
_ = (h, w)
|
||||
}
|
||||
}
|
||||
|
||||
private static func centroid(_ p: [SIMD3<Float>]) -> SIMD2<Float> {
|
||||
guard !p.isEmpty else { return .zero }
|
||||
var s = SIMD2<Float>(0, 0)
|
||||
for v in p { s.x += v.x; s.y += v.y }
|
||||
return s / Float(p.count)
|
||||
}
|
||||
private static func dist(_ p: [SIMD3<Float>], _ a: Int, _ b: Int)
|
||||
-> Float {
|
||||
guard a < p.count, b < p.count else { return 0 }
|
||||
return simd_length(SIMD2(p[a].x, p[a].y) - SIMD2(p[b].x, p[b].y))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
// avlivebody-mac/Sources/AVLiveBody/SettingsPanel.swift
|
||||
import SwiftUI
|
||||
|
||||
/// Collapsible right-side control panel bound to RenderSettings.
|
||||
struct SettingsPanel: View {
|
||||
@ObservedObject var settings: RenderSettings
|
||||
|
||||
var body: some View {
|
||||
ScrollView {
|
||||
VStack(alignment: .leading, spacing: 14) {
|
||||
Text("Reglages").font(.headline)
|
||||
|
||||
group("Couches") {
|
||||
Toggle("Squelette", isOn: $settings.showSkeleton)
|
||||
Toggle("Mesh", isOn: $settings.showMesh)
|
||||
Toggle("Video", isOn: $settings.showVideo)
|
||||
}
|
||||
|
||||
group("Mesh") {
|
||||
Toggle("Metallique", isOn: $settings.meshMetallic)
|
||||
slider("Rugosite", $settings.meshRoughness,
|
||||
0...1)
|
||||
}
|
||||
|
||||
group("Lumieres") {
|
||||
slider("Principale", $settings.keyIntensity,
|
||||
0...10000)
|
||||
slider("Remplissage", $settings.fillIntensity,
|
||||
0...10000)
|
||||
slider("Contre-jour", $settings.rimIntensity,
|
||||
0...10000)
|
||||
}
|
||||
|
||||
group("Vue") {
|
||||
slider("Champ de vision", $settings.fieldOfView,
|
||||
20...120)
|
||||
slider("Opacite video", $settings.videoOpacity,
|
||||
0...1)
|
||||
}
|
||||
|
||||
group("Scene") {
|
||||
Toggle("Fond shader", isOn: $settings.showScene)
|
||||
Picker("Mode", selection: $settings.vizMode) {
|
||||
ForEach(0..<10, id: \.self) { i in
|
||||
Text(Self.modeName(i)).tag(i)
|
||||
}
|
||||
}
|
||||
.pickerStyle(.menu)
|
||||
}
|
||||
}
|
||||
.padding(16)
|
||||
}
|
||||
.frame(width: 280)
|
||||
.background(.ultraThinMaterial)
|
||||
}
|
||||
|
||||
private static let modeNames = ["storm","tunnel","plasma","kaleido",
|
||||
"voronoi","metaballs","starfield","bars","hands3d","openpos"]
|
||||
static func modeName(_ i: Int) -> String {
|
||||
modeNames.indices.contains(i) ? "\(i) \(modeNames[i])" : "\(i)"
|
||||
}
|
||||
|
||||
@ViewBuilder
|
||||
private func group(_ title: String,
|
||||
@ViewBuilder _ content: () -> some View)
|
||||
-> some View {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
Text(title).font(.subheadline).foregroundStyle(.secondary)
|
||||
content()
|
||||
}
|
||||
}
|
||||
|
||||
private func slider(_ label: String,
|
||||
_ value: Binding<Double>,
|
||||
_ range: ClosedRange<Double>) -> some View {
|
||||
VStack(alignment: .leading, spacing: 2) {
|
||||
HStack {
|
||||
Text(label).font(.caption)
|
||||
Spacer()
|
||||
Text(String(format: "%.2f", value.wrappedValue))
|
||||
.font(.caption).foregroundStyle(.secondary)
|
||||
}
|
||||
Slider(value: value, in: range)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import AVLiveWire
|
||||
import AppKit
|
||||
import Foundation
|
||||
import RealityKit
|
||||
import simd
|
||||
|
||||
/// Renders 91-joint skeletons as yellow marker spheres. One marker
|
||||
/// pool per pid. ARKit world coords -> RealityKit space (x, -y, -z).
|
||||
@MainActor
|
||||
final class SkeletonEntity {
|
||||
let root = Entity()
|
||||
|
||||
private static let jointCount = 91
|
||||
private static let markerRadius: Float = 0.012
|
||||
|
||||
private var pools: [Int: [ModelEntity]] = [:]
|
||||
private let mesh = MeshResource.generateSphere(radius: markerRadius)
|
||||
private let material = SimpleMaterial(
|
||||
color: NSColor.systemYellow, roughness: 0.6, isMetallic: false)
|
||||
|
||||
func setVisible(_ visible: Bool) { root.isEnabled = visible }
|
||||
|
||||
func update(_ skeletons: [Int: SkeletonPayload]) {
|
||||
// Drop pools for pids no longer present.
|
||||
for pid in pools.keys where skeletons[pid] == nil {
|
||||
pools[pid]?.forEach { $0.removeFromParent() }
|
||||
pools.removeValue(forKey: pid)
|
||||
}
|
||||
for (pid, payload) in skeletons {
|
||||
let pool = pools[pid] ?? makePool()
|
||||
pools[pid] = pool
|
||||
let n = min(Self.jointCount, payload.joints.count,
|
||||
payload.valid.count)
|
||||
for i in 0..<n {
|
||||
let marker = pool[i]
|
||||
if payload.valid[i] {
|
||||
let j = payload.joints[i]
|
||||
marker.transform.translation = arkitToRealityKit(j)
|
||||
marker.isEnabled = true
|
||||
} else {
|
||||
marker.isEnabled = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func makePool() -> [ModelEntity] {
|
||||
var pool: [ModelEntity] = []
|
||||
pool.reserveCapacity(Self.jointCount)
|
||||
for _ in 0..<Self.jointCount {
|
||||
let e = ModelEntity(mesh: mesh, materials: [material])
|
||||
e.isEnabled = false
|
||||
root.addChild(e)
|
||||
pool.append(e)
|
||||
}
|
||||
return pool
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import SwiftUI
|
||||
|
||||
/// A thin overlay showing the USB connection state.
|
||||
struct StatusBar: View {
|
||||
@ObservedObject var consumer: USBSkeletonConsumer
|
||||
|
||||
var body: some View {
|
||||
HStack(spacing: 6) {
|
||||
Circle()
|
||||
.fill(consumer.connected ? Color.green : Color.orange)
|
||||
.frame(width: 9, height: 9)
|
||||
Text(consumer.connected
|
||||
? "iPhone connected (USB)"
|
||||
: "waiting for iPhone…")
|
||||
.font(.caption)
|
||||
.foregroundStyle(.white)
|
||||
Spacer()
|
||||
}
|
||||
.padding(8)
|
||||
.background(.black.opacity(0.5))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import AppKit
|
||||
import CoreImage
|
||||
import CoreVideo
|
||||
import Foundation
|
||||
import RealityKit
|
||||
|
||||
/// A flat plane at the back of the scene, textured with the iPhone
|
||||
/// camera video. `update(_:)` is called on the main queue per frame.
|
||||
@MainActor
|
||||
final class VideoQuad {
|
||||
let entity = ModelEntity()
|
||||
|
||||
private let ciContext = CIContext()
|
||||
private var opacity: Double = 1.0
|
||||
/// Plane is 1.6 m wide, 16:9; positioned 2 m behind the body.
|
||||
private static let width: Float = 1.6
|
||||
private static let height: Float = 0.9
|
||||
private static let zBack: Float = -2.0
|
||||
|
||||
init() {
|
||||
let plane = MeshResource.generatePlane(
|
||||
width: Self.width, height: Self.height)
|
||||
var material = UnlitMaterial()
|
||||
material.color = .init(tint: .white)
|
||||
entity.model = ModelComponent(mesh: plane,
|
||||
materials: [material])
|
||||
entity.transform.translation =
|
||||
SIMD3<Float>(0, 0, Self.zBack)
|
||||
}
|
||||
|
||||
func setVisible(_ visible: Bool) { entity.isEnabled = visible }
|
||||
|
||||
func setOpacity(_ value: Double) {
|
||||
opacity = value
|
||||
// Re-tint current material; `update(_:)` will also use `opacity`.
|
||||
if var mat = entity.model?.materials.first as? UnlitMaterial {
|
||||
mat.color = .init(tint: NSColor(white: 1, alpha: CGFloat(value)))
|
||||
entity.model?.materials = [mat]
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the plane's texture from a decoded camera frame.
|
||||
func update(_ pixelBuffer: CVPixelBuffer) {
|
||||
let ci = CIImage(cvPixelBuffer: pixelBuffer)
|
||||
guard let cg = ciContext.createCGImage(
|
||||
ci, from: ci.extent) else { return }
|
||||
guard let texture = try? TextureResource(
|
||||
image: cg, options: .init(semantic: .color)) else {
|
||||
NSLog("VideoQuad: TextureResource creation failed (%dx%d)",
|
||||
CVPixelBufferGetWidth(pixelBuffer),
|
||||
CVPixelBufferGetHeight(pixelBuffer))
|
||||
return
|
||||
}
|
||||
var material = UnlitMaterial()
|
||||
material.color = .init(
|
||||
tint: NSColor(white: 1, alpha: CGFloat(opacity)),
|
||||
texture: .init(texture))
|
||||
entity.model?.materials = [material]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import AVLiveWire
|
||||
import Foundation
|
||||
import simd
|
||||
|
||||
/// Overrides the highest-scoring Multi-HMR mesh's pelvis depth with
|
||||
/// the first valid USB skeleton pelvis z. Single-person assumption:
|
||||
/// with multiple skeletons in the dict the source pelvis is arbitrary
|
||||
/// (dict iteration order). Pure, stateless — unit-testable.
|
||||
enum BodyFusion {
|
||||
/// ARSkeleton3D joint 0 = root (hips), per ARSkeletonDefinition.defaultBody3D.
|
||||
static let pelvisJoint = 0
|
||||
|
||||
static func fuse(persons: [MultiHMRPerson],
|
||||
skeletons: [Int: SkeletonPayload])
|
||||
-> [MultiHMRPerson] {
|
||||
let pelvisZs: [Float] = skeletons.values.compactMap { s in
|
||||
guard pelvisJoint < s.valid.count,
|
||||
s.valid[pelvisJoint] else { return nil }
|
||||
return s.joints[pelvisJoint].z
|
||||
}
|
||||
guard !pelvisZs.isEmpty,
|
||||
let primaryIdx = persons.indices.max(by: {
|
||||
persons[$0].score < persons[$1].score
|
||||
}) else { return persons }
|
||||
var out = persons
|
||||
out[primaryIdx].translation.z = pelvisZs[0]
|
||||
return out
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import CoreML
|
||||
import CoreVideo
|
||||
import CoreImage
|
||||
import Foundation
|
||||
|
||||
/// One detected SMPL-X body from Multi-HMR.
|
||||
struct MultiHMRPerson {
|
||||
var vertices: [SIMD3<Float>] // 10475 SMPL-X verts, model space
|
||||
var translation: SIMD3<Float> // pelvis translation
|
||||
var score: Float
|
||||
}
|
||||
|
||||
/// CoreML wrapper around the bundled `multihmr_full_672_s.mlpackage`.
|
||||
/// Mirrors `data_only_viz/multihmr_coreml.py`: two MLMultiArray inputs
|
||||
/// (`image` 1x3x672x672 ImageNet-normalized, `cam_K` 1x3x3), fixed
|
||||
/// K=4 person outputs.
|
||||
final class MultiHMRCoreML {
|
||||
static let inputSize = 672
|
||||
static let vertexCount = 10475
|
||||
static let maxPersons = 4
|
||||
private static let detThreshold: Float = 0.3
|
||||
private static let normMean: [Float] = [0.485, 0.456, 0.406]
|
||||
private static let normStd: [Float] = [0.229, 0.224, 0.225]
|
||||
|
||||
private let model: MLModel
|
||||
private let ciContext = CIContext()
|
||||
/// Serial background queue + busy flag for `inferAsync`: keep the
|
||||
/// ~150-300 ms CoreML prediction off the main thread and drop frames
|
||||
/// that arrive while one is in flight.
|
||||
private let inferQueue = DispatchQueue(
|
||||
label: "cc.saillant.avlivebody.multihmr", qos: .userInitiated)
|
||||
private let inferLock = NSLock()
|
||||
private var inferBusy = false
|
||||
|
||||
/// Loads the bundled model. Returns nil if the resource or load
|
||||
/// fails — callers fall back to skeleton-only rendering.
|
||||
init?() {
|
||||
guard let url = Bundle.main.url(
|
||||
forResource: "multihmr_full_672_s",
|
||||
withExtension: "mlmodelc") else {
|
||||
NSLog("MultiHMRCoreML: mlpackage resource missing")
|
||||
return nil
|
||||
}
|
||||
let cfg = MLModelConfiguration()
|
||||
cfg.computeUnits = .cpuAndGPU
|
||||
do {
|
||||
model = try MLModel(contentsOf: url, configuration: cfg)
|
||||
} catch {
|
||||
NSLog("MultiHMRCoreML: load failed %@",
|
||||
String(describing: error))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Async inference: runs `infer` on a background queue, DROPS the
|
||||
/// frame if a prior prediction is still in flight, and calls
|
||||
/// `completion` with the result ON THE BACKGROUND QUEUE (the caller
|
||||
/// hops back to the main thread for UI). Prevents the per-frame
|
||||
/// CoreML prediction from saturating the run loop.
|
||||
func inferAsync(_ pixelBuffer: CVPixelBuffer,
|
||||
cameraK: [Float],
|
||||
completion: @escaping ([MultiHMRPerson]) -> Void) {
|
||||
inferLock.lock()
|
||||
if inferBusy { inferLock.unlock(); return }
|
||||
inferBusy = true
|
||||
inferLock.unlock()
|
||||
inferQueue.async { [weak self] in
|
||||
guard let self else { return }
|
||||
let result = self.infer(pixelBuffer, cameraK: cameraK)
|
||||
self.inferLock.lock()
|
||||
self.inferBusy = false
|
||||
self.inferLock.unlock()
|
||||
completion(result)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run inference on one camera frame. `cameraK` is the 3x3 camera
|
||||
/// intrinsics row-major.
|
||||
func infer(_ pixelBuffer: CVPixelBuffer,
|
||||
cameraK: [Float]) -> [MultiHMRPerson] {
|
||||
guard let image = makeImageInput(pixelBuffer),
|
||||
let k = makeKInput(cameraK) else { return [] }
|
||||
let inputs: [String: MLFeatureValue] = [
|
||||
"image": MLFeatureValue(multiArray: image),
|
||||
"cam_K": MLFeatureValue(multiArray: k),
|
||||
]
|
||||
guard let provider = try? MLDictionaryFeatureProvider(
|
||||
dictionary: inputs),
|
||||
let out = try? model.prediction(from: provider) else {
|
||||
return []
|
||||
}
|
||||
return parse(out)
|
||||
}
|
||||
|
||||
// MARK: - Input preprocessing
|
||||
|
||||
/// `CVPixelBuffer` -> [1,3,672,672] Float32, RGB, ImageNet-normed.
|
||||
private func makeImageInput(_ pb: CVPixelBuffer) -> MLMultiArray? {
|
||||
let n = Self.inputSize
|
||||
// Resize to n x n BGRA via CoreImage.
|
||||
let ci = CIImage(cvPixelBuffer: pb)
|
||||
let sx = CGFloat(n) / ci.extent.width
|
||||
let sy = CGFloat(n) / ci.extent.height
|
||||
let scaled = ci.transformed(
|
||||
by: CGAffineTransform(scaleX: sx, y: sy))
|
||||
var dst: CVPixelBuffer?
|
||||
CVPixelBufferCreate(kCFAllocatorDefault, n, n,
|
||||
kCVPixelFormatType_32BGRA, nil, &dst)
|
||||
guard let dst else { return nil }
|
||||
ciContext.render(scaled, to: dst)
|
||||
CVPixelBufferLockBaseAddress(dst, .readOnly)
|
||||
defer { CVPixelBufferUnlockBaseAddress(dst, .readOnly) }
|
||||
guard let base = CVPixelBufferGetBaseAddress(dst) else {
|
||||
return nil
|
||||
}
|
||||
let rowBytes = CVPixelBufferGetBytesPerRow(dst)
|
||||
let px = base.assumingMemoryBound(to: UInt8.self)
|
||||
guard let arr = try? MLMultiArray(
|
||||
shape: [1, 3, NSNumber(value: n), NSNumber(value: n)],
|
||||
dataType: .float32) else { return nil }
|
||||
let ptr = arr.dataPointer.assumingMemoryBound(to: Float.self)
|
||||
let plane = n * n
|
||||
for y in 0..<n {
|
||||
for x in 0..<n {
|
||||
let p = y * rowBytes + x * 4 // BGRA
|
||||
let b = Float(px[p]) / 255.0
|
||||
let g = Float(px[p + 1]) / 255.0
|
||||
let r = Float(px[p + 2]) / 255.0
|
||||
let idx = y * n + x
|
||||
ptr[idx] =
|
||||
(r - Self.normMean[0]) / Self.normStd[0]
|
||||
ptr[plane + idx] =
|
||||
(g - Self.normMean[1]) / Self.normStd[1]
|
||||
ptr[2 * plane + idx] =
|
||||
(b - Self.normMean[2]) / Self.normStd[2]
|
||||
}
|
||||
}
|
||||
return arr
|
||||
}
|
||||
|
||||
/// 9 row-major intrinsics -> [1,3,3] Float32.
|
||||
private func makeKInput(_ k: [Float]) -> MLMultiArray? {
|
||||
guard k.count == 9,
|
||||
let arr = try? MLMultiArray(
|
||||
shape: [1, 3, 3], dataType: .float32) else { return nil }
|
||||
let ptr = arr.dataPointer.assumingMemoryBound(to: Float.self)
|
||||
for i in 0..<9 { ptr[i] = k[i] }
|
||||
return arr
|
||||
}
|
||||
|
||||
// MARK: - Output parsing
|
||||
|
||||
private func parse(_ out: MLFeatureProvider) -> [MultiHMRPerson] {
|
||||
guard let v3d = out.featureValue(for: "var_2420")?
|
||||
.multiArrayValue,
|
||||
let transl = out.featureValue(for: "var_2423")?
|
||||
.multiArrayValue,
|
||||
let scores = out.featureValue(for: "var_2436")?
|
||||
.multiArrayValue else { return [] }
|
||||
var persons: [MultiHMRPerson] = []
|
||||
let vc = Self.vertexCount
|
||||
for k in 0..<Self.maxPersons {
|
||||
let score = scores[k].floatValue
|
||||
if score < Self.detThreshold { continue }
|
||||
var verts = [SIMD3<Float>](
|
||||
repeating: .zero, count: vc)
|
||||
let base = k * vc * 3
|
||||
for i in 0..<vc {
|
||||
let o = base + i * 3
|
||||
verts[i] = SIMD3(v3d[o].floatValue,
|
||||
v3d[o + 1].floatValue,
|
||||
v3d[o + 2].floatValue)
|
||||
}
|
||||
let tb = k * 3
|
||||
persons.append(MultiHMRPerson(
|
||||
vertices: verts,
|
||||
translation: SIMD3(transl[tb].floatValue,
|
||||
transl[tb + 1].floatValue,
|
||||
transl[tb + 2].floatValue),
|
||||
score: score))
|
||||
}
|
||||
return persons
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import Foundation
|
||||
import Darwin
|
||||
|
||||
/// Transport abstraction over the usbmuxd Unix socket. The real
|
||||
/// implementation wraps a `socket(AF_UNIX)`; tests inject a mock.
|
||||
protocol MuxTransport {
|
||||
func send(_ data: Data)
|
||||
func receivePacket() -> Data?
|
||||
func close()
|
||||
}
|
||||
|
||||
/// usbmux client: device discovery + connect-to-port. After a
|
||||
/// successful `connect`, the same transport carries the raw tunneled
|
||||
/// byte stream from the device.
|
||||
final class USBClient {
|
||||
private let transport: MuxTransport
|
||||
private var tag: UInt32 = 0
|
||||
|
||||
init(transport: MuxTransport) {
|
||||
self.transport = transport
|
||||
}
|
||||
|
||||
func listDevices() -> [Int] {
|
||||
tag += 1
|
||||
transport.send(USBMuxProtocol.encode(
|
||||
plist: ["MessageType": "ListDevices"], tag: tag))
|
||||
guard let reply = transport.receivePacket(),
|
||||
let plist = USBMuxProtocol.decode(reply),
|
||||
let list = plist["DeviceList"] as? [[String: Any]]
|
||||
else { return [] }
|
||||
return list.compactMap { $0["DeviceID"] as? Int }
|
||||
}
|
||||
|
||||
/// Returns true once the transport is tunneled to `port` on the
|
||||
/// device. usbmux wants the TCP port in big-endian order.
|
||||
func connect(deviceID: Int, port: UInt16) -> Bool {
|
||||
tag += 1
|
||||
let swapped = Int((port << 8) | (port >> 8))
|
||||
transport.send(USBMuxProtocol.encode(plist: [
|
||||
"MessageType": "Connect",
|
||||
"DeviceID": deviceID,
|
||||
"PortNumber": swapped,
|
||||
], tag: tag))
|
||||
guard let reply = transport.receivePacket(),
|
||||
let plist = USBMuxProtocol.decode(reply),
|
||||
let number = plist["Number"] as? Int
|
||||
else { return false }
|
||||
return number == 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Production transport: blocking AF_UNIX socket to usbmuxd.
|
||||
final class UnixMuxTransport: MuxTransport {
|
||||
private var fd: Int32 = -1
|
||||
|
||||
init?(path: String = "/var/run/usbmuxd") {
|
||||
fd = socket(AF_UNIX, SOCK_STREAM, 0)
|
||||
guard fd >= 0 else { return nil }
|
||||
var addr = sockaddr_un()
|
||||
addr.sun_family = sa_family_t(AF_UNIX)
|
||||
precondition(path.utf8.count < 104,
|
||||
"usbmuxd socket path exceeds sun_path limit")
|
||||
_ = path.withCString { src in
|
||||
withUnsafeMutablePointer(to: &addr.sun_path) {
|
||||
$0.withMemoryRebound(to: CChar.self, capacity: 104) {
|
||||
strcpy($0, src)
|
||||
}
|
||||
}
|
||||
}
|
||||
let size = socklen_t(MemoryLayout<sockaddr_un>.size)
|
||||
let ok = withUnsafePointer(to: &addr) {
|
||||
$0.withMemoryRebound(to: sockaddr.self, capacity: 1) {
|
||||
Darwin.connect(fd, $0, size)
|
||||
}
|
||||
}
|
||||
if ok != 0 { Darwin.close(fd); return nil }
|
||||
}
|
||||
|
||||
func send(_ data: Data) {
|
||||
guard fd >= 0 else { return }
|
||||
data.withUnsafeBytes { buf in
|
||||
guard let base = buf.baseAddress else { return }
|
||||
var off = 0
|
||||
while off < data.count {
|
||||
let w = Darwin.write(fd, base.advanced(by: off),
|
||||
data.count - off)
|
||||
if w <= 0 {
|
||||
if w < 0 && errno == EINTR { continue }
|
||||
break
|
||||
}
|
||||
off += w
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read one usbmux packet: 4-byte LE length prefix then body.
|
||||
func receivePacket() -> Data? {
|
||||
guard let head = readN(4) else { return nil }
|
||||
guard let len = USBMuxProtocol.readLE32(head, 0) else { return nil }
|
||||
let total = Int(len)
|
||||
guard total >= 16, let rest = readN(total - 4) else { return nil }
|
||||
return head + rest
|
||||
}
|
||||
|
||||
/// Read raw tunneled bytes after a successful Connect.
|
||||
func readStream(max: Int = 65536) -> Data? {
|
||||
readN(max, exact: false)
|
||||
}
|
||||
|
||||
private func readN(_ n: Int, exact: Bool = true) -> Data? {
|
||||
var buf = [UInt8](repeating: 0, count: n)
|
||||
var got = 0
|
||||
while got < n {
|
||||
let r = buf.withUnsafeMutableBytes {
|
||||
Darwin.read(fd, $0.baseAddress!.advanced(by: got), n - got)
|
||||
}
|
||||
if r < 0 {
|
||||
if errno == EINTR { continue }
|
||||
return got > 0 && !exact ? Data(buf[0..<got]) : nil
|
||||
}
|
||||
if r == 0 { // EOF — peer closed
|
||||
return got > 0 && !exact ? Data(buf[0..<got]) : nil
|
||||
}
|
||||
got += r
|
||||
if !exact { break }
|
||||
}
|
||||
return Data(buf[0..<got])
|
||||
}
|
||||
|
||||
deinit { close() }
|
||||
|
||||
func close() {
|
||||
if fd >= 0 { Darwin.close(fd); fd = -1 }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import Foundation
|
||||
|
||||
/// Codec for the usbmuxd request/response protocol. 16-byte
|
||||
/// little-endian header (length, version=1, message=8, tag) then an
|
||||
/// XML property list.
|
||||
enum USBMuxProtocol {
|
||||
static func encode(plist: [String: Any], tag: UInt32) -> Data {
|
||||
let body = (try? PropertyListSerialization.data(
|
||||
fromPropertyList: plist, format: .xml, options: 0))
|
||||
?? Data()
|
||||
var d = Data()
|
||||
appendLE32(&d, UInt32(16 + body.count)) // length
|
||||
appendLE32(&d, 1) // version
|
||||
appendLE32(&d, 8) // message: plist
|
||||
appendLE32(&d, tag)
|
||||
d.append(body)
|
||||
return d
|
||||
}
|
||||
|
||||
static func decode(_ packet: Data) -> [String: Any]? {
|
||||
guard packet.count >= 16 else { return nil }
|
||||
let body = packet.dropFirst(16)
|
||||
return (try? PropertyListSerialization.propertyList(
|
||||
from: body, options: [], format: nil)) as? [String: Any]
|
||||
}
|
||||
|
||||
static func appendLE32(_ d: inout Data, _ v: UInt32) {
|
||||
for i in 0..<4 { d.append(UInt8((v >> (8 * i)) & 0xFF)) }
|
||||
}
|
||||
|
||||
static func readLE32(_ d: Data, _ offset: Int) -> UInt32? {
|
||||
guard offset >= 0, d.count >= offset + 4 else { return nil }
|
||||
let b = [UInt8](d)
|
||||
var v: UInt32 = 0
|
||||
for i in 0..<4 { v |= UInt32(b[offset + i]) << (8 * i) }
|
||||
return v
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import AVLiveWire
|
||||
import Combine
|
||||
import CoreVideo
|
||||
import Foundation
|
||||
|
||||
/// Connects to the tethered iPhone over USB (usbmuxd), demuxes the
|
||||
/// AVLiveWire stream, republishes skeleton payloads (keyed by pid)
|
||||
/// and forwards decoded camera frames. Blocking transport runs on a
|
||||
/// dedicated background thread; only `@Published` writes hop to main.
|
||||
final class USBSkeletonConsumer: ObservableObject {
|
||||
/// 91-joint skeleton payloads keyed by pid.
|
||||
@Published var skeletons: [Int: SkeletonPayload] = [:]
|
||||
@Published var connected = false
|
||||
/// Latest Vision hand landmarks from the iPhone (tag=4).
|
||||
@Published var hands: HandsPayload?
|
||||
/// Latest Vision face landmarks from the iPhone (tag=5).
|
||||
@Published var face: FacePayload?
|
||||
|
||||
/// Called on the main queue for every decoded camera frame.
|
||||
var onVideoFrame: ((CVPixelBuffer) -> Void)?
|
||||
|
||||
/// TCP port the iPhone `USBServer` listens on.
|
||||
static let devicePort: UInt16 = 7000
|
||||
|
||||
private let videoDecoder = VideoDecoder()
|
||||
private let stateLock = NSLock()
|
||||
private var running = false
|
||||
private var thread: Thread?
|
||||
|
||||
init() {
|
||||
videoDecoder.onFrame = { [weak self] pixelBuffer in
|
||||
DispatchQueue.main.async {
|
||||
self?.onVideoFrame?(pixelBuffer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var isRunning: Bool {
|
||||
stateLock.lock(); defer { stateLock.unlock() }
|
||||
return running
|
||||
}
|
||||
|
||||
func start() {
|
||||
stateLock.lock()
|
||||
if running { stateLock.unlock(); return }
|
||||
running = true
|
||||
let t = Thread { [weak self] in self?.loop() }
|
||||
t.name = "cc.avlive.usbconsumer"
|
||||
thread = t
|
||||
stateLock.unlock()
|
||||
t.start()
|
||||
}
|
||||
|
||||
func stop() {
|
||||
stateLock.lock(); running = false; stateLock.unlock()
|
||||
}
|
||||
|
||||
private func loop() {
|
||||
while isRunning {
|
||||
guard let transport = UnixMuxTransport() else {
|
||||
NSLog("USBSkeletonConsumer: no usbmuxd; retry")
|
||||
Thread.sleep(forTimeInterval: 1.0); continue
|
||||
}
|
||||
let client = USBClient(transport: transport)
|
||||
let devices = client.listDevices()
|
||||
guard let dev = devices.first,
|
||||
client.connect(deviceID: dev,
|
||||
port: Self.devicePort) else {
|
||||
NSLog("USBSkeletonConsumer: no device; retry")
|
||||
transport.close()
|
||||
Thread.sleep(forTimeInterval: 1.0); continue
|
||||
}
|
||||
NSLog("USBSkeletonConsumer: connected to device %d", dev)
|
||||
publishConnected(true)
|
||||
var demux = StreamDemuxer()
|
||||
while isRunning {
|
||||
guard let chunk = transport.readStream(),
|
||||
!chunk.isEmpty else { break }
|
||||
for frame in demux.feed(chunk) { route(frame) }
|
||||
}
|
||||
transport.close()
|
||||
publishConnected(false)
|
||||
NSLog("USBSkeletonConsumer: disconnected")
|
||||
if isRunning { Thread.sleep(forTimeInterval: 1.0) }
|
||||
}
|
||||
}
|
||||
|
||||
private func route(_ frame: StreamDemuxer.Frame) {
|
||||
switch frame.header.tag {
|
||||
case .skeleton:
|
||||
guard let payload =
|
||||
SkeletonPayload(decoding: frame.payload) else { return }
|
||||
let pid = Int(frame.header.pid)
|
||||
DispatchQueue.main.async { [weak self] in
|
||||
self?.skeletons[pid] = payload
|
||||
}
|
||||
case .video:
|
||||
guard let payload =
|
||||
VideoPayload(decoding: frame.payload) else { return }
|
||||
videoDecoder.decode(payload)
|
||||
case .hands:
|
||||
guard let payload = HandsPayload(decoding: frame.payload)
|
||||
else { return }
|
||||
DispatchQueue.main.async { [weak self] in
|
||||
self?.hands = payload
|
||||
}
|
||||
case .face:
|
||||
guard let payload = FacePayload(decoding: frame.payload)
|
||||
else { return }
|
||||
DispatchQueue.main.async { [weak self] in
|
||||
self?.face = payload
|
||||
}
|
||||
case .meta:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
private func publishConnected(_ value: Bool) {
|
||||
DispatchQueue.main.async { [weak self] in
|
||||
self?.connected = value
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import AVLiveWire
|
||||
import CoreMedia
|
||||
import CoreVideo
|
||||
import Foundation
|
||||
import VideoToolbox
|
||||
|
||||
/// HEVC decoder. Feed `VideoPayload`s in; receive `CVPixelBuffer`s via
|
||||
/// `onFrame`. Keyframe payloads must carry the VPS/SPS/PPS parameter
|
||||
/// sets prepended as 4-byte-length-prefixed NAL units (the layout the
|
||||
/// iOS `VideoEncoder` emits); the decoder (re)builds its format
|
||||
/// description from those.
|
||||
final class VideoDecoder {
|
||||
var onFrame: ((CVPixelBuffer) -> Void)?
|
||||
|
||||
private var session: VTDecompressionSession?
|
||||
private var formatDesc: CMVideoFormatDescription?
|
||||
|
||||
/// Decode one access unit.
|
||||
func decode(_ payload: VideoPayload) {
|
||||
var au = payload.data
|
||||
if payload.isKeyframe {
|
||||
let (params, rest) = Self.splitParameterSets(au)
|
||||
if !params.isEmpty {
|
||||
rebuildFormat(params)
|
||||
}
|
||||
au = rest
|
||||
}
|
||||
guard let fmt = formatDesc, !au.isEmpty else { return }
|
||||
if session == nil { makeSession(fmt) }
|
||||
guard let session, let block = Self.blockBuffer(au) else {
|
||||
return
|
||||
}
|
||||
var sample: CMSampleBuffer?
|
||||
var sampleSize = au.count
|
||||
guard CMSampleBufferCreateReady(
|
||||
allocator: kCFAllocatorDefault, dataBuffer: block,
|
||||
formatDescription: fmt, sampleCount: 1,
|
||||
sampleTimingEntryCount: 0, sampleTimingArray: nil,
|
||||
sampleSizeEntryCount: 1, sampleSizeArray: &sampleSize,
|
||||
sampleBufferOut: &sample) == noErr, let sample else {
|
||||
return
|
||||
}
|
||||
VTDecompressionSessionDecodeFrame(
|
||||
session, sampleBuffer: sample, flags: [],
|
||||
infoFlagsOut: nil) { [weak self] status, _, image, _, _ in
|
||||
guard status == noErr, let image else { return }
|
||||
self?.onFrame?(image)
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
if let session { VTDecompressionSessionInvalidate(session) }
|
||||
session = nil
|
||||
formatDesc = nil
|
||||
}
|
||||
|
||||
deinit { stop() }
|
||||
|
||||
// MARK: - Helpers
|
||||
|
||||
/// Leading 4-byte-length-prefixed NAL units of HEVC parameter-set
|
||||
/// type (VPS=32, SPS=33, PPS=34) are split from the frame data.
|
||||
/// Returns (parameterSetData, frameData).
|
||||
private static func splitParameterSets(_ data: Data)
|
||||
-> (Data, Data) {
|
||||
let bytes = [UInt8](data)
|
||||
var offset = 0
|
||||
var paramEnd = 0
|
||||
while offset + 4 <= bytes.count {
|
||||
let len = (Int(bytes[offset]) << 24)
|
||||
| (Int(bytes[offset + 1]) << 16)
|
||||
| (Int(bytes[offset + 2]) << 8)
|
||||
| Int(bytes[offset + 3])
|
||||
let nalStart = offset + 4
|
||||
guard len > 0, nalStart + len <= bytes.count else { break }
|
||||
let nalType = (Int(bytes[nalStart]) >> 1) & 0x3F
|
||||
if nalType == 32 || nalType == 33 || nalType == 34 {
|
||||
offset = nalStart + len
|
||||
paramEnd = offset
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return (data.prefix(paramEnd),
|
||||
data.suffix(from: data.startIndex
|
||||
.advanced(by: paramEnd)))
|
||||
}
|
||||
|
||||
private func rebuildFormat(_ paramData: Data) {
|
||||
var sets: [[UInt8]] = []
|
||||
let bytes = [UInt8](paramData)
|
||||
var offset = 0
|
||||
while offset + 4 <= bytes.count {
|
||||
let len = (Int(bytes[offset]) << 24)
|
||||
| (Int(bytes[offset + 1]) << 16)
|
||||
| (Int(bytes[offset + 2]) << 8)
|
||||
| Int(bytes[offset + 3])
|
||||
let start = offset + 4
|
||||
guard len > 0, start + len <= bytes.count else { break }
|
||||
sets.append(Array(bytes[start..<start + len]))
|
||||
offset = start + len
|
||||
}
|
||||
guard sets.count >= 3 else { return }
|
||||
var fmt: CMFormatDescription?
|
||||
let status = withParameterSetPointers(sets) { pBuf, sBuf in
|
||||
CMVideoFormatDescriptionCreateFromHEVCParameterSets(
|
||||
allocator: kCFAllocatorDefault,
|
||||
parameterSetCount: sets.count,
|
||||
parameterSetPointers: pBuf,
|
||||
parameterSetSizes: sBuf,
|
||||
nalUnitHeaderLength: 4, extensions: nil,
|
||||
formatDescriptionOut: &fmt)
|
||||
}
|
||||
if status == noErr, let fmt {
|
||||
formatDesc = fmt
|
||||
if let session { VTDecompressionSessionInvalidate(session) }
|
||||
session = nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the C-style parallel arrays of parameter-set pointers and
|
||||
/// sizes that `CMVideoFormatDescriptionCreateFromHEVCParameterSets`
|
||||
/// requires, keeping the backing storage alive for the call.
|
||||
private func withParameterSetPointers(
|
||||
_ sets: [[UInt8]],
|
||||
_ body: (UnsafePointer<UnsafePointer<UInt8>>,
|
||||
UnsafePointer<Int>) -> OSStatus) -> OSStatus {
|
||||
func recurse(_ index: Int,
|
||||
_ ptrs: inout [UnsafePointer<UInt8>],
|
||||
_ sizes: inout [Int]) -> OSStatus {
|
||||
if index == sets.count {
|
||||
return ptrs.withUnsafeBufferPointer { pBuf in
|
||||
sizes.withUnsafeBufferPointer { sBuf in
|
||||
body(pBuf.baseAddress!, sBuf.baseAddress!)
|
||||
}
|
||||
}
|
||||
}
|
||||
return sets[index].withUnsafeBufferPointer { buf in
|
||||
ptrs.append(buf.baseAddress!)
|
||||
sizes.append(buf.count)
|
||||
return recurse(index + 1, &ptrs, &sizes)
|
||||
}
|
||||
}
|
||||
var ptrs: [UnsafePointer<UInt8>] = []
|
||||
var sizes: [Int] = []
|
||||
ptrs.reserveCapacity(sets.count)
|
||||
sizes.reserveCapacity(sets.count)
|
||||
return recurse(0, &ptrs, &sizes)
|
||||
}
|
||||
|
||||
private func makeSession(_ fmt: CMVideoFormatDescription) {
|
||||
let attrs: [CFString: Any] = [
|
||||
kCVPixelBufferPixelFormatTypeKey:
|
||||
kCVPixelFormatType_32BGRA,
|
||||
]
|
||||
VTDecompressionSessionCreate(
|
||||
allocator: kCFAllocatorDefault, formatDescription: fmt,
|
||||
decoderSpecification: nil,
|
||||
imageBufferAttributes: attrs as CFDictionary,
|
||||
outputCallback: nil, decompressionSessionOut: &session)
|
||||
}
|
||||
|
||||
private static func blockBuffer(_ data: Data) -> CMBlockBuffer? {
|
||||
var block: CMBlockBuffer?
|
||||
guard CMBlockBufferCreateWithMemoryBlock(
|
||||
allocator: kCFAllocatorDefault, memoryBlock: nil,
|
||||
blockLength: data.count,
|
||||
blockAllocator: kCFAllocatorDefault,
|
||||
customBlockSource: nil, offsetToData: 0,
|
||||
dataLength: data.count, flags: 0,
|
||||
blockBufferOut: &block) == noErr, let block else {
|
||||
return nil
|
||||
}
|
||||
var ok = false
|
||||
data.withUnsafeBytes { raw in
|
||||
if let base = raw.baseAddress,
|
||||
CMBlockBufferReplaceDataBytes(
|
||||
with: base, blockBuffer: block,
|
||||
offsetIntoDestination: 0,
|
||||
dataLength: data.count) == noErr { ok = true }
|
||||
}
|
||||
return ok ? block : nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import XCTest
|
||||
import AVLiveWire
|
||||
@testable import AVLiveBody
|
||||
|
||||
final class BodyFusionTests: XCTestCase {
|
||||
private func skeleton(pelvisZ: Float) -> SkeletonPayload {
|
||||
var p = SkeletonPayload()
|
||||
p.joints[0] = SIMD3(0, 0, pelvisZ)
|
||||
p.valid[0] = true
|
||||
return p
|
||||
}
|
||||
|
||||
func testPelvisDepthOverride() {
|
||||
let mesh = MultiHMRPerson(
|
||||
vertices: [SIMD3<Float>](repeating: .zero, count: 1),
|
||||
translation: SIMD3(0, 0, -1.0), score: 0.9)
|
||||
let fused = BodyFusion.fuse(
|
||||
persons: [mesh], skeletons: [0: skeleton(pelvisZ: -2.5)])
|
||||
XCTAssertEqual(fused[0].translation.z, -2.5, accuracy: 1e-4)
|
||||
}
|
||||
|
||||
func testPassthroughWhenNoSkeleton() {
|
||||
let mesh = MultiHMRPerson(
|
||||
vertices: [SIMD3<Float>](repeating: .zero, count: 1),
|
||||
translation: SIMD3(0, 0, -1.0), score: 0.9)
|
||||
let fused = BodyFusion.fuse(persons: [mesh], skeletons: [:])
|
||||
XCTAssertEqual(fused[0].translation.z, -1.0, accuracy: 1e-4)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// avlivebody-mac/Tests/AVLiveBodyTests/RenderSettingsTests.swift
|
||||
import XCTest
|
||||
@testable import AVLiveBody
|
||||
|
||||
@MainActor
|
||||
final class RenderSettingsTests: XCTestCase {
|
||||
func testDefaults() {
|
||||
let s = RenderSettings()
|
||||
XCTAssertTrue(s.showSkeleton)
|
||||
XCTAssertTrue(s.showMesh)
|
||||
XCTAssertTrue(s.showVideo)
|
||||
XCTAssertFalse(s.meshMetallic)
|
||||
XCTAssertEqual(s.meshRoughness, 0.6, accuracy: 1e-9)
|
||||
XCTAssertEqual(s.videoOpacity, 1.0, accuracy: 1e-9)
|
||||
XCTAssertEqual(s.keyIntensity, 4000, accuracy: 1e-9)
|
||||
XCTAssertEqual(s.fillIntensity, 1500, accuracy: 1e-9)
|
||||
XCTAssertEqual(s.rimIntensity, 2000, accuracy: 1e-9)
|
||||
XCTAssertEqual(s.fieldOfView, 55, accuracy: 1e-9)
|
||||
XCTAssertFalse(s.showPanel)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import XCTest
|
||||
@testable import AVLiveBody
|
||||
|
||||
final class SceneUniformsTests: XCTestCase {
|
||||
func testStride() {
|
||||
XCTAssertEqual(MemoryLayout<SceneRenderer.SceneUniforms>.stride, 144)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// avlivebody-mac/Tests/AVLiveBodyTests/ShaderCompileTests.swift
|
||||
import XCTest
|
||||
import Metal
|
||||
@testable import AVLiveBody
|
||||
|
||||
final class ShaderCompileTests: XCTestCase {
|
||||
func testSceneMetalCompiles() throws {
|
||||
let here = URL(fileURLWithPath: #filePath) // .../Tests/AVLiveBodyTests/ShaderCompileTests.swift
|
||||
let shader = here
|
||||
.deletingLastPathComponent() // AVLiveBodyTests
|
||||
.deletingLastPathComponent() // Tests
|
||||
.deletingLastPathComponent() // avlivebody-mac
|
||||
.appendingPathComponent("Sources/AVLiveBody/Resources/scene.metal")
|
||||
let src = try String(contentsOf: shader, encoding: .utf8)
|
||||
guard let dev = MTLCreateSystemDefaultDevice() else {
|
||||
throw XCTSkip("no Metal device")
|
||||
}
|
||||
let lib = try dev.makeLibrary(source: src, options: MTLCompileOptions())
|
||||
XCTAssertNotNil(lib.makeFunction(name: "bg_vertex"))
|
||||
XCTAssertNotNil(lib.makeFunction(name: "bg_fragment"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import XCTest
|
||||
@testable import AVLiveBody
|
||||
|
||||
/// In-memory stand-in for the usbmuxd Unix socket.
|
||||
final class MockMuxTransport: MuxTransport {
|
||||
var sent: [Data] = []
|
||||
var canned: [Data] = []
|
||||
func send(_ data: Data) { sent.append(data) }
|
||||
func receivePacket() -> Data? {
|
||||
canned.isEmpty ? nil : canned.removeFirst()
|
||||
}
|
||||
func close() {}
|
||||
}
|
||||
|
||||
final class USBClientTests: XCTestCase {
|
||||
func testListDevicesParsesDeviceIDs() {
|
||||
let mock = MockMuxTransport()
|
||||
mock.canned = [USBMuxProtocol.encode(plist: [
|
||||
"DeviceList": [
|
||||
["DeviceID": 42,
|
||||
"Properties": ["ConnectionType": "USB"]],
|
||||
]], tag: 0)]
|
||||
let client = USBClient(transport: mock)
|
||||
let devices = client.listDevices()
|
||||
XCTAssertEqual(devices, [42])
|
||||
}
|
||||
|
||||
func testConnectSendsConnectRequest() {
|
||||
let mock = MockMuxTransport()
|
||||
mock.canned = [USBMuxProtocol.encode(
|
||||
plist: ["MessageType": "Result", "Number": 0], tag: 0)]
|
||||
let client = USBClient(transport: mock)
|
||||
let ok = client.connect(deviceID: 42, port: 7000)
|
||||
XCTAssertTrue(ok)
|
||||
let req = USBMuxProtocol.decode(mock.sent.last!)
|
||||
XCTAssertEqual(req?["MessageType"] as? String, "Connect")
|
||||
XCTAssertEqual(req?["DeviceID"] as? Int, 42)
|
||||
XCTAssertEqual(req?["PortNumber"] as? Int,
|
||||
Int((UInt16(7000) << 8) | (UInt16(7000) >> 8)))
|
||||
}
|
||||
|
||||
func testConnectFailsOnNonZeroResult() {
|
||||
let mock = MockMuxTransport()
|
||||
mock.canned = [USBMuxProtocol.encode(
|
||||
plist: ["MessageType": "Result", "Number": 3], tag: 0)]
|
||||
let client = USBClient(transport: mock)
|
||||
XCTAssertFalse(client.connect(deviceID: 1, port: 7000))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import XCTest
|
||||
@testable import AVLiveBody
|
||||
|
||||
final class USBMuxProtocolTests: XCTestCase {
|
||||
func testEncodeWrapsPlistWith16ByteHeader() {
|
||||
let body: [String: Any] = ["MessageType": "ListDevices"]
|
||||
let packet = USBMuxProtocol.encode(plist: body, tag: 3)
|
||||
XCTAssertGreaterThan(packet.count, 16)
|
||||
XCTAssertEqual(USBMuxProtocol.readLE32(packet, 0).map(Int.init),
|
||||
packet.count)
|
||||
XCTAssertEqual(USBMuxProtocol.readLE32(packet, 4), 1)
|
||||
XCTAssertEqual(USBMuxProtocol.readLE32(packet, 8), 8)
|
||||
XCTAssertEqual(USBMuxProtocol.readLE32(packet, 12), 3)
|
||||
}
|
||||
|
||||
func testDecodeRoundTrip() {
|
||||
let packet = USBMuxProtocol.encode(
|
||||
plist: ["MessageType": "Result", "Number": 0], tag: 1)
|
||||
let decoded = USBMuxProtocol.decode(packet)
|
||||
XCTAssertEqual(decoded?["MessageType"] as? String, "Result")
|
||||
XCTAssertEqual(decoded?["Number"] as? Int, 0)
|
||||
}
|
||||
|
||||
func testDecodeRejectsShortPacket() {
|
||||
XCTAssertNil(USBMuxProtocol.decode(Data([0, 1, 2])))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
name: AVLiveBody
|
||||
options:
|
||||
bundleIdPrefix: cc.saillant
|
||||
deploymentTarget:
|
||||
macOS: "15.0"
|
||||
createIntermediateGroups: true
|
||||
|
||||
configFiles:
|
||||
Debug: Config/Shared.xcconfig
|
||||
Release: Config/Shared.xcconfig
|
||||
|
||||
packages:
|
||||
AVLiveWire:
|
||||
path: ../shared/AVLiveWire
|
||||
|
||||
targets:
|
||||
AVLiveBody:
|
||||
type: application
|
||||
platform: macOS
|
||||
deploymentTarget: "15.0"
|
||||
sources:
|
||||
- path: Sources/AVLiveBody
|
||||
excludes:
|
||||
- Info.plist
|
||||
- Resources/scene.metal
|
||||
postBuildScripts:
|
||||
- script: |
|
||||
cp "${SRCROOT}/Sources/AVLiveBody/Resources/scene.metal" \
|
||||
"${BUILT_PRODUCTS_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/scene.metal"
|
||||
name: "Copy scene.metal source to bundle"
|
||||
basedOnDependencyAnalysis: false
|
||||
dependencies:
|
||||
- package: AVLiveWire
|
||||
product: AVLiveWire
|
||||
configFiles:
|
||||
Debug: Config/Shared.xcconfig
|
||||
Release: Config/Shared.xcconfig
|
||||
settings:
|
||||
base:
|
||||
PRODUCT_NAME: AVLiveBody
|
||||
PRODUCT_BUNDLE_IDENTIFIER: cc.saillant.AVLiveBody
|
||||
INFOPLIST_FILE: Sources/AVLiveBody/Info.plist
|
||||
GENERATE_INFOPLIST_FILE: NO
|
||||
CODE_SIGN_STYLE: Manual
|
||||
CODE_SIGN_IDENTITY: "-"
|
||||
CODE_SIGNING_REQUIRED: NO
|
||||
CODE_SIGNING_ALLOWED: NO
|
||||
SWIFT_VERSION: "5.10"
|
||||
ENABLE_HARDENED_RUNTIME: NO
|
||||
AVLiveBodyTests:
|
||||
type: bundle.unit-test
|
||||
platform: macOS
|
||||
sources:
|
||||
- path: Tests/AVLiveBodyTests
|
||||
dependencies:
|
||||
- target: AVLiveBody
|
||||
- package: AVLiveWire
|
||||
product: AVLiveWire
|
||||
settings:
|
||||
base:
|
||||
GENERATE_INFOPLIST_FILE: YES
|
||||
@@ -0,0 +1,37 @@
|
||||
[osc]
|
||||
host = "127.0.0.1"
|
||||
port = 57127
|
||||
|
||||
[feeds.eco2mix]
|
||||
enabled = true
|
||||
interval_sec = 60
|
||||
|
||||
[feeds.velib]
|
||||
enabled = true
|
||||
interval_sec = 120
|
||||
station_codes = []
|
||||
|
||||
[feeds.hubeau]
|
||||
enabled = true
|
||||
interval_sec = 300
|
||||
codes = ["F050001001"]
|
||||
|
||||
[feeds.gbfs]
|
||||
enabled = false
|
||||
interval_sec = 120
|
||||
url = "https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json"
|
||||
|
||||
[feeds.ais]
|
||||
enabled = false
|
||||
|
||||
[feeds.carburants]
|
||||
enabled = false
|
||||
|
||||
[feeds.prim]
|
||||
enabled = false
|
||||
|
||||
[feeds.sytadin]
|
||||
enabled = false
|
||||
|
||||
[feeds.teleray]
|
||||
enabled = false
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Registry of available feed classes (auto-discovery on import)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .base import Feed
|
||||
from .eco2mix import Eco2MixFeed
|
||||
from .gbfs import GBFSFeed
|
||||
from .hubeau import HubeauFeed
|
||||
from .velib import VelibFeed
|
||||
from .ais import AISFeed
|
||||
from .carburants import CarburantsFeed
|
||||
from .prim import PRIMFeed
|
||||
from .sytadin import SytadinFeed
|
||||
from .teleray import TelerayFeed
|
||||
|
||||
REGISTRY: dict[str, type[Feed]] = {
|
||||
"eco2mix": Eco2MixFeed,
|
||||
"gbfs": GBFSFeed,
|
||||
"hubeau": HubeauFeed,
|
||||
"velib": VelibFeed,
|
||||
"ais": AISFeed,
|
||||
"carburants": CarburantsFeed,
|
||||
"prim": PRIMFeed,
|
||||
"sytadin": SytadinFeed,
|
||||
"teleray": TelerayFeed,
|
||||
}
|
||||
|
||||
__all__ = ["Feed", "REGISTRY"]
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""AIS vessel positions feed — STUB.
|
||||
|
||||
TODO: needs aisstream.io API key + websocket subscription.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from .base import Feed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.ais")
|
||||
|
||||
|
||||
class AISFeed(Feed):
|
||||
name = "ais"
|
||||
interval_sec = 60.0
|
||||
|
||||
def fetch(self):
|
||||
return None
|
||||
|
||||
def publish(self, payload) -> None:
|
||||
LOG.info("stub")
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Abstract base class for data feeds."""
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import logging
|
||||
import time
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
LOG = logging.getLogger("data_feeds.base")
|
||||
|
||||
|
||||
class Feed(abc.ABC):
|
||||
name: str = "feed"
|
||||
interval_sec: float = 60.0
|
||||
|
||||
def __init__(self, osc_send, **cfg) -> None:
|
||||
self.osc_send = osc_send
|
||||
self.cfg = cfg
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self.last_t: float = 0.0
|
||||
|
||||
def configure(self, **kwargs) -> None:
|
||||
self.cfg.update(kwargs)
|
||||
if "interval_sec" in kwargs:
|
||||
self.interval_sec = float(kwargs["interval_sec"])
|
||||
|
||||
@abc.abstractmethod
|
||||
def fetch(self) -> Any: ...
|
||||
|
||||
@abc.abstractmethod
|
||||
def publish(self, payload: Any) -> None: ...
|
||||
|
||||
def tick(self) -> None:
|
||||
try:
|
||||
payload = self.fetch()
|
||||
self.publish(payload)
|
||||
self.last_t = time.time()
|
||||
self.osc_send(f"/data/{self.name}/heartbeat", [self.last_t])
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("%s fetch failed: %s", self.name, e)
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, name=f"feed-{self.name}", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
self.tick()
|
||||
self._stop.wait(self.interval_sec)
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Prix carburants feed — STUB.
|
||||
|
||||
TODO: needs prix-carburants.gouv.fr GeoJSON cache + station selection.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from .base import Feed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.carburants")
|
||||
|
||||
|
||||
class CarburantsFeed(Feed):
|
||||
name = "carburants"
|
||||
interval_sec = 3600.0
|
||||
|
||||
def fetch(self):
|
||||
return None
|
||||
|
||||
def publish(self, payload) -> None:
|
||||
LOG.info("stub")
|
||||
@@ -0,0 +1,58 @@
|
||||
"""RTE eco2mix feed — France electricity production mix in MW.
|
||||
|
||||
Uses the public OpenDataSoft mirror of RTE eco2mix-national-tr (temps reel,
|
||||
15-min resolution). Stdlib HTTP only.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .base import Feed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.eco2mix")
|
||||
|
||||
# OpenDataSoft public mirror — no key required.
|
||||
URL = (
|
||||
"https://odre.opendatasoft.com/api/records/1.0/search/"
|
||||
"?dataset=eco2mix-national-tr&rows=1&sort=-date_heure"
|
||||
)
|
||||
|
||||
|
||||
class Eco2MixFeed(Feed):
|
||||
name = "eco2mix"
|
||||
interval_sec = 60.0
|
||||
|
||||
def fetch(self) -> Any:
|
||||
req = urllib.request.Request(URL, headers={"User-Agent": "av-live/0.1"})
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
data = json.loads(r.read().decode("utf-8"))
|
||||
records = data.get("records") or []
|
||||
if not records:
|
||||
return None
|
||||
return records[0].get("fields") or {}
|
||||
|
||||
def publish(self, payload: Any) -> None:
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
# Pick a representative subset (MW). Keys per eco2mix-national-tr.
|
||||
keys = [
|
||||
"consommation", "nucleaire", "gaz", "charbon", "fioul",
|
||||
"eolien", "solaire", "hydraulique", "bioenergies",
|
||||
"ech_physiques",
|
||||
]
|
||||
count = 0
|
||||
for k in keys:
|
||||
v = payload.get(k)
|
||||
if v is None:
|
||||
continue
|
||||
try:
|
||||
fv = float(v)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
self.osc_send(f"/data/{self.name}/sample", [k, fv])
|
||||
count += 1
|
||||
self.osc_send(f"/data/{self.name}/count", [count])
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Generic GBFS (General Bikeshare Feed Specification) reader.
|
||||
|
||||
Reads a `station_status.json` URL and publishes aggregate counters.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .base import Feed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.gbfs")
|
||||
|
||||
|
||||
class GBFSFeed(Feed):
|
||||
name = "gbfs"
|
||||
interval_sec = 120.0
|
||||
|
||||
def fetch(self) -> Any:
|
||||
url = self.cfg.get("url")
|
||||
if not url:
|
||||
LOG.info("gbfs disabled: no url configured")
|
||||
return None
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "av-live/0.1"})
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
|
||||
def publish(self, payload: Any) -> None:
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
stations = (payload.get("data") or {}).get("stations") or []
|
||||
if not stations:
|
||||
return
|
||||
codes = set(self.cfg.get("station_codes") or [])
|
||||
bikes = 0
|
||||
docks = 0
|
||||
operative = 0
|
||||
sampled = 0
|
||||
for s in stations:
|
||||
sid = str(s.get("station_id", ""))
|
||||
if codes and sid not in codes:
|
||||
continue
|
||||
bikes += int(s.get("num_bikes_available") or 0)
|
||||
docks += int(s.get("num_docks_available") or 0)
|
||||
if s.get("is_renting") or s.get("is_installed"):
|
||||
operative += 1
|
||||
sampled += 1
|
||||
self.osc_send(f"/data/{self.name}/sample", ["bikes_available", float(bikes)])
|
||||
self.osc_send(f"/data/{self.name}/sample", ["docks_available", float(docks)])
|
||||
self.osc_send(f"/data/{self.name}/sample", ["stations_active", float(operative)])
|
||||
self.osc_send(f"/data/{self.name}/count", [sampled])
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Hub'Eau hydrometrie feed — water level and flow rate for French rivers.
|
||||
|
||||
API: https://hubeau.eaufrance.fr/api/v1/hydrometrie/observations_tr
|
||||
Open, no API key required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .base import Feed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.hubeau")
|
||||
|
||||
BASE = "https://hubeau.eaufrance.fr/api/v1/hydrometrie/observations_tr"
|
||||
|
||||
|
||||
class HubeauFeed(Feed):
|
||||
name = "hubeau"
|
||||
interval_sec = 300.0
|
||||
|
||||
def fetch(self) -> Any:
|
||||
codes = self.cfg.get("codes") or ["F050001001"]
|
||||
out: dict[str, dict[str, float]] = {}
|
||||
for code in codes:
|
||||
params = {
|
||||
"code_entite": code,
|
||||
"size": 1,
|
||||
"sort": "desc",
|
||||
"fields": "code_station,grandeur_hydro,resultat_obs,date_obs",
|
||||
}
|
||||
url = BASE + "?" + urllib.parse.urlencode(params)
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "av-live/0.1"})
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
data = json.loads(r.read().decode("utf-8"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("hubeau %s failed: %s", code, e)
|
||||
continue
|
||||
for obs in data.get("data") or []:
|
||||
station = obs.get("code_station") or code
|
||||
gr = obs.get("grandeur_hydro") or "X"
|
||||
v = obs.get("resultat_obs")
|
||||
if v is None:
|
||||
continue
|
||||
try:
|
||||
fv = float(v)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
out.setdefault(station, {})[gr] = fv
|
||||
return out
|
||||
|
||||
def publish(self, payload: Any) -> None:
|
||||
if not isinstance(payload, dict) or not payload:
|
||||
return
|
||||
count = 0
|
||||
for station, vals in payload.items():
|
||||
for gr, v in vals.items():
|
||||
key = f"{station}_{gr}"
|
||||
self.osc_send(f"/data/{self.name}/sample", [key, float(v)])
|
||||
count += 1
|
||||
self.osc_send(f"/data/{self.name}/count", [count])
|
||||
@@ -0,0 +1,22 @@
|
||||
"""PRIM Ile-de-France Mobilites feed — STUB.
|
||||
|
||||
TODO: needs API key (https://prim.iledefrance-mobilites.fr/).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from .base import Feed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.prim")
|
||||
|
||||
|
||||
class PRIMFeed(Feed):
|
||||
name = "prim"
|
||||
interval_sec = 60.0
|
||||
|
||||
def fetch(self):
|
||||
return None
|
||||
|
||||
def publish(self, payload) -> None:
|
||||
LOG.info("stub")
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Sytadin Paris traffic feed — STUB.
|
||||
|
||||
TODO: needs sytadin.fr scraping / cumulative km of congestion.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from .base import Feed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.sytadin")
|
||||
|
||||
|
||||
class SytadinFeed(Feed):
|
||||
name = "sytadin"
|
||||
interval_sec = 300.0
|
||||
|
||||
def fetch(self):
|
||||
return None
|
||||
|
||||
def publish(self, payload) -> None:
|
||||
LOG.info("stub")
|
||||
@@ -0,0 +1,22 @@
|
||||
"""IRSN Teleray radiation feed — STUB.
|
||||
|
||||
TODO: needs https://teleray.irsn.fr/data endpoint research.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from .base import Feed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.teleray")
|
||||
|
||||
|
||||
class TelerayFeed(Feed):
|
||||
name = "teleray"
|
||||
interval_sec = 600.0
|
||||
|
||||
def fetch(self):
|
||||
return None
|
||||
|
||||
def publish(self, payload) -> None:
|
||||
LOG.info("stub")
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Velib Metropole feed — specialization of GBFS against the Paris URL."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from .gbfs import GBFSFeed
|
||||
|
||||
LOG = logging.getLogger("data_feeds.velib")
|
||||
|
||||
VELIB_URL = (
|
||||
"https://velib-metropole-opendata.smoove.pro/opendata/"
|
||||
"Velib_Metropole/station_status.json"
|
||||
)
|
||||
|
||||
|
||||
class VelibFeed(GBFSFeed):
|
||||
name = "velib"
|
||||
interval_sec = 120.0
|
||||
|
||||
def configure(self, **kwargs) -> None:
|
||||
# Force the URL if caller did not provide one.
|
||||
kwargs.setdefault("url", VELIB_URL)
|
||||
super().configure(**kwargs)
|
||||
if not self.cfg.get("url"):
|
||||
self.cfg["url"] = VELIB_URL
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Run all enabled feeds, publish OSC to AVLiveBody."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
from .feeds import REGISTRY
|
||||
from .osc_sender import OscSender
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--config", default="data_feeds/config.avlivedata.toml")
|
||||
p.add_argument("--osc-host")
|
||||
p.add_argument("--osc-port", type=int)
|
||||
p.add_argument("-v", "--verbose", action="store_true")
|
||||
args = p.parse_args(argv)
|
||||
logging.basicConfig(
|
||||
level=logging.INFO if args.verbose else logging.WARNING,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
cfg = tomllib.loads(Path(args.config).read_text())
|
||||
osc_cfg = cfg.get("osc", {})
|
||||
host = args.osc_host or osc_cfg.get("host", "127.0.0.1")
|
||||
port = args.osc_port or osc_cfg.get("port", 57127)
|
||||
sender = OscSender(host, port)
|
||||
feeds = []
|
||||
for name, kwargs in (cfg.get("feeds") or {}).items():
|
||||
if not kwargs.get("enabled", False):
|
||||
continue
|
||||
cls = REGISTRY.get(name)
|
||||
if cls is None:
|
||||
logging.warning("Unknown feed: %s", name)
|
||||
continue
|
||||
f = cls(sender.send)
|
||||
f.configure(**kwargs)
|
||||
f.start()
|
||||
feeds.append(f)
|
||||
logging.info("started feed %s (interval %.0fs)", name, f.interval_sec)
|
||||
if not feeds:
|
||||
logging.warning("No feeds enabled. Exiting.")
|
||||
return 1
|
||||
try:
|
||||
while True:
|
||||
time.sleep(60)
|
||||
except KeyboardInterrupt:
|
||||
return 0
|
||||
finally:
|
||||
for f in feeds:
|
||||
f.stop()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Wrapper around python-osc SimpleUDPClient with per-route helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from pythonosc.udp_client import SimpleUDPClient
|
||||
|
||||
LOG = logging.getLogger("data_feeds.osc")
|
||||
|
||||
|
||||
class OscSender:
|
||||
def __init__(self, host: str, port: int) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self._client = SimpleUDPClient(host, port)
|
||||
|
||||
def send(self, addr: str, args: list[Any]) -> None:
|
||||
try:
|
||||
self._client.send_message(addr, args)
|
||||
except OSError as e:
|
||||
LOG.warning("send %s failed: %s", addr, e)
|
||||
+24
-6
@@ -20,6 +20,7 @@ Python **3.11+** requis. `pyproject.toml` est la source de vérité — ne jamai
|
||||
| Backend | Fichier | Statut |
|
||||
|---------|---------|--------|
|
||||
| MediaPipe Holistic | `holistic.py` | stable |
|
||||
| MediaPipe multi (Pose+Face+Hand) | `multi.py` | stable ; `MEDIAPIPE_DELEGATE=gpu` (défaut) ou `cpu`. **GPU Metal exige SRGBA 4-ch** (3-ch SRGB crashe `gpu_buffer_storage_cv_pixel_buffer.cc`) — multi.py route auto vers `cv2.COLOR_BGR2RGBA` + `mp.ImageFormat.SRGBA` quand delegate=GPU. Bench M5 image-mode SRGBA : pose 2.9 vs 6.7 ms (GPU/CPU), face 1.0 vs 4.1, hand 3.2 vs 6.1 |
|
||||
| Ultralytics YOLOv8-pose | `pose.py` | stable, modèle `yolov8n-pose.pt` à la racine repo |
|
||||
| Apple Vision (Core ML) | `apple_vision_pose.py`, `coreml_pose.py` | macOS uniquement |
|
||||
| DETRPose | `detrpose.py` | clone manuel + checkpoint, voir docstring |
|
||||
@@ -31,12 +32,29 @@ Python **3.11+** requis. `pyproject.toml` est la source de vérité — ne jamai
|
||||
|
||||
- État partagé multi-thread : `state.py` expose `State.lock()` — toujours mutationner sous lock.
|
||||
- Filtrage temporel : `euro_filter.py` (One Euro Filter) sur les keypoints avant tracker.
|
||||
- ARKit fusion : `iphone_osc_listener.py` consume /body3d/kp UDP :57128
|
||||
→ `state.persons_arkit_joints`. `pose_filter.py::ArkitFuse` (stage
|
||||
`arkit_fuse`) splices the 14 mapped body slots into MediaPipe pose
|
||||
before kalman ; `multi_hmr_worker::arkit_pelvis_z_override` locks the
|
||||
SMPL-X cam translation z to the ARKit pelvis. Mapping in
|
||||
`arkit_joint_map.py`.
|
||||
- ARKit body source (**`--iphone-usb`, défaut concert 2026-06-30**) : le corps
|
||||
vient **100% ARKit**, le pose+face MediaPipe du Mac sont **coupés** (comme les
|
||||
mains, cf. `4c0794a`). L'app iPhone envoie le squelette 2D projeté (frame
|
||||
AVLiveWire `skeleton2D`, tag **6**) → `state.persons_arkit_2d` ; le 3D monde
|
||||
arrive via la frame `skeleton` → `state.persons_arkit_joints`.
|
||||
`multi.py` reconstruit `bodies`/`bodies3d` (MP33) via
|
||||
`arkit_joint_map.arkit_body_2d`/`arkit_body_3d` (slots mappés c=1, sinon c=0 ;
|
||||
pid le plus bas, single-person). `/pose/mouth` + le mesh visage disparaissent.
|
||||
`POSE_FILTER` **sans `arkit_fuse`** (ARKit EST la source). Décode :
|
||||
`scripts/iphone_usb_bridge.decode_skeleton2D` (819 octets = 91×2 f32 BE + 91
|
||||
valid). Mapping 14 joints + ancre tête `ARKIT_HEAD_IDX` dans `arkit_joint_map.py`
|
||||
(index à confirmer live). **Orientation : PORTRAIT validé live 2026-07-02**
|
||||
(meilleure détection des mains, ARKit body suit — l'ancienne contrainte
|
||||
paysage de 2026-06-30 ne se reproduit plus). La vidéo HEVC arrive en repère
|
||||
capteur paysage → `VIDEO_ROTATE=ccw` (défaut launcher concert) la redresse ;
|
||||
les DEUX flux 2D (skeleton2D + mains Vision) arrivent déjà droits et ne sont
|
||||
PAS tournés (knobs `IPHONE_SKEL_ROTATE`/`IPHONE_HANDS_ROTATE` si besoin).
|
||||
- ARKit fusion (**legacy, chemin webcam + OSC**) : `iphone_osc_listener.py`
|
||||
consume /body3d/kp UDP :57128 → `state.persons_arkit_joints`.
|
||||
`pose_filter.py::ArkitFuse` (stage `arkit_fuse`) splices the 14 mapped body
|
||||
slots into MediaPipe pose before kalman ; `multi_hmr_worker::arkit_pelvis_z_override`
|
||||
locks the SMPL-X cam translation z to the ARKit pelvis. Mapping in
|
||||
`arkit_joint_map.py`. (Supersédé par l'ARKit body source sous `--iphone-usb`.)
|
||||
- Association multi-personne : `tracker.py` IoU-based, `scipy.optimize.linear_sum_assignment`.
|
||||
- Shaders Metal dans `shaders/` (`.metal`), recompilés au runtime ; topologie mesh (SMPL faces) en binaire dans `mesh_topology.py`.
|
||||
- OSC out : `osc_listener.py` / `pose_bridge.py` — destination `oscope-of` sur `:57123`.
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# data_only_viz
|
||||
|
||||
Python + Metal visualiser for the AV-Live data-only patch.
|
||||
|
||||
## Hand features OSC
|
||||
|
||||
`/pose/hands [pid=0, lx, ly, lopen, lspeed, rx, ry, ropen, rspeed, dist]`
|
||||
|
||||
- Emitted once per pipeline tick by `pose_bridge.send_hands` to sclang `:57121`.
|
||||
- L = leftmost hand on screen, R = rightmost; absent hand -> its 4 floats are 0.
|
||||
- Ranges: positions/openness/dist in [0,1]; speed >= 0 (normalized units/frame).
|
||||
- Consumed by SuperCollider `~handFeat` (data_only) and the Metal renderer uniforms.
|
||||
- Multi-person is not yet carried (pid is always 0).
|
||||
@@ -24,6 +24,8 @@ from data_only_viz.action_head import (
|
||||
J3D_FINGERS_PER_HAND,
|
||||
LABELS,
|
||||
)
|
||||
from data_only_viz.hand_display import gesture_quality, hand_plausible, HandPersistenceGate
|
||||
from data_only_viz.hand_slots import route_hands
|
||||
|
||||
LOG = logging.getLogger("action_head_pub")
|
||||
|
||||
@@ -98,6 +100,9 @@ class ActionHeadPublisher(threading.Thread):
|
||||
self._last_smplx_t = 0.0
|
||||
self._last_body_t = 0.0
|
||||
self._last_pids: set[int] = set()
|
||||
from data_only_viz.hand_features import HandFeatureExtractor
|
||||
self._hand_ext = HandFeatureExtractor()
|
||||
self._init_finger_piano()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
@@ -136,12 +141,249 @@ class ActionHeadPublisher(threading.Thread):
|
||||
idx = LABELS.index(label)
|
||||
self.bridge.send_action(pid, idx, probs, t_now, force=True)
|
||||
self.bridge.send_kin(pid, kin, t_now, force=True)
|
||||
self.bridge.send_mouth(pid, mouth)
|
||||
if pid not in self._last_pids:
|
||||
self.bridge.send_enter(pid=pid)
|
||||
for gone in self._last_pids - current_pids:
|
||||
self.head.forget(gone)
|
||||
self.bridge.send_leave(pid=gone)
|
||||
self._last_pids = current_pids
|
||||
if getattr(self, "_stab", None) is not None:
|
||||
self._stab_slotted = self._step_stab()
|
||||
# Reset finger-strike state for slots that just transitioned from
|
||||
# "held replay" back to a real hand, preventing phantom strikes from
|
||||
# the compressed motion delta accumulated during the hold window.
|
||||
for i, resumed in enumerate(self._stab.resumed_flags()):
|
||||
if resumed:
|
||||
self._finger_det.reset_slot(i)
|
||||
else:
|
||||
self._stab_slotted = None
|
||||
self._emit_hands(t_now)
|
||||
self._emit_fingers(t_now)
|
||||
self._emit_pinch(t_now)
|
||||
self._update_gesture_slot_status()
|
||||
|
||||
def _emit_hands(self, t_now: float) -> None:
|
||||
"""Lock state, extract hand features, emit /pose/hands once per tick."""
|
||||
if getattr(self, "_tick_hands", None) is not None:
|
||||
# Fast path: validated by _step_stab this tick (audit R3/R4: single lock).
|
||||
hands = self._tick_hands
|
||||
chirality = self._tick_chir
|
||||
mirror = self._tick_mirror
|
||||
else:
|
||||
# Fallback for standalone calls (unit tests bypassing _tick).
|
||||
with self.state.lock():
|
||||
hands = list(getattr(self.state, "persons_hands", None) or [])
|
||||
chirality = list(getattr(self.state, "persons_hands_chirality", None) or [])
|
||||
mirror = bool(getattr(self.state, "mirror_2d", False))
|
||||
if not hands and getattr(self.state, "hands_present", False):
|
||||
lkp = getattr(self.state, "left_hand_kp", None)
|
||||
rkp = getattr(self.state, "right_hand_kp", None)
|
||||
hands = [h for h in (lkp, rkp) if h is not None and len(h) > 0]
|
||||
chirality = []
|
||||
# Inline plausibility gate only (no persistence gate in this path).
|
||||
conf_min = getattr(self, "_gesture_conf_min", 0.0)
|
||||
def _plausible_fb(h):
|
||||
try:
|
||||
return hand_plausible(h, conf_min=conf_min)
|
||||
except (AttributeError, TypeError):
|
||||
return True
|
||||
plausible_mask = [_plausible_fb(h) for h in hands]
|
||||
hands = [h for h, ok in zip(hands, plausible_mask) if ok]
|
||||
chirality = [c for c, ok in zip(chirality, plausible_mask) if ok]
|
||||
stab = getattr(self, "_stab", None)
|
||||
fist_enabled = stab.active_flags() if stab is not None else (True, True)
|
||||
feats = self._hand_ext.step(hands, chirality or None,
|
||||
swap=getattr(self, "_hand_swap_lr", False),
|
||||
fist_enabled=fist_enabled,
|
||||
mirror=mirror)
|
||||
with self.state.lock():
|
||||
self.state.hand_feats = feats
|
||||
self.bridge.send_hands(feats, t_now)
|
||||
|
||||
def _init_finger_piano(self) -> None:
|
||||
"""Read FINGER_* env config and build the strike + pinch detectors."""
|
||||
from data_only_viz.config import VizConfig
|
||||
from data_only_viz.finger_strike import (
|
||||
FingerStrikeDetector, PinchDetector,
|
||||
)
|
||||
cfg = VizConfig.from_env()
|
||||
self._finger_enabled = cfg.finger_piano
|
||||
# Pinch detection is gated independently of the air-piano finger-strike:
|
||||
# the matrix global actions want pinches WITHOUT the finger-strike "piano".
|
||||
self._pinch_enabled = cfg.pinch_enable
|
||||
self._finger_det = FingerStrikeDetector(
|
||||
vel_thresh=cfg.finger_strike_vel,
|
||||
refractory_ms=cfg.finger_strike_refractory_ms,
|
||||
)
|
||||
self._pinch_det = PinchDetector(
|
||||
ratio_on=cfg.pinch_ratio_on,
|
||||
ratio_off=cfg.pinch_ratio_off,
|
||||
refractory_ms=cfg.pinch_refractory_ms,
|
||||
margin=cfg.pinch_margin,
|
||||
ext_ratio=cfg.pinch_ext_ratio,
|
||||
ext_min=cfg.pinch_ext_min,
|
||||
debounce_frames=cfg.pinch_debounce_frames,
|
||||
)
|
||||
self._finger_dbg = cfg.finger_debug
|
||||
# auto = iPhone Vision hands if fresh, else MediaPipe. Or force
|
||||
# "iphone" / "mediapipe".
|
||||
self._finger_source = cfg.finger_source
|
||||
self._fdbg_n = 0
|
||||
# Safety knob: invert Vision chirality interpretation.
|
||||
# Chirality validated correct live 2026-07-02; keep False unless a
|
||||
# future iPhone app build flips it.
|
||||
self._hand_swap_lr = cfg.hand_swap_lr
|
||||
self._hand_near_min = cfg.hand_near_min
|
||||
self._hand_near_off = cfg.hand_near_off
|
||||
self._hand_hold_frames = cfg.hand_hold_frames
|
||||
from data_only_viz.hand_slots import GestureSlotStabilizer
|
||||
self._stab = GestureSlotStabilizer(
|
||||
hold_frames=cfg.hand_hold_frames,
|
||||
near_on=cfg.hand_near_min,
|
||||
near_off=cfg.hand_near_off,
|
||||
face_min=cfg.hand_face_min,
|
||||
)
|
||||
self._gesture_conf_min = cfg.hand_conf_min
|
||||
# Gesture-path gate stepped at ~30 Hz (publisher tick) — correctly
|
||||
# cadenced, unlike the renderer's 60 fps instance (audit R6).
|
||||
self._gesture_gate = HandPersistenceGate(
|
||||
min_frames=cfg.hand_persist_frames, grace=cfg.hand_persist_grace
|
||||
)
|
||||
self._tick_hands: list | None = None
|
||||
self._tick_chir: list = []
|
||||
self._tick_mirror: bool = False
|
||||
self._tick_raw_slotted: list = [None, None]
|
||||
|
||||
def _pick_hands(self) -> tuple[list, list, str, bool, bool]:
|
||||
"""Return (hands, chirality, source_label, ip_fresh, mirror).
|
||||
|
||||
mirror is read from state.mirror_2d under the lock so it is consistent
|
||||
with the chirality dropout decision made in the same tick. Callers
|
||||
pass it through to route_hands so the cx fallback slot assignment stays
|
||||
coherent with the renderer's panel layout when chirality is absent.
|
||||
"""
|
||||
with self.state.lock():
|
||||
mp_hands = list(getattr(self.state, "persons_hands", None) or [])
|
||||
ip_hands = list(getattr(self.state, "persons_hands_iphone", None) or [])
|
||||
ip_t = getattr(self.state, "persons_hands_iphone_t", 0.0)
|
||||
ip_chir = list(getattr(self.state, "persons_hands_chirality", None) or [])
|
||||
mirror = bool(getattr(self.state, "mirror_2d", False))
|
||||
ip_fresh = bool(ip_hands) and (time.perf_counter() - ip_t) < 0.5
|
||||
src = getattr(self, "_finger_source", "auto")
|
||||
if src == "iphone":
|
||||
return ip_hands, ip_chir, "iphone", ip_fresh, mirror
|
||||
if src == "mediapipe":
|
||||
return mp_hands, [], "mediapipe", ip_fresh, mirror
|
||||
if ip_fresh:
|
||||
return ip_hands, ip_chir, "iphone", ip_fresh, mirror
|
||||
return mp_hands, [], "mediapipe", ip_fresh, mirror
|
||||
|
||||
def _step_stab(self) -> list:
|
||||
"""Pick hands, apply quality gates, route, step gesture stabilizer.
|
||||
|
||||
Called once per tick from _tick() to share the result across all
|
||||
three gesture emitters. Also callable directly from emitters when
|
||||
_stab_slotted is not yet set (e.g. in unit tests that call emitters
|
||||
without going through _tick).
|
||||
"""
|
||||
hands, chir, _, _, mirror = self._pick_hands()
|
||||
|
||||
# 1. Plausibility gate: reject out-of-frame / undersized / low-conf ghosts.
|
||||
# Raw numpy-array hands (no .x/.y/.c protocol) are passed through unchanged
|
||||
# so _build_hands_map / action-head model paths keep working.
|
||||
def _plausible(h):
|
||||
try:
|
||||
return hand_plausible(h, conf_min=self._gesture_conf_min)
|
||||
except (AttributeError, TypeError):
|
||||
return True
|
||||
plausible_mask = [_plausible(h) for h in hands]
|
||||
plausible_hands = [h for h, ok in zip(hands, plausible_mask) if ok]
|
||||
plausible_chir = [c for c, ok in zip(chir, plausible_mask) if ok]
|
||||
|
||||
# 2. Establishment gate: anti-ghost temporal filter; same chirality mask
|
||||
# keeps chir[i] aligned with plausible_hands[i] through both filters.
|
||||
established_mask = self._gesture_gate.step(plausible_hands)
|
||||
self._tick_hands = [h for h, ok in zip(plausible_hands, established_mask) if ok]
|
||||
self._tick_chir = [c for c, ok in zip(plausible_chir, established_mask) if ok]
|
||||
self._tick_mirror = mirror
|
||||
|
||||
raw = route_hands(
|
||||
self._tick_hands, self._tick_chir or None,
|
||||
swap=getattr(self, "_hand_swap_lr", False),
|
||||
near_min=0.0,
|
||||
mirror=mirror,
|
||||
)
|
||||
self._tick_raw_slotted = list(raw) # save pre-stabilizer routing for status
|
||||
return self._stab.step(raw)
|
||||
|
||||
def _emit_fingers(self, t_now: float) -> None:
|
||||
"""Detect finger strikes (air-piano) and emit /pose/finger."""
|
||||
if not getattr(self, "_finger_enabled", False):
|
||||
return
|
||||
hands, chirality, used, ip_fresh, _ = self._pick_hands()
|
||||
slotted = getattr(self, "_stab_slotted", None)
|
||||
if slotted is None:
|
||||
slotted = self._step_stab()
|
||||
events = self._finger_det.step(slotted, t_now)
|
||||
if getattr(self, "_finger_dbg", False):
|
||||
self._fdbg_n += 1
|
||||
if events or self._fdbg_n % 30 == 0:
|
||||
LOG.info("fingers dbg: src=%s hands=%d strikes=%d (ip_fresh=%s)",
|
||||
used, len(hands), len(events), ip_fresh)
|
||||
for ev in events:
|
||||
self.bridge.send_finger(ev)
|
||||
|
||||
def _emit_pinch(self, t_now: float) -> None:
|
||||
"""Detect thumb-to-finger pinches (matrix global actions) and emit
|
||||
/pose/pinch. Gated by PINCH_ENABLE, independently of the air-piano."""
|
||||
if not getattr(self, "_pinch_enabled", False):
|
||||
return
|
||||
hands, chirality, used, _ip, _ = self._pick_hands()
|
||||
slotted = getattr(self, "_stab_slotted", None)
|
||||
if slotted is None:
|
||||
slotted = self._step_stab()
|
||||
for ev in self._pinch_det.step(slotted, t_now):
|
||||
if getattr(self, "_finger_dbg", False):
|
||||
LOG.info("pinch dbg: src=%s hand=%d finger=%d",
|
||||
used, ev.hand, ev.finger)
|
||||
self.bridge.send_pinch(ev)
|
||||
|
||||
def _update_gesture_slot_status(self) -> None:
|
||||
"""Compute per-slot gesture status and quality, write both to state.
|
||||
|
||||
Status values: 0=absent, 1=detected(plausible+established, not armed),
|
||||
2=armed(near+facing), 3=pinch engaged.
|
||||
Quality ∈ [0, 1]: continuous intensity for frame brightness + thickness.
|
||||
Called once per tick after all emitters have run.
|
||||
"""
|
||||
stab = getattr(self, "_stab", None)
|
||||
if stab is None:
|
||||
return
|
||||
pinch_det = getattr(self, "_pinch_det", None)
|
||||
raw = getattr(self, "_tick_raw_slotted", [None, None])
|
||||
active = stab.active_flags()
|
||||
engaged = pinch_det.engaged_slots() if pinch_det is not None else (False, False)
|
||||
status = [0, 0]
|
||||
quality = [0.0, 0.0]
|
||||
face_min = stab._face_min
|
||||
near_on = stab._near_on
|
||||
for slot in range(2):
|
||||
if engaged[slot]:
|
||||
status[slot] = 3
|
||||
elif active[slot]:
|
||||
status[slot] = 2
|
||||
elif raw[slot] is not None:
|
||||
status[slot] = 1
|
||||
quality[slot] = gesture_quality(
|
||||
raw[slot],
|
||||
face_min=face_min,
|
||||
near_on=near_on,
|
||||
engaged=bool(engaged[slot]),
|
||||
)
|
||||
with self.state.lock():
|
||||
self.state.gesture_slot_status = status
|
||||
self.state.gesture_slot_quality = quality
|
||||
|
||||
def _read_sources(self) -> tuple[
|
||||
list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] | None,
|
||||
@@ -177,8 +419,12 @@ class ActionHeadPublisher(threading.Thread):
|
||||
if t_smplx > self._last_smplx_t:
|
||||
out: list[tuple[int, np.ndarray, np.ndarray, float, np.ndarray]] = []
|
||||
for i, p in enumerate(persons_smplx or []):
|
||||
pid = int(p.get("pid", i))
|
||||
v3d = p.get("v3d")
|
||||
# Support both SMPLXPerson dataclass (multi_hmr_worker, field
|
||||
# names: pid / vertices_3d / expression) and legacy dict format
|
||||
# (keys: "pid" / "v3d" / "expression").
|
||||
_is_dict = isinstance(p, dict)
|
||||
pid = int(p.get("pid", i) if _is_dict else p.pid)
|
||||
v3d = p.get("v3d") if _is_dict else p.vertices_3d
|
||||
if v3d is None:
|
||||
continue
|
||||
# CoreMLArray wraps a numpy array but has no __array__
|
||||
@@ -189,8 +435,8 @@ class ActionHeadPublisher(threading.Thread):
|
||||
if v3d_np.shape[0] < max(SMPLX_JOINT_ANCHOR_VERTS) + 1:
|
||||
continue
|
||||
j3d32 = v3d_np[list(SMPLX_JOINT_ANCHOR_VERTS)].astype(np.float32)
|
||||
# expression
|
||||
expr = p.get("expression")
|
||||
# expression — field name matches in both dict and dataclass
|
||||
expr = p.get("expression") if _is_dict else p.expression
|
||||
if expr is not None:
|
||||
if hasattr(expr, "numpy") and not isinstance(expr, np.ndarray):
|
||||
expr = expr.numpy()
|
||||
@@ -292,22 +538,26 @@ class ActionHeadPublisher(threading.Thread):
|
||||
|
||||
@staticmethod
|
||||
def _kp_list_to_array(body: Any) -> np.ndarray | None:
|
||||
"""Best-effort conversion of a body keypoint list to (N, 3) array."""
|
||||
"""Best-effort conversion of a body keypoint list to (N, 3) array.
|
||||
|
||||
Handles attribute objects (the Kp3D dataclass exposes .x/.y/.z) and
|
||||
indexable rows. Kp3D is NOT subscriptable, so the index fallback must
|
||||
not be evaluated eagerly (the old `getattr(kp, "x", kp[0])` raised
|
||||
TypeError on Kp3D and silently returned None — breaking the body3d /
|
||||
MediaPipe-only path that feeds action-head when no SMPL-X is present)."""
|
||||
if body is None:
|
||||
return None
|
||||
if isinstance(body, np.ndarray):
|
||||
return body
|
||||
try:
|
||||
return np.asarray(
|
||||
[
|
||||
(
|
||||
getattr(kp, "x", kp[0]),
|
||||
getattr(kp, "y", kp[1]),
|
||||
getattr(kp, "z", kp[2] if len(kp) > 2 else 0.0),
|
||||
)
|
||||
for kp in body
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
except (TypeError, IndexError, AttributeError):
|
||||
rows = []
|
||||
for kp in body:
|
||||
if hasattr(kp, "x"):
|
||||
rows.append((kp.x, kp.y, getattr(kp, "z", 0.0)))
|
||||
else:
|
||||
try:
|
||||
rows.append((kp[0], kp[1], kp[2] if len(kp) > 2 else 0.0))
|
||||
except (TypeError, IndexError):
|
||||
return None
|
||||
if not rows:
|
||||
return None
|
||||
return np.asarray(rows, dtype=np.float32)
|
||||
|
||||
@@ -25,9 +25,15 @@ MP33_NUM_LANDMARKS = 33
|
||||
# Used by multi_hmr_worker for cam-translation z lock.
|
||||
ARKIT_PELVIS_IDX = 1
|
||||
|
||||
# ARKit head joint index in the 91-joint enum.
|
||||
# Tentatively mapped to index 51 (head_joint in Apple's canonical ordering).
|
||||
# VERIFY LIVE (controller will confirm the index on the rig)
|
||||
ARKIT_HEAD_IDX = 51
|
||||
|
||||
# (arkit_joint_idx, mediapipe_pose_idx). Match the body slots used
|
||||
# by the SMPL-X body fusion in multi.py.
|
||||
ARKIT91_TO_MP33: tuple[tuple[int, int], ...] = (
|
||||
(ARKIT_HEAD_IDX, 0), # head_joint -> NOSE (anchor; VERIFY LIVE)
|
||||
(50, 11), # left_shoulder_1_joint -> L_SHOULDER
|
||||
(32, 12), # right_shoulder_1_joint -> R_SHOULDER
|
||||
(53, 13), # left_arm_joint -> L_ELBOW
|
||||
@@ -43,3 +49,43 @@ ARKIT91_TO_MP33: tuple[tuple[int, int], ...] = (
|
||||
(65, 31), # left_toes_joint -> L_FOOT_INDEX
|
||||
(60, 32), # right_toes_joint -> R_FOOT_INDEX
|
||||
)
|
||||
|
||||
|
||||
def arkit_body_2d(arr2d) -> list:
|
||||
"""Map a 91×2 normalized ARKit 2D skeleton to a MediaPipe Pose-33 list.
|
||||
|
||||
Mapped slots get c=1.0; all other slots default to c=0.0 (low confidence).
|
||||
|
||||
Args:
|
||||
arr2d: array-like of shape (91, 2), image-normalized coords [0..1].
|
||||
|
||||
Returns:
|
||||
list of 33 PoseKp instances.
|
||||
"""
|
||||
from .state import PoseKp # noqa: PLC0415
|
||||
body: list = [PoseKp() for _ in range(MP33_NUM_LANDMARKS)]
|
||||
for ai, mp in ARKIT91_TO_MP33:
|
||||
if mp < MP33_NUM_LANDMARKS and ai < len(arr2d):
|
||||
body[mp] = PoseKp(x=float(arr2d[ai][0]), y=float(arr2d[ai][1]),
|
||||
z=0.0, c=1.0)
|
||||
return body
|
||||
|
||||
|
||||
def arkit_body_3d(arr3d) -> list:
|
||||
"""Map a 91×3 world ARKit skeleton to a MediaPipe Pose-33 Kp3D list.
|
||||
|
||||
Mapped slots get c=1.0; all other slots default to c=0.0 (low confidence).
|
||||
|
||||
Args:
|
||||
arr3d: array-like of shape (91, 3), metric world coords (metres).
|
||||
|
||||
Returns:
|
||||
list of 33 Kp3D instances.
|
||||
"""
|
||||
from .state import Kp3D # noqa: PLC0415
|
||||
body: list = [Kp3D() for _ in range(MP33_NUM_LANDMARKS)]
|
||||
for ai, mp in ARKIT91_TO_MP33:
|
||||
if mp < MP33_NUM_LANDMARKS and ai < len(arr3d):
|
||||
body[mp] = Kp3D(x=float(arr3d[ai][0]), y=float(arr3d[ai][1]),
|
||||
z=float(arr3d[ai][2]), c=1.0)
|
||||
return body
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Build line segments from a full ARKit skeleton + parent topology.
|
||||
|
||||
Pure functions, no Metal/state dependency, so they are unit-testable.
|
||||
The renderer feeds the resulting (ax, ay, bx, by) tuples straight into
|
||||
the GPU line buffer.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
|
||||
def bones_from_parents(parents: list[int]) -> list[tuple[int, int]]:
|
||||
"""(child, parent) pairs for every joint with a valid parent."""
|
||||
n = len(parents)
|
||||
return [(i, p) for i, p in enumerate(parents) if 0 <= p < n]
|
||||
|
||||
|
||||
def arkit_segments(arr2d, valid, parents, max_len: float = 0.5, bounds: float = 0.1):
|
||||
"""Return (ax, ay, bx, by) for each bone with both endpoints valid.
|
||||
|
||||
arr2d: indexable of (x, y) normalized [0,1], length == len(parents).
|
||||
valid: indexable of bool (length == len(parents)) or None to keep all.
|
||||
parents: parent index per joint (-1 = root).
|
||||
max_len: maximum bone length in normalized units; bones longer than this
|
||||
are skipped (garbage coords during ARKit tracking loss).
|
||||
Caveat: on an extreme close-up (subject filling the frame) a
|
||||
legit bone can approach 0.5 — raise ARKIT_BONE_MAX live if
|
||||
the skeleton visibly loses bones when very near the camera.
|
||||
bounds: margin outside [0,1] that is still accepted; endpoints outside
|
||||
[-bounds, 1+bounds] in x or y are rejected.
|
||||
"""
|
||||
n = len(parents)
|
||||
lo = -bounds
|
||||
hi = 1.0 + bounds
|
||||
segs: list[tuple[float, float, float, float]] = []
|
||||
for child, parent in bones_from_parents(parents):
|
||||
if child >= n or parent >= n:
|
||||
continue
|
||||
if valid is not None and (not valid[child] or not valid[parent]):
|
||||
continue
|
||||
ax, ay = float(arr2d[child][0]), float(arr2d[child][1])
|
||||
bx, by = float(arr2d[parent][0]), float(arr2d[parent][1])
|
||||
# Reject out-of-frame endpoints
|
||||
if not (lo <= ax <= hi and lo <= ay <= hi):
|
||||
continue
|
||||
if not (lo <= bx <= hi and lo <= by <= hi):
|
||||
continue
|
||||
# Reject implausibly long bones (tracking-loss garbage)
|
||||
if math.hypot(bx - ax, by - ay) > max_len:
|
||||
continue
|
||||
segs.append((ax, ay, bx, by))
|
||||
return segs
|
||||
|
||||
|
||||
# ARKit's 91-joint body includes per-finger joints whose 2D projection is
|
||||
# unreliable (the user sees them as "false hands" glued to the skeleton).
|
||||
# The display uses the filtered Vision hands instead; these joints are
|
||||
# masked out of the wireframe. The wrist anchors (left_hand_joint /
|
||||
# right_hand_joint) are kept so the forearm bone still terminates.
|
||||
_FINGER_TOKENS = ("thumb", "index", "mid", "ring", "little", "pinky")
|
||||
|
||||
|
||||
def finger_joint_mask(names: list) -> list[bool]:
|
||||
"""True for each ARKit joint that belongs to a finger (to be hidden)."""
|
||||
out = []
|
||||
for n in names:
|
||||
ln = str(n).lower()
|
||||
out.append("hand" in ln and any(t in ln for t in _FINGER_TOKENS))
|
||||
return out
|
||||
@@ -0,0 +1,43 @@
|
||||
"""AVLiveWire topology payload codec.
|
||||
|
||||
Mirror of the Swift `TopologyPayload` (shared/AVLiveWire). Layout
|
||||
(big-endian): u16 jointCount; per joint [u8 nameLen + utf8 name];
|
||||
then per joint i16 parent (-1 = root).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
|
||||
|
||||
def encode_topology(names: list[str], parents: list[int]) -> bytes:
|
||||
out = bytearray()
|
||||
out += struct.pack(">H", len(names))
|
||||
for nm in names:
|
||||
b = nm.encode("utf-8")[:255]
|
||||
out.append(len(b))
|
||||
out += b
|
||||
for p in parents:
|
||||
out += struct.pack(">h", p)
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def decode_topology(payload: bytes) -> tuple[list[str], list[int]] | None:
|
||||
"""Return (joint_names, parents) or None if malformed."""
|
||||
if len(payload) < 2:
|
||||
return None
|
||||
n = struct.unpack(">H", payload[:2])[0]
|
||||
o = 2
|
||||
names: list[str] = []
|
||||
for _ in range(n):
|
||||
if o >= len(payload):
|
||||
return None
|
||||
ln = payload[o]
|
||||
o += 1
|
||||
if o + ln > len(payload):
|
||||
return None
|
||||
names.append(payload[o:o + ln].decode("utf-8", "replace"))
|
||||
o += ln
|
||||
if o + n * 2 > len(payload):
|
||||
return None
|
||||
parents = list(struct.unpack(">" + "h" * n, payload[o:o + n * 2]))
|
||||
return names, parents
|
||||
@@ -0,0 +1,335 @@
|
||||
"""Central configuration for data_only_viz — single source of truth for env vars.
|
||||
|
||||
All os.environ reads for data_only_viz belong here.
|
||||
Consumers call ``VizConfig.from_env()`` inside their ``__init__`` or the
|
||||
consuming method — never at module import time so that test monkeypatching
|
||||
of os.environ is picked up correctly.
|
||||
|
||||
Env var reference table
|
||||
-----------------------
|
||||
Name Default Type Effect
|
||||
MEDIAPIPE_DELEGATE "gpu" str "gpu" (Metal) or "cpu"
|
||||
VIDEO_ROTATE "none" str none/ccw/cw/180 — rotate frame before MP+JPEG
|
||||
POSE_GHOST_MIN_VISIBLE 10 int min visible kps to accept a body detection
|
||||
POSE_GHOST_MIN_CONF 0.5 float min mean confidence to keep a detection
|
||||
POSE_HAND_MIN_VISIBLE 15 int visibility gate for hand kps
|
||||
POSE_FACE_MIN_VISIBLE 50 int visibility gate for face kps
|
||||
POSE_NMS_IOU 0.7 float IoU threshold for body NMS
|
||||
POSE_FILTER None str filter stage list (+/-sep); None=defaults
|
||||
POSE_FILTER_FACE None str filter stages for face; None=defaults
|
||||
POSE_FILTER_HAND None str filter stages for hands; None=defaults
|
||||
FINGER_PIANO "0" bool enable air-piano finger-strike emission
|
||||
FINGER_STRIKE_VEL 0.02 float downward relative-velocity threshold
|
||||
FINGER_STRIKE_REFRACTORY_MS 120 float min ms between strikes per finger
|
||||
FINGER_SOURCE "auto" str "auto"/"iphone"/"mediapipe"
|
||||
FINGER_DEBUG "0" bool log each detected strike/pinch
|
||||
PINCH_ENABLE "0" bool enable pinch gesture detection
|
||||
PINCH_RATIO_ON 0.45 float thumb-to-finger dist/hand-size to engage
|
||||
PINCH_RATIO_OFF 0.65 float dist/size above which pinch re-arms
|
||||
PINCH_REFRACTORY_MS 250 float min ms between pinches per finger
|
||||
PINCH_MARGIN 0.20 float extra margin added to pinch zone
|
||||
PINCH_EXT_RATIO 1.35 float extended-pinch activation ratio
|
||||
PINCH_EXT_MIN 2 int min extended fingers for a pinch to fire; 0 disables open-hand gate
|
||||
PINCH_DEBOUNCE_FRAMES 3 int debounce frames for pinch state machine
|
||||
HAND_CONF_MIN 0.45 float min hand landmark confidence to render
|
||||
HAND_PERSIST_FRAMES 3 int consecutive frames before a hand track is drawn; 1=off
|
||||
HAND_SWAP_LR "0" bool Safety knob: invert Vision chirality (validated correct live 2026-07-02; keep 0 unless a future iPhone app build flips it)
|
||||
HAND_NEAR_MIN 0.10 float min hand size (norm) to consider "near"
|
||||
HAND_FACE_MIN 0.5 float palm-spread ratio threshold: slot activates gestures only when facing >= this (side-on < 0.4, facing ~0.7-1.0)
|
||||
HAND_PERSIST_GRACE 2 int grace frames before dropping a hand track on Vision miss (0 = strict)
|
||||
HAND_NEAR_OFF 0.08 float hand size below which near gate deactivates (hysteresis off-threshold)
|
||||
HAND_HOLD_FRAMES 2 int gesture slot hold: frames to carry last hand on Vision miss
|
||||
ARKIT_BONE_MAX 0.5 float max bone length (norm) for ARKit skeleton
|
||||
ARKIT_FULL_SKELETON "1" bool draw full 91-joint ARKit body (!=0 = True)
|
||||
VIZ_HUD "0" bool show debug HUD overlay
|
||||
VIZ_SOURCE_KEYS "0" bool wxcvbn + 0-9 source-bundle shortcuts (off: perf safety)
|
||||
VIZ_AUDIO_KEYS "0" bool qsdfghjkl audio-scene shortcuts (off: perf safety; m stays)
|
||||
VIZ_DEFAULT_MODE "auto" str fixed boot viz mode (e.g. voronoi); disables auto-openpos
|
||||
SKEL_LINE_WIDTH "4" int body skeleton stroke width, approx px (1 = thin)
|
||||
IPHONE_OSC_PORT 57128 int UDP port for iPhone ARBodyTracker /body3d/kp
|
||||
CONCERT_MIRROR "1" bool mirror video horizontally (!=0 = True)
|
||||
ICP_FUSION "0" bool enable ICP LiDAR fusion (only "1" = True)
|
||||
ICP_LIDAR_HOST None str iPhone LiDAR stream host when ICP_FUSION=1
|
||||
ICP_LIDAR_PORT 5500 int iPhone LiDAR stream port
|
||||
ICP_LIDAR_EXTRINSIC None str path to extrinsic JSON for ICP
|
||||
MULTIHMR_BACKEND "pytorch" str "pytorch"/"coreml"/"remote"
|
||||
MULTIHMR_LOOP_FPS 30.0 float Multi-HMR worker loop target fps
|
||||
MULTIHMR_AUTOCAST "0" bool MPS fp16 autocast for Multi-HMR (only "1")
|
||||
MULTIHMR_REMOTE_HOST "192.168.0.175" str remote Multi-HMR server host
|
||||
MULTIHMR_REMOTE_PORT 57140 int remote Multi-HMR server port
|
||||
MULTIHMR_REMOTE_JPEG True bool send JPEG-compressed frames to remote
|
||||
MULTIHMR_REMOTE_JPEG_QUALITY 80 int JPEG quality for remote frames
|
||||
MULTIHMR_REMOTE_ASYNC True bool async pipeline for remote backend
|
||||
MULTIHMR_REID "dino" str reid mode: "dino" or "iou"
|
||||
MULTIHMR_REID_ALPHA 0.5 float DINOv2 weight in hybrid reid score
|
||||
COREML_MLPACKAGE None str path to .mlpackage (None=default cache path)
|
||||
COREML_COMPUTE_UNITS "" str "cpu_only"/"cpu_and_gpu"/"all"/"cpu_and_ne"
|
||||
AVBODY_HOST "127.0.0.1" str host for AVLiveBody skeleton UDP/TCP
|
||||
VDMX_OSC_HOST None str VDMX VJ OSC host (None=disabled)
|
||||
VDMX_OSC_PORT 1234 int VDMX VJ OSC port
|
||||
AV_LIVE_MEDIAPIPE True bool enable MediaPipe Multi worker (!=0 = True)
|
||||
AV_LIVE_APPLE_VISION True bool enable Apple Vision fallback (!=0 = True)
|
||||
AV_LIVE_COREML True bool enable CoreML pose fallback (!=0 = True)
|
||||
AV_LIVE_DETRPOSE False bool enable DETRPose fallback (only "1" = True)
|
||||
AV_LIVE_PARALLEL_POSE "both" str parallel pose backend in multi-hmr mode
|
||||
MESH_RIG "1" bool enable SMPL-X mesh rigger (!=0 = True)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
from typing import Mapping
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class VizConfig:
|
||||
"""Frozen snapshot of all data_only_viz environment configuration.
|
||||
|
||||
Construct with ``VizConfig.from_env()`` inside ``__init__`` or the
|
||||
consuming method — never at module import time.
|
||||
"""
|
||||
|
||||
# ---- MediaPipe / video -----------------------------------------------
|
||||
mediapipe_delegate: str = "gpu"
|
||||
video_rotate: str = "none"
|
||||
|
||||
# ---- Pose discrimination thresholds ----------------------------------
|
||||
pose_ghost_min_visible: int = 10
|
||||
pose_ghost_min_conf: float = 0.5
|
||||
pose_hand_min_visible: int = 15
|
||||
pose_face_min_visible: int = 50
|
||||
pose_nms_iou: float = 0.7
|
||||
|
||||
# ---- Pose filter chain -----------------------------------------------
|
||||
pose_filter: str | None = None
|
||||
pose_filter_face: str | None = None
|
||||
pose_filter_hand: str | None = None
|
||||
|
||||
# ---- Finger piano / strike -------------------------------------------
|
||||
finger_piano: bool = False
|
||||
finger_strike_vel: float = 0.02
|
||||
finger_strike_refractory_ms: float = 120.0
|
||||
finger_source: str = "auto"
|
||||
finger_debug: bool = False
|
||||
|
||||
# ---- Pinch detection -------------------------------------------------
|
||||
pinch_enable: bool = False
|
||||
pinch_ratio_on: float = 0.50
|
||||
pinch_ratio_off: float = 0.65
|
||||
pinch_refractory_ms: float = 250.0
|
||||
pinch_margin: float = 0.05
|
||||
pinch_ext_ratio: float = 1.35
|
||||
pinch_ext_min: int = 1
|
||||
pinch_debounce_frames: int = 3
|
||||
|
||||
# ---- Hand display / gesture ------------------------------------------
|
||||
hand_conf_min: float = 0.45
|
||||
hand_persist_frames: int = 3
|
||||
hand_persist_grace: int = 2
|
||||
hand_swap_lr: bool = False
|
||||
hand_near_min: float = 0.10
|
||||
hand_near_off: float = 0.08
|
||||
hand_hold_frames: int = 2
|
||||
hand_face_min: float = 0.5
|
||||
arkit_bone_max: float = 0.5
|
||||
arkit_full_skeleton: bool = True
|
||||
|
||||
# ---- HUD -------------------------------------------------------------
|
||||
viz_hud: bool = False
|
||||
# Source-bundle keyboard shortcuts (wxcvbn): OFF by default — a stray
|
||||
# keypress during a matrix performance switches the audio scene
|
||||
# (user-disabled live 2026-07-02). VIZ_SOURCE_KEYS=1 re-enables.
|
||||
viz_source_keys: bool = False
|
||||
# Audio-scene row (qsdfghjkl): same perf-safety gate, same date. The
|
||||
# special m key (matrix mode: scene stop + openpos viz) stays active.
|
||||
viz_audio_keys: bool = False
|
||||
# Fixed viz mode at boot (a viz_mode_names entry, e.g. "voronoi").
|
||||
# "auto" = historical behavior (mode 0 + auto-openpos when persons
|
||||
# appear). A fixed mode also DISABLES the auto-openpos timer so it is
|
||||
# truly always on (user request live 2026-07-02: t/voronoi permanent).
|
||||
viz_default_mode: str = "auto"
|
||||
# Approximate body-skeleton stroke width in pixels (multi-pass offsets;
|
||||
# Metal lines are 1px). 1 = thin historical look.
|
||||
skel_line_width: int = 4
|
||||
|
||||
# ---- iPhone OSC ------------------------------------------------------
|
||||
iphone_osc_port: int = 57128
|
||||
|
||||
# ---- Concert / ARKit -------------------------------------------------
|
||||
concert_mirror: bool = True
|
||||
|
||||
# ---- ICP LiDAR fusion ------------------------------------------------
|
||||
icp_fusion: bool = False
|
||||
icp_lidar_host: str | None = None
|
||||
icp_lidar_port: int = 5500
|
||||
icp_lidar_extrinsic: str | None = None
|
||||
|
||||
# ---- Multi-HMR -------------------------------------------------------
|
||||
multihmr_backend: str = "pytorch"
|
||||
multihmr_loop_fps: float = 30.0
|
||||
multihmr_autocast: bool = False
|
||||
multihmr_remote_host: str = "192.168.0.175"
|
||||
multihmr_remote_port: int = 57140
|
||||
multihmr_remote_jpeg: bool = True
|
||||
multihmr_remote_jpeg_quality: int = 80
|
||||
multihmr_remote_async: bool = True
|
||||
multihmr_reid: str = "dino"
|
||||
multihmr_reid_alpha: float = 0.5
|
||||
|
||||
# ---- CoreML ----------------------------------------------------------
|
||||
coreml_mlpackage: str | None = None
|
||||
coreml_compute_units: str = ""
|
||||
|
||||
# ---- OSC destinations ------------------------------------------------
|
||||
avbody_host: str = "127.0.0.1"
|
||||
vdmx_osc_host: str | None = None
|
||||
vdmx_osc_port: int = 1234
|
||||
|
||||
# ---- main.py worker selection ----------------------------------------
|
||||
av_live_mediapipe: bool = True
|
||||
av_live_apple_vision: bool = True
|
||||
av_live_coreml: bool = True
|
||||
av_live_detrpose: bool = False
|
||||
av_live_parallel_pose: str = "both"
|
||||
mesh_rig: bool = True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Factory
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, environ: Mapping[str, str] | None = None) -> "VizConfig":
|
||||
"""Build a VizConfig by reading the given mapping (default: os.environ).
|
||||
|
||||
Numeric conversion errors fall back to the field default silently.
|
||||
Bool conventions vary by field to match existing code exactly:
|
||||
- _bool_std: not in ("0","","false","False") → True (standard)
|
||||
- _bool_ne0: != "0" → True (empty string is True; None uses default)
|
||||
- _bool_eq1: == "1" only (all other values → False)
|
||||
- _bool_flag: in ("1","true","yes","on") → True (multihmr_remote style)
|
||||
"""
|
||||
env: Mapping[str, str] = environ if environ is not None else os.environ
|
||||
|
||||
_FALSE_STD = frozenset(("0", "", "false", "False"))
|
||||
|
||||
def _bool_std(name: str, default: bool) -> bool:
|
||||
v = env.get(name)
|
||||
if v is None:
|
||||
return default
|
||||
return v not in _FALSE_STD
|
||||
|
||||
def _bool_ne0(name: str, default: bool) -> bool:
|
||||
"""Matches `env.get(name, "1") != "0"` style."""
|
||||
v = env.get(name)
|
||||
if v is None:
|
||||
return default
|
||||
return v != "0"
|
||||
|
||||
def _bool_eq1(name: str) -> bool:
|
||||
"""Matches `env.get(name, "0") == "1"` style — only "1" is True."""
|
||||
return env.get(name, "") == "1"
|
||||
|
||||
def _bool_flag(name: str, default: bool) -> bool:
|
||||
"""Matches multihmr_remote _env_flag: in ("1","true","yes","on")."""
|
||||
v = env.get(name)
|
||||
if v is None:
|
||||
return default
|
||||
return v.strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
def _int(name: str, default: int) -> int:
|
||||
v = env.get(name)
|
||||
if v is None:
|
||||
return default
|
||||
try:
|
||||
return int(v)
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
def _float(name: str, default: float) -> float:
|
||||
v = env.get(name)
|
||||
if v is None:
|
||||
return default
|
||||
try:
|
||||
return float(v)
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
def _str(name: str, default: str | None = None) -> str | None:
|
||||
v = env.get(name)
|
||||
return v if v is not None else default
|
||||
|
||||
def _str_lower(name: str, default: str) -> str:
|
||||
v = env.get(name)
|
||||
return v.lower() if v is not None else default
|
||||
|
||||
return cls(
|
||||
mediapipe_delegate=_str_lower("MEDIAPIPE_DELEGATE", "gpu"),
|
||||
video_rotate=_str_lower("VIDEO_ROTATE", "none"),
|
||||
pose_ghost_min_visible=_int("POSE_GHOST_MIN_VISIBLE", 10),
|
||||
pose_ghost_min_conf=_float("POSE_GHOST_MIN_CONF", 0.5),
|
||||
pose_hand_min_visible=_int("POSE_HAND_MIN_VISIBLE", 15),
|
||||
pose_face_min_visible=_int("POSE_FACE_MIN_VISIBLE", 50),
|
||||
pose_nms_iou=_float("POSE_NMS_IOU", 0.7),
|
||||
pose_filter=_str("POSE_FILTER"),
|
||||
pose_filter_face=_str("POSE_FILTER_FACE"),
|
||||
pose_filter_hand=_str("POSE_FILTER_HAND"),
|
||||
finger_piano=_bool_std("FINGER_PIANO", False),
|
||||
finger_strike_vel=_float("FINGER_STRIKE_VEL", 0.02),
|
||||
finger_strike_refractory_ms=_float("FINGER_STRIKE_REFRACTORY_MS", 120.0),
|
||||
finger_source=_str_lower("FINGER_SOURCE", "auto"),
|
||||
finger_debug=_bool_std("FINGER_DEBUG", False),
|
||||
pinch_enable=_bool_std("PINCH_ENABLE", False),
|
||||
pinch_ratio_on=_float("PINCH_RATIO_ON", 0.50),
|
||||
pinch_ratio_off=_float("PINCH_RATIO_OFF", 0.65),
|
||||
pinch_refractory_ms=_float("PINCH_REFRACTORY_MS", 250.0),
|
||||
pinch_margin=_float("PINCH_MARGIN", 0.05),
|
||||
pinch_ext_ratio=_float("PINCH_EXT_RATIO", 1.35),
|
||||
pinch_ext_min=_int("PINCH_EXT_MIN", 1),
|
||||
pinch_debounce_frames=_int("PINCH_DEBOUNCE_FRAMES", 3),
|
||||
hand_conf_min=_float("HAND_CONF_MIN", 0.45),
|
||||
hand_persist_frames=_int("HAND_PERSIST_FRAMES", 3),
|
||||
hand_persist_grace=_int("HAND_PERSIST_GRACE", 2),
|
||||
hand_swap_lr=_bool_std("HAND_SWAP_LR", False),
|
||||
hand_near_min=_float("HAND_NEAR_MIN", 0.10),
|
||||
hand_near_off=_float("HAND_NEAR_OFF", 0.08),
|
||||
hand_hold_frames=_int("HAND_HOLD_FRAMES", 2),
|
||||
hand_face_min=_float("HAND_FACE_MIN", 0.5),
|
||||
arkit_bone_max=_float("ARKIT_BONE_MAX", 0.5),
|
||||
# ARKIT_FULL_SKELETON uses != "0" convention (empty = True)
|
||||
arkit_full_skeleton=_bool_ne0("ARKIT_FULL_SKELETON", True),
|
||||
viz_hud=_bool_std("VIZ_HUD", False),
|
||||
viz_source_keys=_bool_std("VIZ_SOURCE_KEYS", False),
|
||||
viz_audio_keys=_bool_std("VIZ_AUDIO_KEYS", False),
|
||||
viz_default_mode=_str("VIZ_DEFAULT_MODE", "auto"),
|
||||
skel_line_width=_int("SKEL_LINE_WIDTH", 4),
|
||||
iphone_osc_port=_int("IPHONE_OSC_PORT", 57128),
|
||||
concert_mirror=_bool_ne0("CONCERT_MIRROR", True),
|
||||
# ICP_FUSION / MULTIHMR_AUTOCAST use strict "1" convention
|
||||
icp_fusion=_bool_eq1("ICP_FUSION"),
|
||||
icp_lidar_host=_str("ICP_LIDAR_HOST"),
|
||||
icp_lidar_port=_int("ICP_LIDAR_PORT", 5500),
|
||||
icp_lidar_extrinsic=_str("ICP_LIDAR_EXTRINSIC"),
|
||||
multihmr_backend=(_str("MULTIHMR_BACKEND", "pytorch") or "pytorch").strip().lower(),
|
||||
multihmr_loop_fps=_float("MULTIHMR_LOOP_FPS", 30.0),
|
||||
multihmr_autocast=_bool_eq1("MULTIHMR_AUTOCAST"),
|
||||
multihmr_remote_host=_str("MULTIHMR_REMOTE_HOST", "192.168.0.175") or "192.168.0.175",
|
||||
multihmr_remote_port=_int("MULTIHMR_REMOTE_PORT", 57140),
|
||||
# MULTIHMR_REMOTE_JPEG/ASYNC use _env_flag convention
|
||||
multihmr_remote_jpeg=_bool_flag("MULTIHMR_REMOTE_JPEG", True),
|
||||
multihmr_remote_jpeg_quality=_int("MULTIHMR_REMOTE_JPEG_QUALITY", 80),
|
||||
multihmr_remote_async=_bool_flag("MULTIHMR_REMOTE_ASYNC", True),
|
||||
multihmr_reid=_str_lower("MULTIHMR_REID", "dino"),
|
||||
multihmr_reid_alpha=_float("MULTIHMR_REID_ALPHA", 0.5),
|
||||
coreml_mlpackage=_str("COREML_MLPACKAGE"),
|
||||
coreml_compute_units=(_str("COREML_COMPUTE_UNITS", "") or "").strip().lower(),
|
||||
avbody_host=_str("AVBODY_HOST", "127.0.0.1") or "127.0.0.1",
|
||||
vdmx_osc_host=_str("VDMX_OSC_HOST"),
|
||||
vdmx_osc_port=_int("VDMX_OSC_PORT", 1234),
|
||||
# AV_LIVE_* use != "0" convention (enabled by default when unset)
|
||||
av_live_mediapipe=_bool_ne0("AV_LIVE_MEDIAPIPE", True),
|
||||
av_live_apple_vision=_bool_ne0("AV_LIVE_APPLE_VISION", True),
|
||||
av_live_coreml=_bool_ne0("AV_LIVE_COREML", True),
|
||||
av_live_detrpose=_bool_eq1("AV_LIVE_DETRPOSE"),
|
||||
av_live_parallel_pose=_str("AV_LIVE_PARALLEL_POSE", "both") or "both",
|
||||
mesh_rig=_bool_ne0("MESH_RIG", True),
|
||||
)
|
||||
@@ -334,11 +334,16 @@ def _load_py_config(path: str):
|
||||
from omegaconf import OmegaConf
|
||||
# Les configs DETRPose sont des fichiers Python qui exposent un dict
|
||||
# `model = LazyCall(...)`. On utilise le helper lazy_config si dispo.
|
||||
try:
|
||||
from src.misc.lazy_config import LazyConfig # type: ignore
|
||||
return LazyConfig.load(path)
|
||||
except ImportError:
|
||||
pass
|
||||
# DETRPose exposes LazyConfig from src.core (see tools/inference/torch_inf.py);
|
||||
# the older src.misc.lazy_config path is kept as a fallback. LazyConfig.load
|
||||
# handles the config's relative imports (`from .include...`) that a raw exec
|
||||
# cannot.
|
||||
for _mod in ("src.core", "src.misc.lazy_config"):
|
||||
try:
|
||||
mod = __import__(_mod, fromlist=["LazyConfig"])
|
||||
return mod.LazyConfig.load(path)
|
||||
except (ImportError, AttributeError):
|
||||
continue
|
||||
# Fallback minimal : exec + recup des noms cles.
|
||||
ns: dict = {}
|
||||
with open(path) as f:
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
"""Air-piano finger strike detection from pre-routed hand slots.
|
||||
|
||||
A "strike" is a fast downward motion of a fingertip RELATIVE to its base
|
||||
knuckle, so translating the whole hand does not fire all fingers. Output feeds
|
||||
the OSC /pose/finger route consumed by SuperCollider.
|
||||
|
||||
Consumers call route_hands() (hand_slots) once per tick and pass the resulting
|
||||
2-element [left|None, right|None] list to FingerStrikeDetector.step() and
|
||||
PinchDetector.step(). slot 0 = left hand (matrix "MG"), slot 1 = right.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
from data_only_viz.hand_features import _clamp, _coord, _finite
|
||||
|
||||
# MediaPipe 21-kp hand: fingertip and base-knuckle indices per finger
|
||||
# (thumb, index, middle, ring, pinky). Thumb base = ThumbMP (2).
|
||||
FINGERTIPS: tuple[int, ...] = (4, 8, 12, 16, 20)
|
||||
FINGER_BASES: tuple[int, ...] = (2, 5, 9, 13, 17)
|
||||
|
||||
# Pinch: thumb tip vs the 4 opposable finger tips (index..pinky).
|
||||
THUMB_TIP: int = 4
|
||||
WRIST: int = 0
|
||||
MIDDLE_MCP: int = 9
|
||||
PINCH_TIPS: tuple[int, ...] = (8, 12, 16, 20)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrikeEvent:
|
||||
hand: int # 0 = left slot, 1 = right slot
|
||||
finger: int # 0..4 = thumb, index, middle, ring, pinky
|
||||
strike_speed: float
|
||||
z: float
|
||||
tipx: float
|
||||
tipy: float
|
||||
|
||||
|
||||
class _FingerState:
|
||||
__slots__ = ("prev_rel", "armed", "last_t")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.prev_rel: float | None = None
|
||||
self.armed: bool = True
|
||||
self.last_t: float = -1e9
|
||||
|
||||
|
||||
class FingerStrikeDetector:
|
||||
def __init__(self, vel_thresh: float = 0.02, refractory_ms: float = 120.0,
|
||||
speed_scale: float = 0.10, history_slots: int = 2) -> None:
|
||||
self.vel_thresh = vel_thresh
|
||||
self.refractory_s = refractory_ms / 1000.0
|
||||
self.speed_scale = max(1e-6, speed_scale)
|
||||
# state[slot][finger]
|
||||
self._state = [[_FingerState() for _ in range(5)]
|
||||
for _ in range(history_slots)]
|
||||
|
||||
def reset_slot(self, slot: int) -> None:
|
||||
"""Clear per-finger prev_rel and re-arm for the given slot.
|
||||
|
||||
Call this BEFORE step() when GestureSlotStabilizer.resumed_flags()
|
||||
reports a held -> real transition for that slot. During a Vision hole
|
||||
the stabilizer replays the last hand (frozen coords, vel=0), but when
|
||||
the real hand returns within the hold window the first delta compresses
|
||||
several frames of motion into one large velocity spike, causing a
|
||||
phantom strike. Clearing prev_rel makes the first real frame a
|
||||
no-op prime that restarts tracking from the new position.
|
||||
"""
|
||||
for f in range(5):
|
||||
self._state[slot][f].prev_rel = None
|
||||
self._state[slot][f].armed = True
|
||||
|
||||
def step(self, slotted: list, t_now: float) -> list[StrikeEvent]:
|
||||
"""Process a pre-routed 2-slot hand list [left|None, right|None].
|
||||
|
||||
Absent slots (None) reset their finger state so re-entry does not
|
||||
produce spurious velocity spikes.
|
||||
"""
|
||||
events: list[StrikeEvent] = []
|
||||
for slot, lm in enumerate(slotted):
|
||||
if lm is None:
|
||||
for f in range(5):
|
||||
self._state[slot][f].prev_rel = None
|
||||
self._state[slot][f].armed = True
|
||||
continue
|
||||
for f in range(5):
|
||||
tip = lm[FINGERTIPS[f]]
|
||||
base = lm[FINGER_BASES[f]]
|
||||
tip_y = _finite(_coord(tip, "y", 1), 0.5)
|
||||
base_y = _finite(_coord(base, "y", 1), 0.5)
|
||||
rel = tip_y - base_y # +down (image y grows downward)
|
||||
st = self._state[slot][f]
|
||||
if st.prev_rel is None:
|
||||
st.prev_rel = rel
|
||||
continue
|
||||
vel = rel - st.prev_rel # +down velocity per frame
|
||||
st.prev_rel = rel
|
||||
if vel < 0.0: # lifting -> rearm
|
||||
st.armed = True
|
||||
if (vel > self.vel_thresh and st.armed
|
||||
and (t_now - st.last_t) >= self.refractory_s):
|
||||
st.armed = False
|
||||
st.last_t = t_now
|
||||
events.append(StrikeEvent(
|
||||
hand=slot, finger=f,
|
||||
strike_speed=_clamp(vel / self.speed_scale, 0.0, 1.0),
|
||||
z=_finite(_coord(tip, "z", 2, 0.0), 0.0),
|
||||
tipx=_finite(_coord(tip, "x", 0), 0.5),
|
||||
tipy=tip_y,
|
||||
))
|
||||
return events
|
||||
|
||||
|
||||
@dataclass
|
||||
class PinchEvent:
|
||||
hand: int # 0 = left slot, 1 = right slot
|
||||
finger: int # 1..4 = index, middle, ring, pinky (thumb is trigger)
|
||||
state: int = 1 # 1 = engage edge, 0 = release edge
|
||||
|
||||
|
||||
class _PinchState:
|
||||
__slots__ = ("engaged", "last_t", "qual")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.engaged: bool = False
|
||||
self.last_t: float = -1e9
|
||||
self.qual: int = 0 # consecutive qualifying frames (debounce)
|
||||
|
||||
|
||||
class PinchDetector:
|
||||
"""Edge-triggered thumb-to-finger pinch with hysteresis.
|
||||
|
||||
Fires one PinchEvent when thumb tip contacts a finger tip (distance,
|
||||
normalized by hand size, drops below ratio_on). Re-arms only after the
|
||||
distance rises back above ratio_off, so one pinch = one event.
|
||||
"""
|
||||
|
||||
def __init__(self, ratio_on: float = 0.45, ratio_off: float = 0.65,
|
||||
refractory_ms: float = 250.0, history_slots: int = 2,
|
||||
margin: float = 0.20, ext_ratio: float = 1.35,
|
||||
ext_min: int = 0, debounce_frames: int = 1) -> None:
|
||||
self.ratio_on = ratio_on
|
||||
self.ratio_off = ratio_off
|
||||
self.refractory_s = refractory_ms / 1000.0
|
||||
# Only the single nearest fingertip may engage, and only if it is at
|
||||
# least `margin` (in size-normalized units) nearer than the runner-up.
|
||||
# Rejects the adjacent-finger ambiguity when fingers curl together.
|
||||
self.margin = margin
|
||||
# Open-hand gate: the winner may engage only when at least ext_min
|
||||
# of the 3 non-pinching fingers are extended (tip-to-wrist distance
|
||||
# above ext_ratio hand-sizes). Rejects relaxed-hand/fist false
|
||||
# pinches during full-body play. ext_min=0 disables the gate; the
|
||||
# live defaults come from the PINCH_EXT_* env vars (action_head_pub),
|
||||
# constructor defaults preserve legacy behavior.
|
||||
self.ext_ratio = ext_ratio
|
||||
self.ext_min = int(ext_min)
|
||||
# Engage fires only after debounce_frames consecutive qualifying
|
||||
# frames; release stays immediate. 1 = no debounce.
|
||||
self.debounce_frames = max(1, int(debounce_frames))
|
||||
self._state = [[_PinchState() for _ in range(4)]
|
||||
for _ in range(history_slots)]
|
||||
|
||||
def engaged_slots(self) -> tuple[bool, bool]:
|
||||
"""Return (slot0_has_engaged_pinch, slot1_has_engaged_pinch)."""
|
||||
return (
|
||||
any(self._state[0][i].engaged for i in range(4)),
|
||||
any(self._state[1][i].engaged for i in range(4)),
|
||||
)
|
||||
|
||||
def step(self, slotted: list, t_now: float) -> list[PinchEvent]:
|
||||
"""Process a pre-routed 2-slot hand list [left|None, right|None].
|
||||
|
||||
Absent slots synthesise release edges for any engaged fingers and
|
||||
reset debounce state.
|
||||
"""
|
||||
events: list[PinchEvent] = []
|
||||
for slot, lm in enumerate(slotted):
|
||||
if lm is None:
|
||||
for i in range(4):
|
||||
st = self._state[slot][i]
|
||||
st.qual = 0
|
||||
if st.engaged:
|
||||
st.engaged = False
|
||||
events.append(PinchEvent(hand=slot, finger=i + 1, state=0))
|
||||
continue
|
||||
tx = _finite(_coord(lm[THUMB_TIP], "x", 0), 0.5)
|
||||
ty = _finite(_coord(lm[THUMB_TIP], "y", 1), 0.5)
|
||||
wx = _finite(_coord(lm[WRIST], "x", 0), 0.5)
|
||||
wy = _finite(_coord(lm[WRIST], "y", 1), 0.5)
|
||||
mx = _finite(_coord(lm[MIDDLE_MCP], "x", 0), 0.5)
|
||||
my = _finite(_coord(lm[MIDDLE_MCP], "y", 1), 0.5)
|
||||
size = math.hypot(mx - wx, my - wy)
|
||||
size = size if size > 1e-4 else 1e-4
|
||||
ratios = []
|
||||
exts = []
|
||||
for tip_idx in PINCH_TIPS:
|
||||
fx = _finite(_coord(lm[tip_idx], "x", 0), 0.5)
|
||||
fy = _finite(_coord(lm[tip_idx], "y", 1), 0.5)
|
||||
ratios.append(math.hypot(fx - tx, fy - ty) / size)
|
||||
exts.append(math.hypot(fx - wx, fy - wy) / size)
|
||||
# closest-wins + margin: pick the single nearest fingertip, and treat
|
||||
# it as a pinch only if it is clearly nearer than the runner-up.
|
||||
order = sorted(range(4), key=lambda j: ratios[j])
|
||||
nearest, runner = order[0], order[1]
|
||||
# open-hand gate: a deliberate pinch keeps the other fingers
|
||||
# extended; a fist/relaxed hand has them curled at the wrist.
|
||||
open_ok = self.ext_min <= 0 or sum(
|
||||
1 for j in range(4)
|
||||
if j != nearest and exts[j] >= self.ext_ratio
|
||||
) >= self.ext_min
|
||||
winner = nearest if (
|
||||
ratios[nearest] < self.ratio_on
|
||||
and (ratios[runner] - ratios[nearest]) >= self.margin
|
||||
and open_ok
|
||||
) else -1
|
||||
for i in range(4):
|
||||
st = self._state[slot][i]
|
||||
if st.engaged:
|
||||
# release when this finger opens, or another finger took over.
|
||||
if i != winner or ratios[i] > self.ratio_off:
|
||||
st.engaged = False
|
||||
events.append(PinchEvent(hand=slot, finger=i + 1, state=0))
|
||||
elif i == winner and (t_now - st.last_t) >= self.refractory_s:
|
||||
# engage only after debounce_frames consecutive qualifying
|
||||
# frames; release below stays edge-immediate.
|
||||
st.qual += 1
|
||||
if st.qual >= self.debounce_frames:
|
||||
st.engaged = True
|
||||
st.last_t = t_now
|
||||
st.qual = 0
|
||||
events.append(PinchEvent(hand=slot, finger=i + 1, state=1))
|
||||
else:
|
||||
st.qual = 0
|
||||
return events
|
||||
@@ -0,0 +1,522 @@
|
||||
"""Pure geometry helpers for hand overlay filtering and panel rendering.
|
||||
|
||||
No Metal, no numpy, no pyobjc — safe to unit-test anywhere.
|
||||
|
||||
Panel frame legend (renderer.py):
|
||||
pid 7 conf=0.25+0.75*q → status 0: absent (q=0); status 1: detected (q≥0.30)
|
||||
pid 8 conf=0.25+0.75*q → status 2: armed (near+facing camera, q≥0.30)
|
||||
pid 9 conf=1.0 (q forced)→ status 3: pinch engaged
|
||||
Thickness: 1 stroke always; +1 at q≥0.50; +1 at q≥0.85 or status==3.
|
||||
See gesture_quality() for the continuous quality formula.
|
||||
|
||||
Position gauges (renderer.py, drawn around each panel):
|
||||
X gauge: horizontal rail BELOW panel (pid 7, conf 0.4 or 0.25 if absent);
|
||||
vertical notch marker at cx position (pid 5/6, conf 1.0).
|
||||
Y gauge: vertical rail on outer side (pid 7, conf 0.4 or 0.25 if absent);
|
||||
horizontal notch marker at cy position (pid 5/6, conf 1.0).
|
||||
See gauge_segments() — marker x is mirror-flipped if mirror=True.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Panel layout constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Panel square size in normalized HEIGHT units. A panel occupies
|
||||
#: PANEL_SIDE * view_height pixels on both axes (square in pixel space).
|
||||
PANEL_SIDE: float = 0.30
|
||||
|
||||
#: Gap between the screen edge and the near edge of the panel (normalized).
|
||||
PANEL_MARGIN: float = 0.02
|
||||
|
||||
#: Fraction of each panel dimension removed from every side to form the
|
||||
#: inner rect used as the drawing area for hand wireframes.
|
||||
PANEL_INNER: float = 0.10
|
||||
|
||||
|
||||
class _HasXYC(Protocol):
|
||||
x: float
|
||||
y: float
|
||||
c: float
|
||||
|
||||
|
||||
def arkit_2d_fresh(ts_by_pid: dict, now: float, max_age: float = 1.0) -> bool:
|
||||
"""Return True if any pid has a timestamp younger than *max_age* seconds.
|
||||
|
||||
Args:
|
||||
ts_by_pid: mapping pid -> perf_counter timestamp (from
|
||||
``State.persons_arkit_2d_t``).
|
||||
now: current ``time.perf_counter()`` value.
|
||||
max_age: staleness threshold in seconds (exclusive: age < max_age).
|
||||
|
||||
Returns:
|
||||
True if at least one entry satisfies ``now - ts < max_age``.
|
||||
False if the dict is empty or every entry is stale.
|
||||
"""
|
||||
for ts in ts_by_pid.values():
|
||||
if (now - ts) < max_age:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def hand_size(kp: list[_HasXYC]) -> float:
|
||||
"""Euclidean distance wrist(0) -> middle-MCP(9) in normalised image units."""
|
||||
w = kp[0]
|
||||
m = kp[9]
|
||||
return math.hypot(m.x - w.x, m.y - w.y)
|
||||
|
||||
|
||||
def hand_facing(kp: list) -> float:
|
||||
"""Palm-spread ratio for camera-facing detection.
|
||||
|
||||
Returns dist(index-MCP kp[5], pinky-MCP kp[17]) / hand_size(kp).
|
||||
A palm facing the camera spans ~70-100 % of hand_size; a side-on
|
||||
hand collapses to < 40 %. Returns 0.0 when hand_size is ~0.
|
||||
|
||||
Accepts both attribute-style (.x/.y) and index-style ([x, y, ...]) kp.
|
||||
"""
|
||||
def _gx(p: object) -> float:
|
||||
return float(p.x) if hasattr(p, "x") else float(p[0]) # type: ignore[index]
|
||||
|
||||
def _gy(p: object) -> float:
|
||||
return float(p.y) if hasattr(p, "y") else float(p[1]) # type: ignore[index]
|
||||
|
||||
sz = math.hypot(_gx(kp[9]) - _gx(kp[0]), _gy(kp[9]) - _gy(kp[0]))
|
||||
if sz < 1e-6:
|
||||
return 0.0
|
||||
return math.hypot(_gx(kp[17]) - _gx(kp[5]), _gy(kp[17]) - _gy(kp[5])) / sz
|
||||
|
||||
|
||||
def hand_plausible(
|
||||
kp: list[_HasXYC],
|
||||
conf_min: float = 0.3,
|
||||
size_min: float = 0.02,
|
||||
size_max: float = 0.5,
|
||||
) -> bool:
|
||||
"""Return True if the hand landmark list looks like a real hand.
|
||||
|
||||
Rejects:
|
||||
- fewer than 21 landmarks
|
||||
- wrist (kp[0]) outside [-0.1, 1.1] in x or y (out-of-frame ghost anchor)
|
||||
- size (wrist->middle-MCP) outside [size_min, size_max]
|
||||
- mean confidence below conf_min
|
||||
"""
|
||||
if len(kp) < 21:
|
||||
return False
|
||||
w = kp[0]
|
||||
if not (-0.1 <= w.x <= 1.1 and -0.1 <= w.y <= 1.1):
|
||||
return False
|
||||
size = hand_size(kp)
|
||||
if size < size_min or size > size_max:
|
||||
return False
|
||||
mean_c = sum(p.c for p in kp) / len(kp)
|
||||
if mean_c < conf_min:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def segment_ok(
|
||||
A: _HasXYC,
|
||||
B: _HasXYC,
|
||||
size: float,
|
||||
conf_min: float = 0.3,
|
||||
max_bone_ratio: float = 1.2,
|
||||
) -> bool:
|
||||
"""Return True if the bone segment between A and B is plausible.
|
||||
|
||||
Rejects:
|
||||
- min endpoint confidence below conf_min
|
||||
- bone length exceeding max_bone_ratio * size
|
||||
"""
|
||||
if min(A.c, B.c) < conf_min:
|
||||
return False
|
||||
bone_len = math.hypot(B.x - A.x, B.y - A.y)
|
||||
if bone_len > max_bone_ratio * size:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Panel helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def panel_rect(side: str, aspect: float) -> tuple[float, float, float, float]:
|
||||
"""Normalized [0,1] screen rect of the hand side panel.
|
||||
|
||||
The panel is square in pixel space:
|
||||
pixel_width = norm_width * view_width = (PANEL_SIDE / aspect) * aspect * view_height
|
||||
= PANEL_SIDE * view_height = pixel_height.
|
||||
|
||||
Args:
|
||||
side: "left" or "right".
|
||||
aspect: view width / view height (> 0).
|
||||
|
||||
Returns:
|
||||
(x0, y0, x1, y1) in normalized [0,1] coords, y down, origin top-left.
|
||||
"""
|
||||
norm_w = PANEL_SIDE / aspect # normalized width (square in pixel space)
|
||||
norm_h = PANEL_SIDE # normalized height
|
||||
y0 = 0.5 - norm_h / 2.0
|
||||
y1 = 0.5 + norm_h / 2.0
|
||||
if side == "left":
|
||||
x0 = PANEL_MARGIN
|
||||
x1 = PANEL_MARGIN + norm_w
|
||||
else: # "right"
|
||||
x1 = 1.0 - PANEL_MARGIN
|
||||
x0 = x1 - norm_w
|
||||
return x0, y0, x1, y1
|
||||
|
||||
|
||||
def panel_frame(side: str, aspect: float) -> list[tuple[float, float, float, float]]:
|
||||
"""Four border segments tracing the panel rect (clock-wise, starting top).
|
||||
|
||||
Returns:
|
||||
List of (ax, ay, bx, by) in normalized [0,1] screen coords, y down.
|
||||
"""
|
||||
x0, y0, x1, y1 = panel_rect(side, aspect)
|
||||
return [
|
||||
(x0, y0, x1, y0), # top edge (left→right)
|
||||
(x1, y0, x1, y1), # right edge (top→bottom)
|
||||
(x1, y1, x0, y1), # bottom edge (right→left)
|
||||
(x0, y1, x0, y0), # left edge (bottom→top)
|
||||
]
|
||||
|
||||
|
||||
def panel_segments(
|
||||
kp: list[_HasXYC],
|
||||
side: str,
|
||||
bones: list[tuple[int, int]],
|
||||
aspect: float,
|
||||
mirror: bool = True,
|
||||
conf_min: float = 0.3,
|
||||
) -> list[tuple[float, float, float, float]]:
|
||||
"""Map hand landmarks into a side panel using uniform pixel-space scale.
|
||||
|
||||
The hand bounding box is fit into the inner panel rect (panel rect shrunk
|
||||
by PANEL_INNER fraction on each side) with a uniform pixel-space scale —
|
||||
meaning the pixel aspect ratio of the hand is preserved regardless of the
|
||||
view aspect ratio.
|
||||
|
||||
Scale derivation:
|
||||
s = min(inner_pixel_w / bbox_pixel_w, inner_pixel_h / bbox_pixel_h)
|
||||
= min(iw / kw, ih / kh)
|
||||
where iw/kw are normalized-coord ratios whose view dimensions cancel.
|
||||
Moving Δkp.x in kp-x corresponds to Δkp.x * s in normalized screen x, and
|
||||
Δkp.y in kp-y corresponds to Δkp.y * s in normalized screen y — preserving
|
||||
the original pixel ratio between any two segment endpoints.
|
||||
|
||||
When mirror=True, X is flipped within the panel so the zoomed hand matches
|
||||
the mirrored video background (i.e. the panel always shows a "front view").
|
||||
|
||||
Args:
|
||||
kp: 21+ keypoints with .x, .y (normalized image coords, y down), .c.
|
||||
side: "left" or "right".
|
||||
bones: List of (index_a, index_b) pairs defining which keypoints to connect.
|
||||
aspect: view width / view height.
|
||||
mirror: If True, flip X within the panel.
|
||||
conf_min: Minimum confidence for hand_plausible and segment_ok checks.
|
||||
|
||||
Returns:
|
||||
List of (ax, ay, bx, by) in normalized [0,1] screen coords, y down.
|
||||
Returns [] if hand_plausible(kp) fails.
|
||||
"""
|
||||
if not hand_plausible(kp, conf_min=conf_min):
|
||||
return []
|
||||
|
||||
x0, y0, x1, y1 = panel_rect(side, aspect)
|
||||
pw = x1 - x0
|
||||
ph = y1 - y0
|
||||
|
||||
# Inner rect: shrink by PANEL_INNER fraction on every side
|
||||
ix0 = x0 + PANEL_INNER * pw
|
||||
ix1 = x1 - PANEL_INNER * pw
|
||||
iy0 = y0 + PANEL_INNER * ph
|
||||
iy1 = y1 - PANEL_INNER * ph
|
||||
iw = ix1 - ix0
|
||||
ih = iy1 - iy0
|
||||
|
||||
# Keypoint bounding box in normalized image coords
|
||||
kx_min = min(p.x for p in kp)
|
||||
kx_max = max(p.x for p in kp)
|
||||
ky_min = min(p.y for p in kp)
|
||||
ky_max = max(p.y for p in kp)
|
||||
kw = kx_max - kx_min
|
||||
kh = ky_max - ky_min
|
||||
|
||||
if kw == 0.0 and kh == 0.0:
|
||||
return []
|
||||
|
||||
# Uniform pixel-space scale factor (view dimensions cancel in the ratio)
|
||||
if kw > 0.0 and kh > 0.0:
|
||||
s = min(iw / kw, ih / kh)
|
||||
elif kw > 0.0:
|
||||
s = iw / kw
|
||||
else:
|
||||
s = ih / kh
|
||||
|
||||
# Centre the scaled hand bbox inside the inner rect
|
||||
offset_x = (iw - kw * s) / 2.0
|
||||
offset_y = (ih - kh * s) / 2.0
|
||||
|
||||
def _map(p: _HasXYC) -> tuple[float, float]:
|
||||
rel_x = (kx_max - p.x) if mirror else (p.x - kx_min)
|
||||
nx = ix0 + offset_x + rel_x * s
|
||||
ny = iy0 + offset_y + (p.y - ky_min) * s
|
||||
return nx, ny
|
||||
|
||||
sz = hand_size(kp)
|
||||
result: list[tuple[float, float, float, float]] = []
|
||||
for a, b in bones:
|
||||
if a >= len(kp) or b >= len(kp):
|
||||
continue
|
||||
A = kp[a]
|
||||
B = kp[b]
|
||||
if not segment_ok(A, B, sz, conf_min=conf_min):
|
||||
continue
|
||||
ax, ay = _map(A)
|
||||
bx, by = _map(B)
|
||||
result.append((ax, ay, bx, by))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hand temporal persistence gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class HandPersistenceGate:
|
||||
"""Suppress ghost hand detections that only flash for 1-2 frames.
|
||||
|
||||
Tracks each hand by wrist position (landmark 0). A hand is only
|
||||
drawable once the same wrist position has been matched for
|
||||
*min_frames* consecutive calls.
|
||||
|
||||
With ``grace=0`` (default) a track that goes unmatched for even one
|
||||
frame is immediately dropped (resets the consecutive count). With
|
||||
``grace>0`` an unmatched track survives up to *grace* consecutive
|
||||
missed calls, so a single Vision drop-frame does not wipe the
|
||||
accumulated count and cause a 4-frame visual flicker.
|
||||
|
||||
min_frames=1 disables gating: every incoming hand is drawable on the
|
||||
first call (useful for the single-person MediaPipe fallback path).
|
||||
|
||||
Known limit: matching is greedy nearest-track — two hands crossing
|
||||
within `radius` can inherit each other's track (and a ghost appearing
|
||||
exactly where a real hand just left inherits its count). Grace widens
|
||||
that inheritance window to `grace` frames after the hand leaves, and a
|
||||
ghost re-flashing at one spot every (grace+1)th frame can accumulate to
|
||||
min_frames. Acceptable for a display gate; do not reuse for
|
||||
gesture-state tracking.
|
||||
"""
|
||||
|
||||
def __init__(self, min_frames: int = 3, radius: float = 0.15, grace: int = 0) -> None:
|
||||
self._min_frames = min_frames
|
||||
self._radius = radius
|
||||
self._grace = grace
|
||||
# Each track: [wrist_x, wrist_y, consecutive_count, miss_count]
|
||||
self._tracks: list[list] = []
|
||||
|
||||
def step(self, hands: list) -> list[bool]:
|
||||
"""Update tracks and return a draw flag per hand.
|
||||
|
||||
Args:
|
||||
hands: list of 21-landmark hand objects; landmark 0 is the wrist
|
||||
and must expose .x and .y attributes.
|
||||
|
||||
Returns:
|
||||
list[bool] of length len(hands). True = this hand has been
|
||||
seen at the same position for at least min_frames consecutive
|
||||
calls and should be drawn.
|
||||
"""
|
||||
result: list[bool] = [False] * len(hands)
|
||||
new_tracks: list[list] = []
|
||||
used: set[int] = set()
|
||||
|
||||
for h_idx, hand in enumerate(hands):
|
||||
try:
|
||||
wx = hand[0].x
|
||||
wy = hand[0].y
|
||||
except (TypeError, IndexError, AttributeError):
|
||||
continue # malformed entry: never drawable, no track
|
||||
|
||||
# Find the nearest existing track within radius (greedy).
|
||||
best_dist = float("inf")
|
||||
best_t = -1
|
||||
for t_idx, track in enumerate(self._tracks):
|
||||
if t_idx in used:
|
||||
continue
|
||||
tx, ty = track[0], track[1]
|
||||
d = math.hypot(wx - tx, wy - ty)
|
||||
if d < best_dist and d < self._radius:
|
||||
best_dist = d
|
||||
best_t = t_idx
|
||||
|
||||
if best_t >= 0:
|
||||
used.add(best_t)
|
||||
new_count = self._tracks[best_t][2] + 1
|
||||
new_tracks.append([wx, wy, new_count, 0]) # reset miss
|
||||
result[h_idx] = new_count >= self._min_frames
|
||||
else:
|
||||
# New track — starts at count 1, miss 0.
|
||||
new_tracks.append([wx, wy, 1, 0])
|
||||
result[h_idx] = 1 >= self._min_frames
|
||||
|
||||
# Carry forward unmatched tracks within grace window.
|
||||
for t_idx, track in enumerate(self._tracks):
|
||||
if t_idx not in used:
|
||||
miss = track[3] + 1
|
||||
if miss <= self._grace:
|
||||
new_tracks.append([track[0], track[1], track[2], miss])
|
||||
|
||||
self._tracks = new_tracks
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gauge layout constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Normalised gap between a panel edge and its adjacent gauge rail.
|
||||
GAUGE_GAP: float = 0.012
|
||||
|
||||
#: Half the tick length (total = 2 * GAUGE_TICK_HALF), in normalised coords.
|
||||
GAUGE_TICK_HALF: float = 0.004
|
||||
|
||||
#: Parallel offset for the second stroke of a bold-notch marker.
|
||||
GAUGE_BOLD: float = 0.001
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Continuous quality score
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def gesture_quality(
|
||||
hand: object,
|
||||
*,
|
||||
face_min: float,
|
||||
near_on: float,
|
||||
engaged: bool = False,
|
||||
) -> float:
|
||||
"""Continuous quality score ∈ [0, 1] for one gesture slot.
|
||||
|
||||
Drives panel frame brightness and stroke thickness in renderer.py.
|
||||
|
||||
Args:
|
||||
hand: 21-kp hand object (attribute-style .x/.y) or None.
|
||||
``None`` → 0.0 (slot absent or not yet validated by gates).
|
||||
face_min: palm-spread ratio at which the slot is considered fully
|
||||
armed (from GestureSlotStabilizer; default env 0.5).
|
||||
near_on: wrist-to-middle-MCP distance (normalised) at which the slot
|
||||
is fully near (from GestureSlotStabilizer; default 0.10).
|
||||
engaged: True when a pinch is held for this slot → forces 1.0.
|
||||
|
||||
Returns:
|
||||
quality ∈ [0, 1]:
|
||||
* 0.0 when ``hand`` is None.
|
||||
* 1.0 when ``engaged`` is True.
|
||||
* else: 0.30·established + 0.35·facing_norm + 0.35·near_norm
|
||||
where established = 1.0 (hand passed plausibility + temporal gates),
|
||||
facing_norm = clamp((hand_facing(hand) − 0.25) / (face_min − 0.25), 0, 1),
|
||||
near_norm = clamp((hand_size(hand) − 0.5·near_on) / (0.5·near_on), 0, 1).
|
||||
"""
|
||||
if hand is None:
|
||||
return 0.0
|
||||
if engaged:
|
||||
return 1.0
|
||||
# facing_norm: 0 at facing = 0.25, 1 at facing = face_min.
|
||||
denom_f = face_min - 0.25
|
||||
if denom_f > 1e-9:
|
||||
facing_norm = min(1.0, max(0.0, (hand_facing(hand) - 0.25) / denom_f))
|
||||
else:
|
||||
# face_min ≤ 0.25 — any spread qualifies.
|
||||
facing_norm = 1.0
|
||||
# near_norm: 0 at hand_size = 0.5·near_on, 1 at hand_size = near_on.
|
||||
half_on = 0.5 * near_on
|
||||
if half_on > 1e-9:
|
||||
near_norm = min(1.0, max(0.0, (hand_size(hand) - half_on) / half_on))
|
||||
else:
|
||||
near_norm = 1.0
|
||||
return 0.30 + 0.35 * facing_norm + 0.35 * near_norm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Position gauges
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def gauge_segments(
|
||||
cx: float | None,
|
||||
cy: float | None,
|
||||
side: str,
|
||||
aspect: float,
|
||||
mirror: bool = False,
|
||||
*,
|
||||
content_pid: int = 5,
|
||||
rail_pid: int = 7,
|
||||
) -> list[tuple[float, float, float, float, float, int]]:
|
||||
"""Rail + marker segments for the X and Y position gauges around a panel.
|
||||
|
||||
X gauge: a horizontal rail placed GAUGE_GAP below the panel bottom edge,
|
||||
spanning the full panel width. A bold vertical notch marks the
|
||||
hand's cx position on the rail.
|
||||
|
||||
Y gauge: a vertical rail placed GAUGE_GAP outside the panel's outer edge
|
||||
(left of the left panel, right of the right panel), spanning the
|
||||
full panel height. A bold horizontal notch marks cy.
|
||||
|
||||
Args:
|
||||
cx: Normalised hand centre-x ∈ [0, 1], or None when absent.
|
||||
cy: Normalised hand centre-y ∈ [0, 1], or None when absent.
|
||||
side: "left" or "right".
|
||||
aspect: View width / view height (> 0).
|
||||
mirror: If True, flip the cx marker position (1 − cx) so it moves
|
||||
in the same direction as the mirrored video. cy is never
|
||||
flipped.
|
||||
content_pid: pid for marker segments (5 = left slot, 6 = right slot).
|
||||
rail_pid: pid for rail segments (default 7).
|
||||
|
||||
Returns:
|
||||
List of (ax, ay, bx, by, conf, pid) tuples ready to feed
|
||||
``push_panel`` in the renderer. When cx/cy is None the rails are
|
||||
drawn dim (conf 0.25) with no marker; otherwise rails use conf 0.4
|
||||
and markers use conf 1.0.
|
||||
"""
|
||||
x0, y0, x1, y1 = panel_rect(side, aspect)
|
||||
has_data = cx is not None and cy is not None
|
||||
rail_conf = 0.4 if has_data else 0.25
|
||||
out: list[tuple[float, float, float, float, float, int]] = []
|
||||
|
||||
# ---- X gauge: horizontal rail below panel ----
|
||||
xg_y = y1 + GAUGE_GAP
|
||||
out.append((x0, xg_y, x1, xg_y, rail_conf, rail_pid))
|
||||
if has_data:
|
||||
assert cx is not None and cy is not None # narrowing for type checkers
|
||||
cx_eff = 1.0 - cx if mirror else cx
|
||||
mx = x0 + cx_eff * (x1 - x0)
|
||||
# Bold vertical notch: two parallel vertical ticks
|
||||
out.append((mx, xg_y - GAUGE_TICK_HALF, mx, xg_y + GAUGE_TICK_HALF,
|
||||
1.0, content_pid))
|
||||
out.append((mx + GAUGE_BOLD, xg_y - GAUGE_TICK_HALF,
|
||||
mx + GAUGE_BOLD, xg_y + GAUGE_TICK_HALF,
|
||||
1.0, content_pid))
|
||||
|
||||
# ---- Y gauge: vertical rail on outer side ----
|
||||
if side == "left":
|
||||
yg_x = x0 - GAUGE_GAP
|
||||
else:
|
||||
yg_x = x1 + GAUGE_GAP
|
||||
out.append((yg_x, y0, yg_x, y1, rail_conf, rail_pid))
|
||||
if has_data:
|
||||
assert cx is not None and cy is not None
|
||||
my = y0 + cy * (y1 - y0)
|
||||
# Bold horizontal notch: two parallel horizontal ticks
|
||||
out.append((yg_x - GAUGE_TICK_HALF, my, yg_x + GAUGE_TICK_HALF, my,
|
||||
1.0, content_pid))
|
||||
out.append((yg_x - GAUGE_TICK_HALF, my + GAUGE_BOLD,
|
||||
yg_x + GAUGE_TICK_HALF, my + GAUGE_BOLD,
|
||||
1.0, content_pid))
|
||||
|
||||
return out
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Turn MediaPipe hand landmarks into a compact expressive feature vector.
|
||||
|
||||
Mirrors the kinematics pattern of action_head.FeatureExtractor: a small ring
|
||||
buffer per hand slot, finite-guarded, clamped to declared ranges. Output feeds
|
||||
both the OSC /pose/hands route (audio) and the Metal renderer uniforms (visual).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
WRIST, THUMB_TIP, MIDDLE_MCP, PINKY_TIP = 0, 4, 9, 20
|
||||
|
||||
# index/middle/ring/pinky tips + their MCP knuckles, for the "all fingers
|
||||
# curled" fist metric (thumb excluded — a fist curls the four fingers).
|
||||
_FINGER_TIPS = (8, 12, 16, 20)
|
||||
_FINGER_MCPS = (5, 9, 13, 17)
|
||||
|
||||
NEUTRAL_HAND = {"cx": 0.5, "cy": 0.5, "openness": 0.0, "speed": 0.0, "fist": 0.0}
|
||||
|
||||
# openness calibration: (span/size) maps fist~0.3 -> 0, open~2.0 -> 1
|
||||
_OPEN_LO, _OPEN_HI = 0.5, 2.0
|
||||
# fist calibration: per-finger tip->MCP distance / hand size. curled ~0.25,
|
||||
# extended ~0.75. closedness = 1 at/below LO, 0 at/above HI.
|
||||
_EXT_LO, _EXT_HI = 0.25, 0.75
|
||||
|
||||
|
||||
def _finite(v: float, fallback: float) -> float:
|
||||
return v if isinstance(v, float) and math.isfinite(v) else fallback
|
||||
|
||||
|
||||
def _coord(p, attr: str, idx: int, fallback: float = 0.5) -> float:
|
||||
"""Extract a coordinate from a landmark object or an indexable row."""
|
||||
if hasattr(p, attr):
|
||||
return float(getattr(p, attr))
|
||||
try:
|
||||
return float(p[idx])
|
||||
except (TypeError, IndexError):
|
||||
return fallback
|
||||
|
||||
|
||||
def _clamp(v: float, lo: float, hi: float) -> float:
|
||||
return lo if v < lo else hi if v > hi else v
|
||||
|
||||
|
||||
class HandFeatureExtractor:
|
||||
def __init__(self, history: int = 5, fist_enable: bool | None = None):
|
||||
self._history = max(2, history)
|
||||
# two routed slots (hand_slots.route_hands): 0 = user's left (L),
|
||||
# 1 = user's right (R); cx fallback when chirality is absent.
|
||||
self._buf = [deque(maxlen=self._history), deque(maxlen=self._history)]
|
||||
# Master fist kill-switch: user-disabled live 2026-07-02 (too many
|
||||
# false held-fist actions). fist stays 0.0 for BOTH hands whatever
|
||||
# the per-slot fist_enabled kwarg says. FIST_ENABLE=1 re-enables.
|
||||
# TODO: migrate the env read to VizConfig once config.py is free.
|
||||
if fist_enable is None:
|
||||
import os
|
||||
fist_enable = os.environ.get("FIST_ENABLE", "0") not in (
|
||||
"0", "", "false", "False",
|
||||
)
|
||||
self._fist_enable = fist_enable
|
||||
|
||||
def _features(self, lm: list) -> dict:
|
||||
xs = [_finite(_coord(p, "x", 0), 0.5) for p in lm[:21]]
|
||||
ys = [_finite(_coord(p, "y", 1), 0.5) for p in lm[:21]]
|
||||
# cx/cy anchor = the WRIST (kp 0), not the landmark centroid: the
|
||||
# centroid shifts when fingers open/pinch, cross-talking gestures
|
||||
# into the X/Y voice mods. The wrist decouples them (user request
|
||||
# live 2026-07-02).
|
||||
cx = _clamp(xs[WRIST], 0.0, 1.0)
|
||||
cy = _clamp(ys[WRIST], 0.0, 1.0)
|
||||
size = math.hypot(xs[MIDDLE_MCP] - xs[WRIST], ys[MIDDLE_MCP] - ys[WRIST])
|
||||
size = size if size > 1e-4 else 1e-4
|
||||
span = math.hypot(xs[THUMB_TIP] - xs[PINKY_TIP],
|
||||
ys[THUMB_TIP] - ys[PINKY_TIP])
|
||||
openness = _clamp((span / size - _OPEN_LO) / (_OPEN_HI - _OPEN_LO),
|
||||
0.0, 1.0)
|
||||
# fist = 1.0 only when ALL four fingers are curled to the palm (min over
|
||||
# fingers): one extended finger drops it toward 0, so a true fist is
|
||||
# required rather than just a small thumb-pinky span.
|
||||
fist = 1.0
|
||||
for tip, mcp in zip(_FINGER_TIPS, _FINGER_MCPS):
|
||||
ext = math.hypot(xs[tip] - xs[mcp], ys[tip] - ys[mcp]) / size
|
||||
curled = _clamp(1.0 - (ext - _EXT_LO) / (_EXT_HI - _EXT_LO), 0.0, 1.0)
|
||||
fist = min(fist, curled)
|
||||
return {"cx": cx, "cy": cy, "openness": openness, "fist": fist}
|
||||
|
||||
def _speed(self, slot: int, cx: float, cy: float) -> float:
|
||||
buf = self._buf[slot]
|
||||
spd = 0.0
|
||||
if buf:
|
||||
px, py = buf[-1]
|
||||
spd = math.hypot(cx - px, cy - py)
|
||||
buf.append((cx, cy))
|
||||
return _clamp(spd, 0.0, 1.0)
|
||||
|
||||
def step(self, hands: list, chirality=None, swap: bool = False,
|
||||
fist_enabled: tuple = (True, True), mirror: bool = False) -> dict:
|
||||
from data_only_viz.hand_slots import route_hands
|
||||
# swap must mirror the gesture path (HAND_SWAP_LR) or per-voice
|
||||
# L/R mods contradict pinch/strike in the same performance.
|
||||
# mirror must match state.mirror_2d so the cx fallback slot assignment
|
||||
# stays coherent with the renderer panel layout on chirality dropout.
|
||||
left_lm, right_lm = route_hands(hands, chirality, swap=swap, mirror=mirror)
|
||||
|
||||
if left_lm is None and right_lm is None:
|
||||
self._buf[0].clear()
|
||||
self._buf[1].clear()
|
||||
return {"L": None, "R": None, "dist": 0.0}
|
||||
|
||||
out_l = None
|
||||
out_r = None
|
||||
|
||||
if left_lm is not None:
|
||||
lf = self._features(left_lm)
|
||||
if not (self._fist_enable and fist_enabled[0]):
|
||||
lf["fist"] = 0.0
|
||||
lf["speed"] = self._speed(0, lf["cx"], lf["cy"])
|
||||
out_l = lf
|
||||
else:
|
||||
self._buf[0].clear()
|
||||
|
||||
if right_lm is not None:
|
||||
rf = self._features(right_lm)
|
||||
if not (self._fist_enable and fist_enabled[1]):
|
||||
rf["fist"] = 0.0
|
||||
rf["speed"] = self._speed(1, rf["cx"], rf["cy"])
|
||||
out_r = rf
|
||||
else:
|
||||
self._buf[1].clear()
|
||||
|
||||
dist = 0.0
|
||||
if out_l is not None and out_r is not None:
|
||||
dist = _clamp(
|
||||
math.hypot(out_r["cx"] - out_l["cx"], out_r["cy"] - out_l["cy"]),
|
||||
0.0, 1.0,
|
||||
)
|
||||
|
||||
return {"L": out_l, "R": out_r, "dist": dist}
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Hand L/R slotting: route raw hands to a fixed [left|None, right|None] pair.
|
||||
|
||||
Chirality path (iPhone ARBodyTracker Vision): if `chirality` is aligned with
|
||||
`hands` (same length), route chir==0 -> slot 0 (left), chir==1 -> slot 1 (right).
|
||||
First match per slot wins; extras dropped.
|
||||
|
||||
Fallback (chirality absent/misaligned): sort valid hands by screen cx ascending
|
||||
(flipped when mirror=True) and assign leftmost to slot 0, next to slot 1.
|
||||
|
||||
`swap=True`: inverts the chirality INTERPRETATION only (chir 0 -> right slot,
|
||||
1 -> left slot). Safety knob for a source delivering flipped chirality;
|
||||
ARBodyTracker chirality was validated CORRECT live 2026-07-02, so
|
||||
HAND_SWAP_LR defaults to 0 in the consumers. The cx fallback is
|
||||
screen-relative and is never swapped.
|
||||
|
||||
`near_min > 0`: after routing, replace a slot with None when the hand's
|
||||
wrist-to-middle-MCP distance (normalised) is below `near_min`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
# ---- micro-helpers imported from hand_features (acyclic: hand_features only
|
||||
# imports hand_slots lazily inside step(), so a top-level import here is safe) ---
|
||||
from .hand_features import _finite, _coord, _clamp
|
||||
from .hand_display import hand_facing as _hand_facing
|
||||
|
||||
|
||||
def _hand_size(hand) -> float:
|
||||
"""Wrist(0) -> middle-MCP(9) distance; works for both attr and list kp formats."""
|
||||
w = hand[0]
|
||||
m = hand[9]
|
||||
wx = _finite(_coord(w, "x", 0), 0.5)
|
||||
wy = _finite(_coord(w, "y", 1), 0.5)
|
||||
mx = _finite(_coord(m, "x", 0), 0.5)
|
||||
my = _finite(_coord(m, "y", 1), 0.5)
|
||||
return math.hypot(mx - wx, my - wy)
|
||||
|
||||
|
||||
def _is_valid(hand) -> bool:
|
||||
try:
|
||||
return hand is not None and len(hand) >= 21
|
||||
except TypeError:
|
||||
return False
|
||||
|
||||
|
||||
def route_hands(
|
||||
hands,
|
||||
chirality=None,
|
||||
*,
|
||||
mirror: bool = False,
|
||||
swap: bool = False,
|
||||
near_min: float = 0.0,
|
||||
) -> list: # always [left|None, right|None]
|
||||
"""Route hands to a 2-element [left, right] slot list.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
hands: raw hand list (any length, may contain None / short entries).
|
||||
chirality: int list aligned 1-to-1 with `hands` (0=left, 1=right).
|
||||
None / wrong length -> cx fallback.
|
||||
mirror: flip cx for the fallback path so screen-left stays slot 0.
|
||||
swap: invert the chirality mapping (0 -> right, 1 -> left). Safety
|
||||
knob for a source whose Vision chirality arrives inverted.
|
||||
Never touches the cx fallback (already screen-relative).
|
||||
near_min: replace a routed slot with None when hand_size < near_min.
|
||||
"""
|
||||
chir_aligned = (
|
||||
chirality is not None
|
||||
and len(chirality) == len(hands)
|
||||
and len(chirality) > 0
|
||||
)
|
||||
|
||||
# Build valid (chirality_value_or_None, hand) pairs — BEFORE filtering invalids
|
||||
# so chirality[i] stays paired with hands[i] even when hands[j] is skipped.
|
||||
valid_pairs: list[tuple] = []
|
||||
for i, hand in enumerate(hands):
|
||||
if not _is_valid(hand):
|
||||
continue
|
||||
c = chirality[i] if chir_aligned else None
|
||||
valid_pairs.append((c, hand))
|
||||
|
||||
slots: list = [None, None]
|
||||
|
||||
if chir_aligned and any(c is not None for c, _ in valid_pairs):
|
||||
# ---- Chirality path: chir 0 -> slot 0 (left), 1 -> slot 1 (right);
|
||||
# swap inverts the mapping when the source chirality is flipped ----
|
||||
for c, hand in valid_pairs:
|
||||
target = c if c in (0, 1) else None
|
||||
if target is not None and swap:
|
||||
target = 1 - target
|
||||
if target is not None and slots[target] is None:
|
||||
slots[target] = hand
|
||||
else:
|
||||
# ---- cx fallback: leftmost on screen -> slot 0 ----
|
||||
def _cx(h):
|
||||
xs = [_finite(_coord(p, "x", 0), 0.5) for p in h[:21]]
|
||||
raw = _clamp(sum(xs) / len(xs), 0.0, 1.0)
|
||||
return (1.0 - raw) if mirror else raw
|
||||
|
||||
ordered = sorted((h for _, h in valid_pairs), key=_cx)
|
||||
if ordered:
|
||||
slots[0] = ordered[0]
|
||||
if len(ordered) >= 2:
|
||||
slots[1] = ordered[1]
|
||||
|
||||
# ---- near_min gate (applied after routing, both paths) ----
|
||||
if near_min > 0.0:
|
||||
for i in range(2):
|
||||
if slots[i] is not None and _hand_size(slots[i]) < near_min:
|
||||
slots[i] = None
|
||||
|
||||
return slots
|
||||
|
||||
|
||||
class GestureSlotStabilizer:
|
||||
"""Stateful per-slot stabilizer for the gesture detection path.
|
||||
|
||||
Applied AFTER route_hands (with near_min=0.0) and BEFORE the gesture
|
||||
detectors (FingerStrikeDetector, PinchDetector).
|
||||
|
||||
Two behaviours:
|
||||
- **Hold-last**: when a slot is None this call but held a hand within the
|
||||
last ``hold_frames`` calls, output the LAST seen hand for that slot.
|
||||
Fills 1-2 frame Vision holes so pinch debounce never sees a spurious
|
||||
absence. After ``hold_frames`` consecutive misses the slot yields None
|
||||
for real.
|
||||
- **Near hysteresis**: a slot ACTIVATES when hand_size >= ``near_on`` and
|
||||
stays active until hand_size < ``near_off``. Inactive slot -> None output
|
||||
(but hold-last still tracks so re-activation is instant).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hold_frames: int = 2,
|
||||
near_on: float = 0.10,
|
||||
near_off: float = 0.08,
|
||||
face_min: float = 0.5,
|
||||
) -> None:
|
||||
self._hold = hold_frames
|
||||
self._near_on = near_on
|
||||
self._near_off = near_off
|
||||
self._face_min = face_min
|
||||
# face_off uses 0.8× hysteresis (mirrors the near on/off pattern) so that
|
||||
# a slot active at face_min does not flap when facing oscillates near the
|
||||
# threshold — it only deactivates when facing drops below face_off.
|
||||
self._face_off = 0.8 * face_min
|
||||
# Per-slot state
|
||||
self._last: list = [None, None] # last seen hand object
|
||||
self._miss: list[int] = [0, 0] # consecutive miss count
|
||||
self._near_active: list[bool] = [False, False]
|
||||
self._face_active: list[bool] = [False, False]
|
||||
self._out: list = [None, None] # last step() output (for active_flags)
|
||||
# Hold-tracking: True when previous step() output was a held replay.
|
||||
self._held_flag: list[bool] = [False, False]
|
||||
# Resumed: True when this step() transitioned held -> real for that slot.
|
||||
self._resumed: list[bool] = [False, False]
|
||||
|
||||
def step(self, slotted: list) -> list:
|
||||
"""Stabilize a 2-element [left|None, right|None] slot list.
|
||||
|
||||
Args:
|
||||
slotted: output of route_hands (near_min=0.0) — length 2.
|
||||
|
||||
Returns:
|
||||
Stabilized 2-element list; each slot is the stabilized hand or None.
|
||||
"""
|
||||
out: list = [None, None]
|
||||
for i in range(2):
|
||||
hand = slotted[i] if i < len(slotted) else None
|
||||
prev_held = self._held_flag[i]
|
||||
if hand is not None:
|
||||
# Real hand present this frame.
|
||||
# Transition held -> real: mark resumed so callers can reset
|
||||
# per-finger state before consuming this step's output.
|
||||
self._resumed[i] = prev_held
|
||||
self._held_flag[i] = False
|
||||
# Update near hysteresis
|
||||
size = _hand_size(hand)
|
||||
if self._near_active[i]:
|
||||
if size < self._near_off:
|
||||
self._near_active[i] = False
|
||||
else:
|
||||
if size >= self._near_on:
|
||||
self._near_active[i] = True
|
||||
# Update face hysteresis (face_off = 0.8 * face_min)
|
||||
facing = _hand_facing(hand)
|
||||
if self._face_active[i]:
|
||||
if facing < self._face_off:
|
||||
self._face_active[i] = False
|
||||
else:
|
||||
if facing >= self._face_min:
|
||||
self._face_active[i] = True
|
||||
# Track the hand regardless of near/face state
|
||||
self._last[i] = hand
|
||||
self._miss[i] = 0
|
||||
# Output only if near-active AND facing the camera
|
||||
out[i] = hand if (self._near_active[i] and self._face_active[i]) else None
|
||||
else:
|
||||
# Hand absent this frame
|
||||
self._resumed[i] = False
|
||||
self._miss[i] += 1
|
||||
if self._miss[i] <= self._hold:
|
||||
# Hold: output last hand only if it was near+face-active
|
||||
if (self._near_active[i] and self._face_active[i]
|
||||
and self._last[i] is not None):
|
||||
self._held_flag[i] = True
|
||||
out[i] = self._last[i]
|
||||
else:
|
||||
# Not active: no hold replay; slot stays None
|
||||
self._held_flag[i] = False
|
||||
else:
|
||||
# Past hold limit — clear both gates so next appearance
|
||||
# must re-qualify from near_on and face_min.
|
||||
self._held_flag[i] = False
|
||||
self._near_active[i] = False
|
||||
self._face_active[i] = False
|
||||
out[i] = None
|
||||
self._out = out
|
||||
return out
|
||||
|
||||
def active_flags(self) -> tuple:
|
||||
"""Return (slot0_active, slot1_active).
|
||||
|
||||
True when the last step() yielded a non-None for that slot — meaning the
|
||||
hand is established (held or present) AND within the near zone. Used by
|
||||
action_head_pub._emit_hands to gate the fist feature.
|
||||
"""
|
||||
return (self._out[0] is not None, self._out[1] is not None)
|
||||
|
||||
def resumed_flags(self) -> tuple[bool, bool]:
|
||||
"""Return (slot0_resumed, slot1_resumed).
|
||||
|
||||
True for a slot when the current step() transitioned from a "held
|
||||
replay" (Vision hole, last hand replayed) back to a real hand. Callers
|
||||
that track per-frame deltas (e.g. FingerStrikeDetector) should reset
|
||||
their state for that slot BEFORE consuming the step output to avoid
|
||||
phantom events caused by the compressed motion accumulated during the
|
||||
hold window.
|
||||
|
||||
Semantics: True only for the ONE step where the transition occurs;
|
||||
False on all subsequent frames where the hand stays real, and False
|
||||
after an expired hold (miss > hold_frames, slot went None) since the
|
||||
absent-slot logic in gesture detectors already resets per-finger state.
|
||||
"""
|
||||
return (self._resumed[0], self._resumed[1])
|
||||
@@ -0,0 +1,339 @@
|
||||
"""IphoneUSBSource — a cv2-VideoCapture-shaped frame source backed by the
|
||||
iPhone ARBodyTracker USB stream. Decodes the AVLiveWire HEVC video to BGR
|
||||
frames for MediaPipe, and (Task 2) writes ARKit skeleton + Vision hands into
|
||||
State. Substitutes for cv2.VideoCapture in multi.py under --iphone-usb."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import socket
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
|
||||
try:
|
||||
import av
|
||||
except ImportError: # pragma: no cover — av missing is caught at runtime by caller
|
||||
av = None # type: ignore[assignment]
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from data_only_viz.scripts.iphone_usb_bridge import (
|
||||
connect_device, decode_skeleton, decode_skeleton2D,
|
||||
iter_frames, TAG_SKELETON, TAG_SKELETON2D,
|
||||
)
|
||||
from data_only_viz.arkit_topology import decode_topology
|
||||
from data_only_viz.state import PoseKp
|
||||
|
||||
TAG_VIDEO = 2
|
||||
TAG_HANDS = 4
|
||||
TAG_TOPOLOGY = 7
|
||||
LOG = logging.getLogger("iphone_usb_source")
|
||||
|
||||
_HAND_BYTES = 1 + 21 * 12 # chirality:u8 + 21 × (x,y,z) × 4 bytes BE f32
|
||||
_ARKIT_JOINTS = 91
|
||||
|
||||
|
||||
# Normalized-coordinate counterpart of multi._apply_video_rotate (np.rot90
|
||||
# conventions): the iPhone computes 2D landmarks on the UNROTATED sensor
|
||||
# frame, so when VIDEO_ROTATE turns the displayed video the 2D points must
|
||||
# turn the same way or overlays/panels/gestures go misaligned. ARKit 3D
|
||||
# world joints are gravity-aligned and unaffected.
|
||||
_ROT_XY = {
|
||||
"ccw": lambda x, y: (y, 1.0 - x),
|
||||
"cw": lambda x, y: (1.0 - y, x),
|
||||
"180": lambda x, y: (1.0 - x, 1.0 - y),
|
||||
}
|
||||
|
||||
|
||||
def rotate_norm_xy(x: float, y: float, mode: str) -> "tuple[float, float]":
|
||||
"""Rotate a normalized (x, y) like np.rot90 rotates the frame."""
|
||||
f = _ROT_XY.get(mode)
|
||||
return f(x, y) if f else (x, y)
|
||||
|
||||
|
||||
def effective_point_rotate(mode: str, mirror: bool) -> str:
|
||||
"""Point-rotation mode equivalent to the video pipeline's transform.
|
||||
|
||||
The VIDEO is horizontally flipped at decode (mirror) THEN rotated by
|
||||
multi.py; the POINTS are rotated here THEN flipped at display time by
|
||||
the renderer. Flip and 90-degree rotations do not commute: under the
|
||||
mirror the point rotation must be the mirror-conjugate of the video
|
||||
rotation (cw <-> ccw swapped; 180 commutes)."""
|
||||
if mirror and mode in ("cw", "ccw"):
|
||||
return "cw" if mode == "ccw" else "ccw"
|
||||
return mode
|
||||
|
||||
|
||||
def _make_point(x: float, y: float, z: float, conf: float) -> PoseKp:
|
||||
"""Wrap raw floats from the wire into a PoseKp landmark.
|
||||
|
||||
z receives the raw wire value; c is clamped to [0, 1] from conf (≈ Vision
|
||||
confidence, transmitted in the z slot of the wire format).
|
||||
"""
|
||||
return PoseKp(x=x, y=y, z=z, c=max(0.0, min(1.0, conf)))
|
||||
|
||||
|
||||
def _decode_hands(
|
||||
payload: bytes,
|
||||
) -> "tuple[list[list[PoseKp]], list[int]] | None":
|
||||
"""Decode TAG_HANDS payload.
|
||||
|
||||
Returns ``(hands, chirality)`` where *chirality[i]* is 0=left / 1=right
|
||||
(wire byte 1=right, any other value → 0=left). Returns ``([], [])`` when
|
||||
count==0. Returns ``None`` only on empty payload.
|
||||
|
||||
Wire layout: count:u8, then per hand: chirality:u8 (1=right) followed by
|
||||
21 × (x, y, z) big-endian f32. x,y are normalized image coords; z≈confidence.
|
||||
Landmark .z = raw wire z; .c = clamp(z, 0, 1).
|
||||
"""
|
||||
if not payload:
|
||||
return None
|
||||
n = payload[0]
|
||||
if n == 0:
|
||||
return ([], [])
|
||||
off = 1
|
||||
hands: list[list[PoseKp]] = []
|
||||
chirality: list[int] = []
|
||||
for _ in range(n):
|
||||
if off + _HAND_BYTES > len(payload):
|
||||
break
|
||||
chir_byte = payload[off]
|
||||
chirality.append(1 if chir_byte == 1 else 0)
|
||||
off += 1
|
||||
pts = struct.unpack(">" + "f" * 63, payload[off:off + 21 * 12])
|
||||
off += 21 * 12
|
||||
hand = [_make_point(pts[i * 3], pts[i * 3 + 1], pts[i * 3 + 2],
|
||||
pts[i * 3 + 2])
|
||||
for i in range(21)]
|
||||
hands.append(hand)
|
||||
return (hands, chirality)
|
||||
|
||||
|
||||
def _to_annexb(data: bytes) -> bytes:
|
||||
out = bytearray(); i = 0
|
||||
while i + 4 <= len(data):
|
||||
n = int.from_bytes(data[i:i + 4], "big"); i += 4
|
||||
if i + n > len(data):
|
||||
break
|
||||
out += b"\x00\x00\x00\x01" + data[i:i + n]; i += n
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def apply_skeleton_joints(
|
||||
prev_arr: "np.ndarray | None",
|
||||
joints: "list[tuple[float, float, float, bool]]",
|
||||
n_joints: int = _ARKIT_JOINTS,
|
||||
) -> np.ndarray:
|
||||
"""Return a NEW array with valid joints from *joints* applied.
|
||||
|
||||
Joints not marked valid this frame keep their values from *prev_arr*
|
||||
(copy semantics — never mutates *prev_arr*). Callers holding a
|
||||
reference to the previous array continue to see a consistent snapshot
|
||||
even without a lock, eliminating the tearing race described in B4.
|
||||
|
||||
Args:
|
||||
prev_arr: Array currently stored for this pid, or None / wrong shape.
|
||||
joints: Iterable of (x, y, z, valid) from decode_skeleton().
|
||||
n_joints: Expected joint count (default _ARKIT_JOINTS = 91).
|
||||
|
||||
Returns:
|
||||
A fresh np.ndarray of shape (n_joints, 3), dtype float32.
|
||||
"""
|
||||
if prev_arr is None or prev_arr.shape != (n_joints, 3):
|
||||
new_arr = np.zeros((n_joints, 3), dtype=np.float32)
|
||||
else:
|
||||
new_arr = prev_arr.copy()
|
||||
for i, (x, y, z, valid) in enumerate(joints):
|
||||
if valid:
|
||||
new_arr[i] = (x, y, z)
|
||||
return new_arr
|
||||
|
||||
|
||||
class IphoneUSBSource:
|
||||
def __init__(self, state=None, target_size=(640, 480),
|
||||
write_hands: bool = True, mirror: bool = True) -> None:
|
||||
self.state = state
|
||||
self.target_w, self.target_h = target_size
|
||||
self._write_hands = write_hands
|
||||
# Mirror the video horizontally so the performer facing the camera
|
||||
# interacts naturally (move right -> goes right; raise right arm ->
|
||||
# the right side responds). Env CONCERT_MIRROR=0 disables it.
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
self._mirror = mirror and _cfg.concert_mirror
|
||||
# The iPhone app delivers BOTH 2D streams in an already-uprighted
|
||||
# frame in portrait: Vision hands run with a hardcoded orientation
|
||||
# (.right, ARBodySession.swift) and the ARKit skeleton2D projection
|
||||
# follows the interface orientation. VIDEO_ROTATE therefore only
|
||||
# rotates the VIDEO (multi.py); the 2D points are NOT rotated by
|
||||
# default. Two independent knobs remain for exotic app/device
|
||||
# combinations (mirror-conjugated, see effective_point_rotate):
|
||||
# IPHONE_SKEL_ROTATE and IPHONE_HANDS_ROTATE (none/ccw/cw/180).
|
||||
import os as _os
|
||||
self._skel_rotate = effective_point_rotate(
|
||||
_os.environ.get("IPHONE_SKEL_ROTATE", "none"), self._mirror)
|
||||
self._hands_rotate = effective_point_rotate(
|
||||
_os.environ.get("IPHONE_HANDS_ROTATE", "none"), self._mirror)
|
||||
self._codec = av.codec.CodecContext.create("hevc", "r") if av is not None else None
|
||||
self._lock = threading.Lock()
|
||||
self._frame = None # latest BGR np.ndarray
|
||||
self._stop = threading.Event()
|
||||
self._thread = None
|
||||
self._sock = None # stored for shutdown() in release()
|
||||
self._opened = False
|
||||
|
||||
def start(self) -> bool:
|
||||
# Try an immediate connect, but do NOT give up if the iPhone app isn't
|
||||
# streaming yet: start the read thread regardless. _run retries the
|
||||
# connect (with backoff) until the app appears, so a startup race (app
|
||||
# not ready / phone locked at launch) recovers the same way a mid-session
|
||||
# drop does -- no data_only_viz restart needed.
|
||||
sock = connect_device()
|
||||
if sock is None:
|
||||
LOG.warning("iphone usb: no device yet -- thread will keep retrying")
|
||||
else:
|
||||
self._sock = sock
|
||||
self._opened = True
|
||||
self._thread = threading.Thread(
|
||||
target=self._run, args=(sock,), name="iphone_usb_src", daemon=True)
|
||||
self._thread.start()
|
||||
return True
|
||||
|
||||
def isOpened(self) -> bool:
|
||||
return self._opened
|
||||
|
||||
def _run(self, sock) -> None:
|
||||
if self.state is not None:
|
||||
with self.state.lock():
|
||||
self.state.mirror_2d = self._mirror
|
||||
while not self._stop.is_set():
|
||||
if sock is None: # no connection yet (startup or post-drop)
|
||||
sock = self._reconnect()
|
||||
if sock is None: # release() was called while waiting
|
||||
break
|
||||
self._sock = sock
|
||||
self._opened = True
|
||||
LOG.info("iphone usb connected")
|
||||
try:
|
||||
for tag, pid, payload in iter_frames(sock):
|
||||
if self._stop.is_set():
|
||||
break
|
||||
if tag == TAG_VIDEO and len(payload) > 1:
|
||||
annexb = _to_annexb(payload[1:])
|
||||
if self._codec is not None and av is not None:
|
||||
try:
|
||||
for fr in self._codec.decode(av.Packet(annexb)):
|
||||
img = fr.to_ndarray(format="bgr24")
|
||||
img = cv2.resize(img, (self.target_w, self.target_h))
|
||||
if self._mirror:
|
||||
img = cv2.flip(img, 1)
|
||||
with self._lock:
|
||||
self._frame = img
|
||||
except Exception as e: # av.AVError is a removal-prone alias
|
||||
LOG.debug("hevc decode: %s", e)
|
||||
elif tag == TAG_SKELETON and self.state is not None:
|
||||
joints = decode_skeleton(payload)
|
||||
if joints is not None:
|
||||
with self.state.lock():
|
||||
prev = self.state.persons_arkit_joints.get(pid)
|
||||
new_arr = apply_skeleton_joints(prev, joints)
|
||||
self.state.persons_arkit_joints[pid] = new_arr
|
||||
self.state.persons_arkit_last_t[pid] = time.perf_counter()
|
||||
elif tag == TAG_SKELETON2D and self.state is not None:
|
||||
pts = decode_skeleton2D(payload)
|
||||
if pts is not None:
|
||||
_rot = self._skel_rotate
|
||||
arr = np.array(
|
||||
[rotate_norm_xy(x, y, _rot)
|
||||
for (x, y, _v) in pts],
|
||||
dtype=np.float32)
|
||||
valid = np.array([v for (_x, _y, v) in pts],
|
||||
dtype=bool)
|
||||
with self.state.lock():
|
||||
self.state.persons_arkit_2d[pid] = arr
|
||||
self.state.persons_arkit_2d_valid[pid] = valid
|
||||
self.state.persons_arkit_2d_t[pid] = \
|
||||
time.perf_counter()
|
||||
elif tag == TAG_TOPOLOGY and self.state is not None:
|
||||
topo = decode_topology(payload)
|
||||
if topo is not None:
|
||||
names, parents = topo
|
||||
with self.state.lock():
|
||||
self.state.arkit_joint_names = names
|
||||
self.state.arkit_parents = parents
|
||||
elif tag == TAG_HANDS:
|
||||
result = _decode_hands(payload)
|
||||
if result is not None and self.state is not None:
|
||||
hands, chirality = result
|
||||
if self._hands_rotate != "none":
|
||||
for hand in hands:
|
||||
for p in hand:
|
||||
p.x, p.y = rotate_norm_xy(
|
||||
p.x, p.y, self._hands_rotate)
|
||||
with self.state.lock():
|
||||
# Always expose iPhone Vision hands (stable,
|
||||
# rotation-invariant) for the air-piano.
|
||||
self.state.persons_hands_iphone = hands
|
||||
self.state.persons_hands_iphone_t = \
|
||||
time.perf_counter()
|
||||
self.state.persons_hands_chirality = chirality
|
||||
if self._write_hands:
|
||||
self.state.persons_hands = hands
|
||||
except OSError as e:
|
||||
LOG.warning("iphone usb stream error: %s", e)
|
||||
finally:
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
# Stream ended. Drop the stale frame so read() reports no-frame,
|
||||
# then reconnect (unless stopping) so a transient USB/app drop
|
||||
# doesn't freeze the pose pipeline (the iPhone backgrounding or a
|
||||
# USB hiccup ends iter_frames; we re-establish :7000).
|
||||
with self._lock:
|
||||
self._frame = None
|
||||
# Clear stale hand state so the renderer doesn't freeze on the
|
||||
# last known hands after a stream drop.
|
||||
if self.state is not None:
|
||||
with self.state.lock():
|
||||
self.state.persons_hands_iphone = []
|
||||
self.state.persons_hands_chirality = []
|
||||
if self._write_hands:
|
||||
self.state.persons_hands = []
|
||||
self._opened = False
|
||||
sock = None # forces the reconnect at the loop top
|
||||
if not self._stop.is_set():
|
||||
LOG.info("iphone usb stream ended -- reconnecting...")
|
||||
self._opened = False
|
||||
|
||||
def _reconnect(self):
|
||||
"""Retry connect_device() with capped backoff until it succeeds or
|
||||
release() is called. Returns the new socket, or None if stopping."""
|
||||
delay = 0.5
|
||||
while not self._stop.is_set():
|
||||
sock = connect_device()
|
||||
if sock is not None:
|
||||
return sock
|
||||
self._stop.wait(delay)
|
||||
delay = min(delay * 2, 3.0)
|
||||
return None
|
||||
|
||||
def read(self):
|
||||
with self._lock:
|
||||
if self._frame is None or not self._opened:
|
||||
return False, None
|
||||
return True, self._frame.copy()
|
||||
|
||||
def set(self, *args) -> bool:
|
||||
return True # cv2 CAP_PROP_* no-op
|
||||
|
||||
def release(self) -> None:
|
||||
self._stop.set()
|
||||
if self._sock is not None:
|
||||
try:
|
||||
self._sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=2.0)
|
||||
self._opened = False
|
||||
@@ -54,7 +54,8 @@ def load_extrinsic(path: Path | None = None) -> Extrinsic:
|
||||
|
||||
|
||||
def _path_from_env() -> Path:
|
||||
p = os.environ.get("ICP_LIDAR_EXTRINSIC")
|
||||
from .config import VizConfig
|
||||
p = VizConfig.from_env().icp_lidar_extrinsic
|
||||
return Path(p) if p else DEFAULT_EXTRINSIC_PATH
|
||||
|
||||
|
||||
|
||||
+122
-57
@@ -71,6 +71,10 @@ class AppDelegate(NSObject):
|
||||
# Sender vers sclang pour les changements de scene depuis le clavier
|
||||
self._scClient = SimpleUDPClient("127.0.0.1", opts.sclang_port)
|
||||
self._pose_worker = None
|
||||
from .config import VizConfig as _VizConfig
|
||||
_kcfg = _VizConfig.from_env()
|
||||
self._source_keys = _kcfg.viz_source_keys
|
||||
self._audio_keys = _kcfg.viz_audio_keys
|
||||
return self
|
||||
|
||||
def applicationDidFinishLaunching_(self, notification): # noqa: N802
|
||||
@@ -161,23 +165,39 @@ class AppDelegate(NSObject):
|
||||
self._window.makeFirstResponder_(self._container)
|
||||
LOG.info("window shown + key focus forced + floating level")
|
||||
|
||||
# 2b) HUD : overlay NSTextView semi-transparent au-dessus du MTKView.
|
||||
# NSTextView prend en charge le rendu CoreText sans avoir a passer
|
||||
# par un MTLBuffer texte. On le pose comme subview du MTKView.
|
||||
self._hud = NSTextView.alloc().initWithFrame_(
|
||||
NSMakeRect(12, 12, 340, 240))
|
||||
self._hud.setEditable_(False)
|
||||
self._hud.setSelectable_(False)
|
||||
self._hud.setDrawsBackground_(True)
|
||||
self._hud.setBackgroundColor_(
|
||||
NSColor.colorWithCalibratedRed_green_blue_alpha_(0, 0, 0, 0.45))
|
||||
self._hud.setFont_(NSFont.fontWithName_size_("Menlo", 11))
|
||||
self._hud.setTextColor_(NSColor.whiteColor())
|
||||
self._hud.setAutoresizingMask_(NSViewHeightSizable)
|
||||
self._mtkview.addSubview_(self._hud)
|
||||
# Timer 10 Hz, rafraichit le texte avec les valeurs du State.
|
||||
self._hudTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
|
||||
0.1, self, "refreshHud:", None, True)
|
||||
# 2b.1) Keyboard monitors — installed here so they are always active
|
||||
# regardless of which pose backend is chosen (or even when --pose is
|
||||
# omitted). The original placement at the END of _start_pose_worker
|
||||
# was unreachable because every normal backend path returns early.
|
||||
# Global monitor is read-only; we ignore its return to avoid
|
||||
# double-triggering alongside the local monitor.
|
||||
self._kb_monitor = NSEvent.addLocalMonitorForEventsMatchingMask_handler_(
|
||||
NSEventMaskKeyDown, self._on_key)
|
||||
self._kb_global = NSEvent.addGlobalMonitorForEventsMatchingMask_handler_(
|
||||
NSEventMaskKeyDown, self._on_key_global)
|
||||
LOG.info("keyboard monitors installed (local + global)")
|
||||
|
||||
# 2b.2) HUD : overlay NSTextView semi-transparent au-dessus du MTKView.
|
||||
# OFF by default (performance display); VIZ_HUD=1 brings the debug
|
||||
# panel back. It would also collide with the left-hand side panel.
|
||||
self._hud = None
|
||||
self._hudTimer = None
|
||||
from .config import VizConfig as _VizConfig
|
||||
if _VizConfig.from_env().viz_hud:
|
||||
self._hud = NSTextView.alloc().initWithFrame_(
|
||||
NSMakeRect(12, 12, 340, 240))
|
||||
self._hud.setEditable_(False)
|
||||
self._hud.setSelectable_(False)
|
||||
self._hud.setDrawsBackground_(True)
|
||||
self._hud.setBackgroundColor_(
|
||||
NSColor.colorWithCalibratedRed_green_blue_alpha_(0, 0, 0, 0.45))
|
||||
self._hud.setFont_(NSFont.fontWithName_size_("Menlo", 11))
|
||||
self._hud.setTextColor_(NSColor.whiteColor())
|
||||
self._hud.setAutoresizingMask_(NSViewHeightSizable)
|
||||
self._mtkview.addSubview_(self._hud)
|
||||
# Timer 10 Hz, rafraichit le texte avec les valeurs du State.
|
||||
self._hudTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
|
||||
0.1, self, "refreshHud:", None, True)
|
||||
|
||||
# 2c) Timer webcam update (NSImageView fullscreen deja cree en 2a)
|
||||
self._camTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
|
||||
@@ -186,9 +206,19 @@ class AppDelegate(NSObject):
|
||||
# 2d) Auto-engage du mode openpos (#9) quand des personnes sont
|
||||
# detectees. Si l'utilisateur a force un mode au clavier dans
|
||||
# les 8 dernieres secondes, on n'override pas (lock manuel).
|
||||
# VIZ_DEFAULT_MODE=<name> fixes the boot mode and DISABLES the
|
||||
# auto-switch entirely (user request: t/voronoi always active).
|
||||
self._user_viz_lock_t = 0.0 # set par _on_key, lu par autoOpenpos
|
||||
self._autoOpenposTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
|
||||
0.5, self, "autoOpenpos:", None, True)
|
||||
from .config import VizConfig as _VC
|
||||
_vmode = _VC.from_env().viz_default_mode
|
||||
self._autoOpenposTimer = None
|
||||
if _vmode != "auto" and _vmode in self._state.viz_mode_names:
|
||||
with self._state.lock():
|
||||
self._state.viz_mode = self._state.viz_mode_names.index(_vmode)
|
||||
LOG.info("[viz] fixed default mode %s (auto-openpos disabled)", _vmode)
|
||||
else:
|
||||
self._autoOpenposTimer = NSTimer.scheduledTimerWithTimeInterval_target_selector_userInfo_repeats_(
|
||||
0.5, self, "autoOpenpos:", None, True)
|
||||
|
||||
if self._opts.fullscreen:
|
||||
self._window.toggleFullScreen_(None)
|
||||
@@ -207,8 +237,16 @@ class AppDelegate(NSObject):
|
||||
vienne du MAIN THREAD AppKit. OpenCV depuis un worker thread plante
|
||||
avec 'can not spin main run loop from other thread'. On demande ici,
|
||||
sur le main thread, PUIS on lance le pose worker avec
|
||||
OPENCV_AVFOUNDATION_SKIP_AUTH=1 pour qu'il ne tente pas une 2e demande."""
|
||||
OPENCV_AVFOUNDATION_SKIP_AUTH=1 pour qu'il ne tente pas une 2e demande.
|
||||
|
||||
Exception: when --iphone-usb is set, IphoneUSBSource connects via
|
||||
usbmuxd — no Mac-camera TCC needed — so we skip straight to the worker.
|
||||
"""
|
||||
import os
|
||||
if getattr(self._opts, "iphone_usb", False):
|
||||
LOG.info("iphone-usb mode — skipping camera TCC request")
|
||||
self._start_pose_worker()
|
||||
return
|
||||
os.environ["OPENCV_AVFOUNDATION_SKIP_AUTH"] = "1"
|
||||
try:
|
||||
from AVFoundation import (
|
||||
@@ -249,15 +287,17 @@ class AppDelegate(NSObject):
|
||||
# 2. Apple Vision body pose (fallback si MediaPipe casse)
|
||||
# 3. CoreML pose, DETRPose, Holistic, YOLO — fallbacks
|
||||
import os as _os
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
# iPhone ARBodyTracker (option 2 LiDAR fusion) : always-on
|
||||
# listener on :57128. Harmless if no iPhone is broadcasting ;
|
||||
# state.persons_arkit_joints stays empty and the arkit_fuse
|
||||
# stage no-ops. Activated via POSE_FILTER=...+arkit_fuse.
|
||||
try:
|
||||
from .iphone_osc_listener import IphoneOSCListener
|
||||
self._iphone_osc = IphoneOSCListener(self._state)
|
||||
self._iphone_osc = IphoneOSCListener(self._state, port=_cfg.iphone_osc_port)
|
||||
self._iphone_osc.start()
|
||||
LOG.info("worker: + iPhone OSC listener :57128")
|
||||
LOG.info("worker: + iPhone OSC listener :%d", _cfg.iphone_osc_port)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("iphone OSC listener start failed (%s)", e)
|
||||
# ICP LiDAR fusion (opt-in via ICP_FUSION=1). Parallel to the
|
||||
@@ -265,8 +305,8 @@ class AppDelegate(NSObject):
|
||||
# joints. Requires a calibrated extrinsic on disk (see
|
||||
# scripts/calibrate_lidar.py) and an iPhone LiDAR stream
|
||||
# broadcasting on ICP_LIDAR_HOST:ICP_LIDAR_PORT.
|
||||
if _os.environ.get("ICP_FUSION", "0") == "1":
|
||||
host = _os.environ.get("ICP_LIDAR_HOST")
|
||||
if _cfg.icp_fusion:
|
||||
host = _cfg.icp_lidar_host
|
||||
if not host:
|
||||
LOG.warning("ICP_FUSION=1 but ICP_LIDAR_HOST unset — "
|
||||
"fusion disabled")
|
||||
@@ -276,11 +316,11 @@ class AppDelegate(NSObject):
|
||||
self._icp_fusion = IcpFusionThread(
|
||||
self._state,
|
||||
host=host,
|
||||
port=int(_os.environ.get("ICP_LIDAR_PORT", "5500")),
|
||||
port=_cfg.icp_lidar_port,
|
||||
)
|
||||
self._icp_fusion.start()
|
||||
LOG.info("worker: + ICP LiDAR fusion -> %s:%s", host,
|
||||
_os.environ.get("ICP_LIDAR_PORT", "5500"))
|
||||
LOG.info("worker: + ICP LiDAR fusion -> %s:%d",
|
||||
host, _cfg.icp_lidar_port)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("icp fusion start failed (%s)", e)
|
||||
# 0. Multi-HMR (SMPL-X 10475 verts mesh dense) — opt-in via flag
|
||||
@@ -289,9 +329,14 @@ class AppDelegate(NSObject):
|
||||
from .multi_hmr_worker import MultiHMRWorker
|
||||
from .smplx_osc_sender import SMPLXTCPSender
|
||||
if MultiHMRWorker.is_available():
|
||||
# target_fps=30 : the worker loop used to self-throttle
|
||||
# at 10 fps (sleep(period - dt)). With the async remote
|
||||
# backend (drop-newest in / latest out queue), we want
|
||||
# the loop to spin at camera rate so we always submit
|
||||
# the freshest frame and drain the freshest result.
|
||||
self._pose_worker = MultiHMRWorker(
|
||||
self._state, num_persons=4,
|
||||
target_fps=10.0,
|
||||
target_fps=_cfg.multihmr_loop_fps,
|
||||
device=getattr(self._opts, "pose_device", "mps"),
|
||||
det_thresh=getattr(self._opts, "det_thresh", 0.15),
|
||||
nms_kernel_size=getattr(
|
||||
@@ -305,7 +350,7 @@ class AppDelegate(NSObject):
|
||||
# issues introduced by the hybrid rigging path).
|
||||
self._smplx_tcp = SMPLXTCPSender(
|
||||
self._state,
|
||||
enable_rigging=os.environ.get("MESH_RIG", "1") != "0",
|
||||
enable_rigging=_cfg.mesh_rig,
|
||||
)
|
||||
self._smplx_tcp.start()
|
||||
LOG.info("worker: Multi-HMR + SMPL-X (mesh dense)")
|
||||
@@ -321,8 +366,7 @@ class AppDelegate(NSObject):
|
||||
# AV_LIVE_PARALLEL_POSE=apple_vision pour ne garder que
|
||||
# le path ANE (face/hand fin disparait), ou =mediapipe
|
||||
# pour ne garder que CPU.
|
||||
parallel = _os.environ.get(
|
||||
"AV_LIVE_PARALLEL_POSE", "both")
|
||||
parallel = _cfg.av_live_parallel_pose
|
||||
if parallel in ("apple_vision", "both"):
|
||||
try:
|
||||
from .apple_vision_pose import AppleVisionPoseWorker
|
||||
@@ -357,10 +401,18 @@ class AppDelegate(NSObject):
|
||||
# 1. MediaPipe Multi : DEFAUT pour le mapping sonore + openpos
|
||||
# (33 body + 478 face + 21x2 hands × 4 personnes). Skip via
|
||||
# AV_LIVE_MEDIAPIPE=0 si on prefere body-only ANE-accelere.
|
||||
if _os.environ.get("AV_LIVE_MEDIAPIPE") != "0":
|
||||
if _cfg.av_live_mediapipe:
|
||||
try:
|
||||
from .multi import MultiWorker
|
||||
self._pose_worker = MultiWorker(self._state, num_persons=4)
|
||||
_iphone_usb = getattr(self._opts, "iphone_usb", False)
|
||||
# iphone-usb path: camera decode + ARKit conversion is cheap
|
||||
# (no MediaPipe inference on the Mac). Frame-based gesture
|
||||
# calibrations (debounce/hold/grace/persist) assume 30 Hz.
|
||||
# Keep 18 Hz for the webcam MediaPipe path (M5 CPU protection).
|
||||
self._pose_worker = MultiWorker(
|
||||
self._state, num_persons=4,
|
||||
iphone_usb=_iphone_usb,
|
||||
target_fps=30.0 if _iphone_usb else 18.0)
|
||||
self._pose_worker.start()
|
||||
LOG.info("worker: MediaPipe Multi (Pose+Face+Hand × 4)")
|
||||
return
|
||||
@@ -368,7 +420,7 @@ class AppDelegate(NSObject):
|
||||
LOG.warning("MediaPipe Multi unavailable (%s) — fallback", e)
|
||||
# 2. Apple Vision body pose : fallback si MediaPipe casse.
|
||||
# Body only, 13 joints, pas de face/hands.
|
||||
if _os.environ.get("AV_LIVE_APPLE_VISION") != "0":
|
||||
if _cfg.av_live_apple_vision:
|
||||
try:
|
||||
from .apple_vision_pose import AppleVisionPoseWorker
|
||||
if AppleVisionPoseWorker.is_available():
|
||||
@@ -382,7 +434,7 @@ class AppDelegate(NSObject):
|
||||
"(macOS < 11 ?) — fallback")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("Apple Vision pose indisponible (%s) — fallback", e)
|
||||
if _os.environ.get("AV_LIVE_COREML") != "0":
|
||||
if _cfg.av_live_coreml:
|
||||
try:
|
||||
from .coreml_pose import CoreMLPoseWorker
|
||||
if CoreMLPoseWorker.is_available():
|
||||
@@ -397,7 +449,7 @@ class AppDelegate(NSObject):
|
||||
"puis relancer pour activer le pipeline ANE")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("CoreML pose indisponible (%s) — fallback", e)
|
||||
if _os.environ.get("AV_LIVE_DETRPOSE") == "1":
|
||||
if _cfg.av_live_detrpose:
|
||||
try:
|
||||
from .detrpose import DETRPoseWorker, is_available
|
||||
if is_available():
|
||||
@@ -426,20 +478,6 @@ class AppDelegate(NSObject):
|
||||
self._state, device=self._opts.pose_device)
|
||||
self._pose_worker.start()
|
||||
|
||||
# 4) Hook clavier : local (app au focus) + global (app au fond).
|
||||
# Le global monitor est read-only mais permet de garder le pilotage
|
||||
# quand l'utilisateur a une autre app au premier plan (IDE,
|
||||
# browser). On ignore le retour pour le global (sinon double-trigger).
|
||||
self._kb_monitor = NSEvent.addLocalMonitorForEventsMatchingMask_handler_(
|
||||
NSEventMaskKeyDown, self._on_key)
|
||||
self._kb_global = NSEvent.addGlobalMonitorForEventsMatchingMask_handler_(
|
||||
NSEventMaskKeyDown, self._on_key_global)
|
||||
# Force le focus initial pour que le local monitor reçoive
|
||||
# tout de suite les touches sans nécessiter un clic.
|
||||
NSApp().activateIgnoringOtherApps_(True)
|
||||
self._window.makeKeyAndOrderFront_(None)
|
||||
self._window.makeFirstResponder_(self._container)
|
||||
|
||||
_cam_log_count = 0
|
||||
|
||||
def refreshCam_(self, _timer): # noqa: N802
|
||||
@@ -469,6 +507,8 @@ class AppDelegate(NSObject):
|
||||
def refreshHud_(self, _timer): # noqa: N802
|
||||
"""Format le HUD avec la valeur courante des flux + correspondance
|
||||
avec le rendu visuel. Appele a 10 Hz par NSTimer."""
|
||||
if self._hud is None:
|
||||
return
|
||||
s = self._state
|
||||
with s.lock():
|
||||
mode = s.viz_mode
|
||||
@@ -567,14 +607,16 @@ class AppDelegate(NSObject):
|
||||
LOG.debug("[key] raw=%r lower=%r", key, k)
|
||||
if key == "\x1b":
|
||||
NSApp().terminate_(self); return None
|
||||
if key == " ":
|
||||
self._scClient.send_message("/control/doScene", ["stop"])
|
||||
if key == " ": # espace : morceau suivant (concert)
|
||||
self._scClient.send_message("/control/concertNext", [1])
|
||||
return None
|
||||
# Cmd+F geree par macOS pour fullscreen ; on garde shift+F en raccourci
|
||||
if key == "F":
|
||||
self._window.toggleFullScreen_(None); return None
|
||||
if key == "H":
|
||||
self._hud.setHidden_(not self._hud.isHidden()); return None
|
||||
if self._hud is not None:
|
||||
self._hud.setHidden_(not self._hud.isHidden())
|
||||
return None
|
||||
if key == "C": # Shift+C : toggle webcam overlay
|
||||
self._cam.setHidden_(not self._cam.isHidden()); return None
|
||||
# azertyuiop -> video (viz mode)
|
||||
@@ -591,16 +633,37 @@ class AppDelegate(NSObject):
|
||||
self._user_viz_lock_t = _t.monotonic()
|
||||
LOG.info("[video] viz -> %s (%d) (lock 8s)", name, idx)
|
||||
return None
|
||||
# qsdfghjklm -> audio (scene SC)
|
||||
for kk, scene in KEYMAP_AUDIO:
|
||||
# m = matrix performance mode: stop the auto data-synth scene AND
|
||||
# force the openpos viz (the matrix is not a scene — it keeps
|
||||
# playing). User request live 2026-07-02.
|
||||
if k == "m":
|
||||
self._scClient.send_message("/control/doScene", ["stop"])
|
||||
names = list(self._state.viz_mode_names)
|
||||
idx = names.index("openpos") if "openpos" in names else 0
|
||||
with self._state.lock():
|
||||
self._state.viz_mode = idx
|
||||
self._state.active_scene = "stop"
|
||||
import time as _t
|
||||
self._user_viz_lock_t = _t.monotonic()
|
||||
LOG.info("[key] m -> scene stop + viz openpos (matrix mode)")
|
||||
return None
|
||||
# qsdfghjkl -> audio (scene SC): gated OFF by default like wxcvbn
|
||||
# (VIZ_AUDIO_KEYS=1 re-enables) — stray keys fire auto-synth scenes
|
||||
# mid-performance. The m matrix-mode shortcut above stays active.
|
||||
_audio_maps = KEYMAP_AUDIO if getattr(self, "_audio_keys", False) else ()
|
||||
for kk, scene in _audio_maps:
|
||||
if k == kk:
|
||||
self._scClient.send_message("/control/doScene", [scene])
|
||||
with self._state.lock():
|
||||
self._state.active_scene = scene
|
||||
LOG.info("[audio] scene -> %s", scene)
|
||||
return None
|
||||
# wxcvbn + 0-9 -> preset bundle (source + scene audio + viz video)
|
||||
for kk, source, scene, viz in (*KEYMAP_SOURCE, *KEYMAP_SOURCE_NUM):
|
||||
# wxcvbn + 0-9 -> preset bundles: ALL gated OFF by default
|
||||
# (VIZ_SOURCE_KEYS=1 re-enables) — stray keypresses mid-performance
|
||||
# switch the audio scene (user-disabled live 2026-07-02).
|
||||
_src_maps = ((*KEYMAP_SOURCE, *KEYMAP_SOURCE_NUM)
|
||||
if getattr(self, "_source_keys", False) else ())
|
||||
for kk, source, scene, viz in _src_maps:
|
||||
if key == kk or k == kk:
|
||||
# Audio : envoie a sclang
|
||||
self._scClient.send_message("/control/doScene", [scene])
|
||||
@@ -662,6 +725,8 @@ def main() -> int:
|
||||
choices=["n", "s", "l"],
|
||||
default="n",
|
||||
help="DETRPose model size (default: n)")
|
||||
p.add_argument("--iphone-usb", dest="iphone_usb", action="store_true",
|
||||
help="drive pose from the iPhone USB stream (no Mac camera)")
|
||||
p.add_argument("-v", "--verbose", action="store_true")
|
||||
opts = p.parse_args()
|
||||
|
||||
|
||||
+176
-72
@@ -19,7 +19,10 @@ import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .action_head_pub import ActionHeadPublisher
|
||||
from .arkit_joint_map import arkit_body_2d, arkit_body_3d
|
||||
from .euro_filter import SkeletonFilter
|
||||
from .pose_bridge import PoseSoundBridge
|
||||
from .pose_filter import PoseFilterChain
|
||||
@@ -29,6 +32,25 @@ from .tracker import IoUTracker
|
||||
|
||||
LOG = logging.getLogger("multi")
|
||||
|
||||
# Rotation cosmetique + detection de la frame video (env VIDEO_ROTATE).
|
||||
# Appliquee AVANT MediaPipe : la detection tourne sur l'image redressee et
|
||||
# l'overlay reste aligne. Les joints ARKit (monde 3D, gravity-aligned) sont
|
||||
# invariants a l'orientation du device, donc non concernes.
|
||||
_ROTATE_K = {"none": 0, "ccw": 1, "180": 2, "cw": 3}
|
||||
|
||||
|
||||
def _apply_video_rotate(frame, mode: str):
|
||||
"""Rotate a BGR frame by mode (none/ccw/cw/180) via numpy rot90.
|
||||
|
||||
Returns a C-contiguous array (MediaPipe / cv2 need contiguous input).
|
||||
Unknown modes are treated as 'none' (no rotation).
|
||||
"""
|
||||
k = _ROTATE_K.get(mode, 0)
|
||||
if k == 0:
|
||||
return frame
|
||||
return np.ascontiguousarray(np.rot90(frame, k))
|
||||
|
||||
|
||||
MODELS = {
|
||||
"pose": (
|
||||
"https://storage.googleapis.com/mediapipe-models/pose_landmarker/"
|
||||
@@ -78,9 +100,11 @@ class MultiWorker:
|
||||
target_fps: float = 18.0,
|
||||
num_persons: int = 4,
|
||||
min_conf: float = 0.4,
|
||||
iphone_usb: bool = False,
|
||||
) -> None:
|
||||
self.state = state
|
||||
self.camera_index = camera_index
|
||||
self.iphone_usb = iphone_usb
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self.num_persons = num_persons
|
||||
self.min_conf = min_conf
|
||||
@@ -108,12 +132,13 @@ class MultiWorker:
|
||||
self._pid_missing: dict[int, int] = {}
|
||||
self._pid_last_bbox: dict[int, tuple[float, float, float, float]] = {}
|
||||
# Discrimination thresholds — tunable via env.
|
||||
import os as _os
|
||||
self._ghost_min_visible = int(_os.environ.get("POSE_GHOST_MIN_VISIBLE", "10"))
|
||||
self._ghost_min_conf = float(_os.environ.get("POSE_GHOST_MIN_CONF", "0.5"))
|
||||
self._hand_min_visible = int(_os.environ.get("POSE_HAND_MIN_VISIBLE", "15"))
|
||||
self._face_min_visible = int(_os.environ.get("POSE_FACE_MIN_VISIBLE", "50"))
|
||||
self._nms_iou = float(_os.environ.get("POSE_NMS_IOU", "0.7"))
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
self._ghost_min_visible = _cfg.pose_ghost_min_visible
|
||||
self._ghost_min_conf = _cfg.pose_ghost_min_conf
|
||||
self._hand_min_visible = _cfg.pose_hand_min_visible
|
||||
self._face_min_visible = _cfg.pose_face_min_visible
|
||||
self._nms_iou = _cfg.pose_nms_iou
|
||||
# Counters exposed for debug.
|
||||
self._n_ghost_dropped = 0
|
||||
self._n_hand_dropped = 0
|
||||
@@ -281,61 +306,98 @@ class MultiWorker:
|
||||
self._stop.set()
|
||||
|
||||
def _run(self) -> None:
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
_rot = _cfg.video_rotate
|
||||
LOG.info("video rotate = %s (env VIDEO_ROTATE: none/ccw/cw/180)", _rot)
|
||||
|
||||
try:
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
from mediapipe.tasks.python import BaseOptions
|
||||
from mediapipe.tasks.python.vision import (
|
||||
PoseLandmarker, PoseLandmarkerOptions,
|
||||
FaceLandmarker, FaceLandmarkerOptions,
|
||||
HandLandmarker, HandLandmarkerOptions,
|
||||
RunningMode,
|
||||
)
|
||||
except ModuleNotFoundError as e:
|
||||
LOG.error("deps manquantes : %s — uv sync --extra pose", e)
|
||||
LOG.error("deps manquantes (cv2) : %s", e)
|
||||
return
|
||||
|
||||
try:
|
||||
pose_p = _ensure_model("pose")
|
||||
face_p = _ensure_model("face")
|
||||
hand_p = _ensure_model("hand")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("download models failed: %s", e)
|
||||
return
|
||||
# MediaPipe landmarkers: only loaded when Mac webcam is the source.
|
||||
# Under --iphone-usb, body+face come from ARKit; loading MP here
|
||||
# would waste RAM and slow concert boot.
|
||||
pose = face = hand = _deleg = _mp = None
|
||||
if not self.iphone_usb:
|
||||
try:
|
||||
import mediapipe as _mp
|
||||
from mediapipe.tasks.python import BaseOptions
|
||||
from mediapipe.tasks.python.vision import (
|
||||
PoseLandmarker, PoseLandmarkerOptions,
|
||||
FaceLandmarker, FaceLandmarkerOptions,
|
||||
HandLandmarker, HandLandmarkerOptions,
|
||||
RunningMode,
|
||||
)
|
||||
except ModuleNotFoundError as e:
|
||||
LOG.error("deps manquantes : %s — uv sync --extra pose", e)
|
||||
return
|
||||
|
||||
pose = PoseLandmarker.create_from_options(PoseLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(pose_p)),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_poses=self.num_persons,
|
||||
min_pose_detection_confidence=self.min_conf,
|
||||
min_pose_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
face = FaceLandmarker.create_from_options(FaceLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(face_p)),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_faces=self.num_persons,
|
||||
min_face_detection_confidence=self.min_conf,
|
||||
min_face_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
hand = HandLandmarker.create_from_options(HandLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(hand_p)),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_hands=self.num_persons * 2,
|
||||
min_hand_detection_confidence=self.min_conf,
|
||||
min_hand_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
LOG.info("3 landmarkers prets (num=%d)", self.num_persons)
|
||||
try:
|
||||
pose_p = _ensure_model("pose")
|
||||
face_p = _ensure_model("face")
|
||||
hand_p = _ensure_model("hand")
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("download models failed: %s", e)
|
||||
return
|
||||
|
||||
cap = cv2.VideoCapture(self.camera_index)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index %d indisponible (TCC ?)", self.camera_index)
|
||||
return
|
||||
LOG.info("camera ouverte (index %d)", self.camera_index)
|
||||
# GPU delegate (Metal sur macOS) : libere le CPU pour OSC, state,
|
||||
# mesh_rigger. Toggle via MEDIAPIPE_DELEGATE=cpu si plante.
|
||||
_deleg_name = _cfg.mediapipe_delegate
|
||||
_deleg = (BaseOptions.Delegate.GPU if _deleg_name == "gpu"
|
||||
else BaseOptions.Delegate.CPU)
|
||||
LOG.info("MediaPipe delegate = %s (env MEDIAPIPE_DELEGATE)",
|
||||
_deleg.name)
|
||||
pose = PoseLandmarker.create_from_options(PoseLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(pose_p),
|
||||
delegate=_deleg),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_poses=self.num_persons,
|
||||
min_pose_detection_confidence=self.min_conf,
|
||||
min_pose_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
face = FaceLandmarker.create_from_options(FaceLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(face_p),
|
||||
delegate=_deleg),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_faces=self.num_persons,
|
||||
min_face_detection_confidence=self.min_conf,
|
||||
min_face_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
hand = HandLandmarker.create_from_options(HandLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=str(hand_p),
|
||||
delegate=_deleg),
|
||||
running_mode=RunningMode.VIDEO,
|
||||
num_hands=self.num_persons * 2,
|
||||
min_hand_detection_confidence=self.min_conf,
|
||||
min_hand_presence_confidence=self.min_conf,
|
||||
min_tracking_confidence=self.min_conf,
|
||||
))
|
||||
LOG.info("3 landmarkers prets (num=%d, delegate=%s)",
|
||||
self.num_persons, _deleg.name)
|
||||
|
||||
if self.iphone_usb:
|
||||
from .iphone_usb_source import IphoneUSBSource # noqa: PLC0415
|
||||
# write_hands=True: the iPhone Vision hands ARE the hand source under
|
||||
# --iphone-usb (rendering + /pose/hands openness + pinch). The Mac
|
||||
# MediaPipe hand detector is skipped below.
|
||||
cap = IphoneUSBSource(self.state, write_hands=True)
|
||||
if not cap.start():
|
||||
LOG.error("iphone USB source unavailable (app running? phone unlocked?)")
|
||||
return
|
||||
LOG.info("iphone USB source")
|
||||
else:
|
||||
cap = cv2.VideoCapture(self.camera_index)
|
||||
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
|
||||
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
|
||||
if not cap.isOpened():
|
||||
LOG.error("camera index %d indisponible (TCC ?)", self.camera_index)
|
||||
return
|
||||
LOG.info("camera ouverte (index %d)", self.camera_index)
|
||||
|
||||
t0_ms = int(time.monotonic() * 1000)
|
||||
while not self._stop.is_set():
|
||||
@@ -344,18 +406,35 @@ class MultiWorker:
|
||||
if not ok or frame_bgr is None:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
# Redresse la frame (iPhone tourne physiquement) AVANT MediaPipe
|
||||
# et l'encodage JPEG : detection + overlay + affichage coherents.
|
||||
frame_bgr = _apply_video_rotate(frame_bgr, _rot)
|
||||
h, w = frame_bgr.shape[:2]
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
mp_img = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame_rgb)
|
||||
ts = int(time.monotonic() * 1000) - t0_ms
|
||||
try:
|
||||
pose_res = pose.detect_for_video(mp_img, ts)
|
||||
face_res = face.detect_for_video(mp_img, ts)
|
||||
hand_res = hand.detect_for_video(mp_img, ts)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("inference: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
if self.iphone_usb:
|
||||
# body+face come from ARKit (IphoneUSBSource); skip MP inference.
|
||||
pose_res = face_res = hand_res = None
|
||||
else:
|
||||
# MediaPipe GPU delegate on macOS uploads via CVPixelBuffer
|
||||
# which only accepts 4-channel formats. SRGB (3ch) crashes
|
||||
# in gpu_buffer_storage_cv_pixel_buffer.cc with
|
||||
# "unsupported ImageFrame format: 1". Use SRGBA when on GPU.
|
||||
if _deleg == BaseOptions.Delegate.GPU:
|
||||
frame_rgba = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGBA)
|
||||
mp_img = _mp.Image(image_format=_mp.ImageFormat.SRGBA,
|
||||
data=frame_rgba)
|
||||
else:
|
||||
frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
|
||||
mp_img = _mp.Image(image_format=_mp.ImageFormat.SRGB,
|
||||
data=frame_rgb)
|
||||
try:
|
||||
pose_res = pose.detect_for_video(mp_img, ts)
|
||||
face_res = face.detect_for_video(mp_img, ts)
|
||||
hand_res = hand.detect_for_video(mp_img, ts)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("inference: %s", e)
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# Encode webcam JPEG pour overlay
|
||||
ok2, jpg = cv2.imencode(".jpg", frame_bgr,
|
||||
@@ -366,7 +445,7 @@ class MultiWorker:
|
||||
# fournit aussi z, plus precis que rien). pose_world_landmarks
|
||||
# donnerait des metres mais on garde un repere coherent avec face/hands.
|
||||
bodies = []
|
||||
pose_list = pose_res.pose_landmarks or []
|
||||
pose_list = (pose_res.pose_landmarks if pose_res is not None else None) or []
|
||||
for landmarks_list in pose_list:
|
||||
kp_list = []
|
||||
for lm in landmarks_list[:33]:
|
||||
@@ -380,7 +459,7 @@ class MultiWorker:
|
||||
# Aligned 1:1 with pose_landmarks order. Empty fallback if
|
||||
# the MediaPipe build doesn't populate it.
|
||||
bodies3d: list[list[Kp3D]] = []
|
||||
world_list = getattr(pose_res, "pose_world_landmarks", None) or []
|
||||
world_list = (getattr(pose_res, "pose_world_landmarks", None) if pose_res is not None else None) or []
|
||||
for landmarks_list in world_list:
|
||||
kp3_list: list[Kp3D] = []
|
||||
for lm in landmarks_list[:33]:
|
||||
@@ -392,7 +471,7 @@ class MultiWorker:
|
||||
bodies3d.append(kp3_list)
|
||||
|
||||
faces = []
|
||||
for landmarks_list in (face_res.face_landmarks or []):
|
||||
for landmarks_list in ((face_res.face_landmarks if face_res is not None else None) or []):
|
||||
kp_list = []
|
||||
for lm in landmarks_list[:478]:
|
||||
z = float(lm.z) if lm.z is not None else 0.0
|
||||
@@ -400,8 +479,21 @@ class MultiWorker:
|
||||
x=float(lm.x), y=float(lm.y), z=z, c=1.0))
|
||||
faces.append(kp_list)
|
||||
|
||||
# iphone-usb: bodies + bodies3d come from ARKit skeleton (2D + 3D).
|
||||
# MP inference was skipped above; rebuild from state.persons_arkit_*.
|
||||
# faces stays [] — ARKit has no face landmarks at this layer.
|
||||
if self.iphone_usb:
|
||||
with self.state.lock():
|
||||
_a2d = dict(self.state.persons_arkit_2d)
|
||||
_a3d = dict(self.state.persons_arkit_joints)
|
||||
_pid = min(_a2d.keys()) if _a2d else None
|
||||
if _pid is not None:
|
||||
bodies = [arkit_body_2d(_a2d[_pid])]
|
||||
_arr3d = _a3d.get(_pid)
|
||||
bodies3d = [arkit_body_3d(_arr3d)] if _arr3d is not None else []
|
||||
|
||||
hands = []
|
||||
for landmarks_list in (hand_res.hand_landmarks or []):
|
||||
for landmarks_list in ((hand_res.hand_landmarks if hand_res is not None else None) or []):
|
||||
kp_list = []
|
||||
for lm in landmarks_list[:21]:
|
||||
z = float(lm.z) if lm.z is not None else 0.0
|
||||
@@ -444,20 +536,27 @@ class MultiWorker:
|
||||
if not hasattr(self, "_dbg_b3d_t") or t_now - self._dbg_b3d_t > 5.0:
|
||||
LOG.info("body3d: n=%d (pose_world_landmarks)", len(bodies3d))
|
||||
self._dbg_b3d_t = t_now
|
||||
# iphone-usb: action_head_pub emits /pose/hands from the iPhone hands
|
||||
# (in persons_hands); don't double-emit a (skipped) MP hand set here.
|
||||
self._sound_bridge.send(
|
||||
bodies, ids_body, t_now,
|
||||
persons_face=faces, persons_face_ids=ids_face,
|
||||
persons_hands=hands, persons_hands_ids=ids_hand,
|
||||
persons_hands=(None if self.iphone_usb else hands),
|
||||
persons_hands_ids=(None if self.iphone_usb else ids_hand),
|
||||
persons_body3d=bodies3d, persons_body3d_ids=ids_body3d)
|
||||
|
||||
with self.state.lock():
|
||||
self.state.persons_body = bodies
|
||||
self.state.persons_face = faces
|
||||
self.state.persons_hands = hands
|
||||
# iphone-usb: persons_hands is owned by IphoneUSBSource (iPhone
|
||||
# Vision hands); don't clobber it with the skipped Mac MP result.
|
||||
if not self.iphone_usb:
|
||||
self.state.persons_hands = hands
|
||||
self.state.persons_body_ids = ids_body
|
||||
self.state.persons_body3d = bodies3d
|
||||
self.state.persons_face_ids = ids_face
|
||||
self.state.persons_hands_ids = ids_hand
|
||||
if not self.iphone_usb:
|
||||
self.state.persons_hands_ids = ids_hand
|
||||
# Compat single-person (1ere personne)
|
||||
if bodies:
|
||||
self.state.body_present = True
|
||||
@@ -481,5 +580,10 @@ class MultiWorker:
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
cap.release()
|
||||
pose.close(); face.close(); hand.close()
|
||||
if pose is not None:
|
||||
pose.close()
|
||||
if face is not None:
|
||||
face.close()
|
||||
if hand is not None:
|
||||
hand.close()
|
||||
LOG.info("multi worker stopped")
|
||||
|
||||
@@ -83,9 +83,8 @@ class MultiHMRWorker:
|
||||
# backend: 'pytorch' (default) or 'coreml'. CoreML uses the
|
||||
# .mlpackage at COREML_MLPACKAGE, bypasses MPS torch, and runs
|
||||
# on ANE/GPU/CPU via CoreML.framework natively (3-4x faster).
|
||||
self.backend = (backend
|
||||
or os.environ.get("MULTIHMR_BACKEND", "pytorch")
|
||||
).strip().lower()
|
||||
from .config import VizConfig as _VizConfig
|
||||
self.backend = (backend or _VizConfig.from_env().multihmr_backend).strip().lower()
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._smooth_shape = [
|
||||
@@ -107,9 +106,16 @@ class MultiHMRWorker:
|
||||
|
||||
@staticmethod
|
||||
def is_available() -> bool:
|
||||
backend = os.environ.get("MULTIHMR_BACKEND", "pytorch").strip().lower()
|
||||
from .config import VizConfig as _VizConfig
|
||||
backend = _VizConfig.from_env().multihmr_backend
|
||||
if backend == "coreml":
|
||||
return COREML_MLPACKAGE.exists()
|
||||
if backend == "remote":
|
||||
try:
|
||||
from .multihmr_remote import MultiHMRRemoteBackend
|
||||
return MultiHMRRemoteBackend.is_available()
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
return CKPT.exists() and SMPLX_PATH.exists() and MULTIHMR_REPO.exists()
|
||||
|
||||
def start(self) -> None:
|
||||
@@ -199,7 +205,10 @@ class MultiHMRWorker:
|
||||
|
||||
def _run(self) -> None:
|
||||
if self.backend == "coreml":
|
||||
self._run_coreml()
|
||||
self._run_coreml(remote=False)
|
||||
return
|
||||
if self.backend == "remote":
|
||||
self._run_coreml(remote=True)
|
||||
return
|
||||
self._run_pytorch()
|
||||
|
||||
@@ -255,9 +264,10 @@ class MultiHMRWorker:
|
||||
# autocast MPS teste 2026-05-13 : plus lent (400ms vs 270ms
|
||||
# baseline) car overhead de cast dans le forward. Defaut OFF.
|
||||
# Opt-in via MULTIHMR_AUTOCAST=1.
|
||||
from .config import VizConfig as _VizConfig
|
||||
self._use_autocast = (
|
||||
device == "mps"
|
||||
and os.environ.get("MULTIHMR_AUTOCAST", "0") == "1")
|
||||
and _VizConfig.from_env().multihmr_autocast)
|
||||
if self._use_autocast:
|
||||
LOG.info("Multi-HMR PyTorch : MPS autocast (fp16) enabled")
|
||||
# torch.compile teste 2026-05-13 : plante en runtime avec
|
||||
@@ -548,20 +558,35 @@ class MultiHMRWorker:
|
||||
# ------------------------------------------------------------------
|
||||
# CoreML backend
|
||||
# ------------------------------------------------------------------
|
||||
def _run_coreml(self) -> None:
|
||||
def _run_coreml(self, remote: bool = False) -> None:
|
||||
"""CoreML inference path (ANE+GPU+CPU via Apple's framework).
|
||||
|
||||
Mirrors _run_pytorch but loads the .mlpackage via pyobjc + the
|
||||
CoreML.framework, bypassing torch/MPS entirely. ~3-4x faster
|
||||
on M5 (28.8ms median vs ~100ms with MPS)."""
|
||||
on M5 (28.8ms median vs ~100ms with MPS).
|
||||
|
||||
If ``remote=True``, the local CoreML backend is swapped for a
|
||||
TCP client (``MultiHMRRemoteBackend``) that talks to a server
|
||||
running the same mlpackage on a faster Mac (macm1, M1 Max).
|
||||
"""
|
||||
try:
|
||||
import cv2
|
||||
except ImportError as e:
|
||||
LOG.error("opencv-python missing: %s", e)
|
||||
return
|
||||
try:
|
||||
from .multihmr_coreml import MultiHMRCoreMLBackend
|
||||
backend = MultiHMRCoreMLBackend(COREML_MLPACKAGE)
|
||||
if remote:
|
||||
from .config import VizConfig as _VizConfig
|
||||
from .multihmr_remote import MultiHMRRemoteBackend
|
||||
_rc = _VizConfig.from_env()
|
||||
backend = MultiHMRRemoteBackend(
|
||||
host=_rc.multihmr_remote_host,
|
||||
port=_rc.multihmr_remote_port)
|
||||
LOG.info("Multi-HMR remote backend (%s:%d)",
|
||||
_rc.multihmr_remote_host, _rc.multihmr_remote_port)
|
||||
else:
|
||||
from .multihmr_coreml import MultiHMRCoreMLBackend
|
||||
backend = MultiHMRCoreMLBackend(COREML_MLPACKAGE)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.error("CoreML backend init failed: %s", e)
|
||||
return
|
||||
@@ -589,12 +614,14 @@ class MultiHMRWorker:
|
||||
if not cap.start():
|
||||
LOG.error("AVCapture start failed pour %s", info["name"])
|
||||
return
|
||||
LOG.info("camera ouverte %s (%s) [coreml backend]",
|
||||
info["name"], info["type"])
|
||||
LOG.info("camera ouverte %s (%s) [%s backend]",
|
||||
info["name"], info["type"],
|
||||
"remote" if remote else "coreml")
|
||||
|
||||
frame_count = 0
|
||||
persons_count = 0
|
||||
skipped_static = 0
|
||||
fresh_count = 0
|
||||
next_heartbeat = time.monotonic() + 5.0
|
||||
prev_thumb: np.ndarray | None = None
|
||||
|
||||
@@ -642,18 +669,34 @@ class MultiHMRWorker:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# Async remote backend may return None when no fresh result
|
||||
# is ready yet — reuse the previous frame's humans so the
|
||||
# visualiser keeps drawing instead of clearing.
|
||||
if humans is None:
|
||||
humans = getattr(self, "_last_humans", []) or []
|
||||
reused_humans = True
|
||||
else:
|
||||
self._last_humans = humans
|
||||
reused_humans = False
|
||||
fresh_count += 1
|
||||
|
||||
t_post_start = time.monotonic()
|
||||
t_now = time.monotonic()
|
||||
frame_count += 1
|
||||
persons_count += len(humans) if humans else 0
|
||||
if reused_humans:
|
||||
LOG.debug("hb[remote]: reusing %d cached humans "
|
||||
"(no fresh result)", len(humans))
|
||||
if t_now >= next_heartbeat:
|
||||
fps = frame_count / 5.0
|
||||
fresh_fps = fresh_count / 5.0
|
||||
avg = persons_count / max(1, frame_count)
|
||||
LOG.info(
|
||||
"hb[coreml]: %.1f fps, %.2f persons/frame, %d skipped",
|
||||
fps, avg, skipped_static)
|
||||
"hb[coreml]: %.1f fps (fresh=%.1f), %.2f persons/frame, "
|
||||
"%d skipped", fps, fresh_fps, avg, skipped_static)
|
||||
frame_count = 0
|
||||
persons_count = 0
|
||||
fresh_count = 0
|
||||
skipped_static = 0
|
||||
next_heartbeat = t_now + 5.0
|
||||
|
||||
@@ -663,6 +706,16 @@ class MultiHMRWorker:
|
||||
time.sleep(self.period)
|
||||
continue
|
||||
|
||||
# If async backend reused last humans, keep state untouched and
|
||||
# spin to the next frame without re-running dedup/tracker/
|
||||
# smoothing (saves ~3-5 ms CPU per loop iteration and avoids
|
||||
# walking the One-Euro filter forward on stale data).
|
||||
if reused_humans:
|
||||
dt = time.monotonic() - t_cap_start
|
||||
if dt < self.period:
|
||||
time.sleep(self.period - dt)
|
||||
continue
|
||||
|
||||
# Dedup intra-frame (same logic as pytorch path).
|
||||
cand: list[tuple[
|
||||
float, float, float, float, float,
|
||||
|
||||
@@ -39,12 +39,24 @@ DEFAULT_MLPACKAGE = (
|
||||
N_PERSONS_FIXED = 4
|
||||
N_VERTS = 10475
|
||||
|
||||
# CoreML output names from the exported .mlpackage.
|
||||
OUT_V3D = "var_2412" # (4, 10475, 3)
|
||||
OUT_TRANSL = "var_2415" # (4, 1, 3)
|
||||
OUT_SCORES = "var_2428" # (4,)
|
||||
OUT_BETAS = "var_2431" # (4, 10)
|
||||
OUT_EXPR = "var_2434" # (4, 10)
|
||||
# CoreML output names from the exported .mlpackage. The exported
|
||||
# `multihmr_full_672_s.mlpackage` (2026-05-14 re-convert) renumbered
|
||||
# the MIL vars; verified against the on-disk artifact's spec.
|
||||
OUT_V3D = "var_2420" # (4, 10475, 3)
|
||||
OUT_TRANSL = "var_2423" # (4, 1, 3)
|
||||
OUT_SCORES = "var_2436" # (4,)
|
||||
OUT_BETAS = "var_2439" # (4, 10)
|
||||
OUT_EXPR = "var_2442" # (4, 10)
|
||||
# var_2445 (4, 127, 3) = j3d joints — present but unused here.
|
||||
|
||||
# DINOv2 backbone was trained on ImageNet-normalized RGB; the public
|
||||
# `infer()` contract takes [0,1] CHW input and applies this here so
|
||||
# every caller stays normalization-agnostic. Feeding raw [0,1] to the
|
||||
# model collapses all detection scores to ~0.01 ("0 detections" bug).
|
||||
_IMG_NORM_MEAN = np.array([0.485, 0.456, 0.406],
|
||||
dtype=np.float32).reshape(1, 3, 1, 1)
|
||||
_IMG_NORM_STD = np.array([0.229, 0.224, 0.225],
|
||||
dtype=np.float32).reshape(1, 3, 1, 1)
|
||||
|
||||
# MLMultiArrayDataType raw values (from CoreML headers).
|
||||
ML_DTYPE_FLOAT32 = 65568
|
||||
@@ -172,7 +184,8 @@ class MultiHMRCoreMLBackend:
|
||||
# Standalone (no contention) FP32 = 139 ms = 7.2 fps. Default
|
||||
# stays CPU+GPU. Override with COREML_COMPUTE_UNITS env var
|
||||
# (`all`, `cpu_and_gpu`, `cpu_and_ne`, `cpu_only`) for A/B testing.
|
||||
cu_env = os.environ.get("COREML_COMPUTE_UNITS", "").strip().lower()
|
||||
from .config import VizConfig as _VizConfig
|
||||
cu_env = _VizConfig.from_env().coreml_compute_units
|
||||
cu_map = {"cpu_only": 0, "cpu_and_gpu": 1, "all": 2,
|
||||
"cpu_and_ne": 3}
|
||||
cu = cu_map.get(cu_env, 1)
|
||||
@@ -245,7 +258,8 @@ class MultiHMRCoreMLBackend:
|
||||
"""Run a forward pass and return list of humans dicts.
|
||||
|
||||
Args:
|
||||
image_chw_float32: (3, 672, 672) or (1, 3, 672, 672) in [0,1].
|
||||
image_chw_float32: (3, 672, 672) or (1, 3, 672, 672), RGB in
|
||||
[0,1]. ImageNet normalization is applied internally.
|
||||
K_33: (3, 3) or (1, 3, 3) camera intrinsics.
|
||||
det_thresh: scores threshold; CoreML forwards K=4 always.
|
||||
|
||||
@@ -264,6 +278,7 @@ class MultiHMRCoreMLBackend:
|
||||
if K.shape != (1, 3, 3):
|
||||
raise ValueError(f"K shape {K.shape}, expected (1,3,3)")
|
||||
|
||||
img = (img - _IMG_NORM_MEAN) / _IMG_NORM_STD
|
||||
raw = self._predict(img, K)
|
||||
v3d = raw.get(OUT_V3D)
|
||||
transl = raw.get(OUT_TRANSL)
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
"""Multi-HMR remote backend: drop-in replacement of MultiHMRCoreMLBackend
|
||||
that delegates inference to a remote TCP server (see
|
||||
``scripts/multihmr_server.py``).
|
||||
|
||||
Protocol (little-endian, persistent connection):
|
||||
|
||||
Request:
|
||||
[4B uint32 payload_len]
|
||||
[4B magic "REQ\x01"]
|
||||
[1B uint8 format_id] # 1 = raw RGB uint8 HWC, 2 = JPEG (variable length)
|
||||
[3B padding]
|
||||
[variable image bytes] # IMG_BYTES if format=1, else JPEG bytes
|
||||
[9 float32 K = 36 bytes]
|
||||
|
||||
The K block is *always* the last 36 bytes of the payload, regardless of
|
||||
``format_id`` — the server slices it off before treating the rest as the
|
||||
image.
|
||||
|
||||
Response:
|
||||
[4B uint32 payload_len]
|
||||
[4B magic "RSP\x01"]
|
||||
[4B int32 status]
|
||||
[v3d : 4*10475*3 f32]
|
||||
[transl: 4*1*3 f32]
|
||||
[scores: 4 f32]
|
||||
[betas: 4*10 f32]
|
||||
[expr : 4*10 f32]
|
||||
|
||||
Two extra features over the bare RPC:
|
||||
|
||||
* JPEG compression (``MULTIHMR_REMOTE_JPEG=1``, default ON, quality 80).
|
||||
Cuts wire bytes from ~1.35 MB to ~50-150 KB.
|
||||
|
||||
* Asynchronous double-buffer (``MULTIHMR_REMOTE_ASYNC=1``, default ON).
|
||||
``infer()`` is decoupled from the I/O round-trip via a dedicated worker
|
||||
thread and two ``Queue(maxsize=1)`` slots. When the out-queue is empty
|
||||
``infer()`` returns ``None`` — the worker loop reuses its last humans
|
||||
list so the visualiser keeps drawing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import socket
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
LOG = logging.getLogger("multihmr_remote")
|
||||
|
||||
IMG_SIZE = 672
|
||||
N_PERSONS_FIXED = 4
|
||||
N_VERTS = 10475
|
||||
|
||||
MAGIC_REQ = b"REQ\x01"
|
||||
MAGIC_RSP = b"RSP\x01"
|
||||
|
||||
FORMAT_RAW = 1
|
||||
FORMAT_JPEG = 2
|
||||
|
||||
IMG_BYTES = IMG_SIZE * IMG_SIZE * 3
|
||||
K_BYTES = 9 * 4
|
||||
REQ_HEADER = 4 + 1 + 3 # magic + format_id + 3-byte pad
|
||||
# Fixed RAW-format request payload (mirrors RSP_PAYLOAD_LEN). The JPEG
|
||||
# path is variable-length, so this is the upper-bound / RAW case only.
|
||||
REQ_PAYLOAD_LEN = REQ_HEADER + IMG_BYTES + K_BYTES
|
||||
|
||||
V3D_BYTES = N_PERSONS_FIXED * N_VERTS * 3 * 4
|
||||
TRANSL_BYTES = N_PERSONS_FIXED * 1 * 3 * 4
|
||||
SCORES_BYTES = N_PERSONS_FIXED * 4
|
||||
BETAS_BYTES = N_PERSONS_FIXED * 10 * 4
|
||||
EXPR_BYTES = N_PERSONS_FIXED * 10 * 4
|
||||
RSP_HEADER = 4 + 4
|
||||
RSP_PAYLOAD_LEN = (RSP_HEADER + V3D_BYTES + TRANSL_BYTES
|
||||
+ SCORES_BYTES + BETAS_BYTES + EXPR_BYTES)
|
||||
|
||||
|
||||
def _env_flag(name: str, default: bool) -> bool:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None:
|
||||
return default
|
||||
return raw.strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _recv_exact(sock: socket.socket, n: int) -> bytes:
|
||||
buf = bytearray(n)
|
||||
view = memoryview(buf)
|
||||
pos = 0
|
||||
while pos < n:
|
||||
got = sock.recv_into(view[pos:])
|
||||
if got == 0:
|
||||
raise ConnectionError("peer closed mid-stream")
|
||||
pos += got
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def encode_request_raw(image_uint8_hwc: np.ndarray,
|
||||
K_33: np.ndarray) -> bytes:
|
||||
"""Raw uint8 HWC request (format_id=1, fixed payload length)."""
|
||||
if image_uint8_hwc.shape != (IMG_SIZE, IMG_SIZE, 3):
|
||||
raise ValueError(
|
||||
f"image shape {image_uint8_hwc.shape} != "
|
||||
f"({IMG_SIZE},{IMG_SIZE},3)")
|
||||
if image_uint8_hwc.dtype != np.uint8:
|
||||
raise ValueError(f"image dtype {image_uint8_hwc.dtype} != uint8")
|
||||
K = np.ascontiguousarray(K_33, dtype="<f4").reshape(9)
|
||||
img = np.ascontiguousarray(image_uint8_hwc, dtype=np.uint8)
|
||||
img_bytes = img.tobytes()
|
||||
header_after_magic = bytes([FORMAT_RAW, 0, 0, 0])
|
||||
payload_len = REQ_HEADER + len(img_bytes) + K_BYTES
|
||||
return b"".join([
|
||||
struct.pack("<I", payload_len),
|
||||
MAGIC_REQ,
|
||||
header_after_magic,
|
||||
img_bytes,
|
||||
K.tobytes(),
|
||||
])
|
||||
|
||||
|
||||
def encode_request_jpeg(jpeg_bytes: bytes, K_33: np.ndarray) -> bytes:
|
||||
"""JPEG request (format_id=2, variable payload length)."""
|
||||
K = np.ascontiguousarray(K_33, dtype="<f4").reshape(9)
|
||||
header_after_magic = bytes([FORMAT_JPEG, 0, 0, 0])
|
||||
payload_len = REQ_HEADER + len(jpeg_bytes) + K_BYTES
|
||||
return b"".join([
|
||||
struct.pack("<I", payload_len),
|
||||
MAGIC_REQ,
|
||||
header_after_magic,
|
||||
jpeg_bytes,
|
||||
K.tobytes(),
|
||||
])
|
||||
|
||||
|
||||
def decode_response(payload: bytes) -> tuple[
|
||||
np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, int]:
|
||||
if len(payload) != RSP_PAYLOAD_LEN:
|
||||
raise ValueError(
|
||||
f"rsp payload {len(payload)} != {RSP_PAYLOAD_LEN}")
|
||||
if payload[:4] != MAGIC_RSP:
|
||||
raise ValueError(f"bad rsp magic {payload[:4]!r}")
|
||||
status = struct.unpack("<i", payload[4:8])[0]
|
||||
off = 8
|
||||
v3d = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED * N_VERTS * 3,
|
||||
offset=off).reshape(N_PERSONS_FIXED, N_VERTS, 3)
|
||||
off += V3D_BYTES
|
||||
transl = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED * 1 * 3,
|
||||
offset=off).reshape(N_PERSONS_FIXED, 1, 3)
|
||||
off += TRANSL_BYTES
|
||||
scores = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED,
|
||||
offset=off).reshape(N_PERSONS_FIXED)
|
||||
off += SCORES_BYTES
|
||||
betas = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED * 10,
|
||||
offset=off).reshape(N_PERSONS_FIXED, 10)
|
||||
off += BETAS_BYTES
|
||||
expr = np.frombuffer(payload, dtype="<f4",
|
||||
count=N_PERSONS_FIXED * 10,
|
||||
offset=off).reshape(N_PERSONS_FIXED, 10)
|
||||
return (v3d.copy(), transl.copy(), scores.copy(),
|
||||
betas.copy(), expr.copy(), int(status))
|
||||
|
||||
|
||||
# Back-compat shim — old call sites used encode_request(img, K) for raw.
|
||||
def encode_request(image_uint8_hwc: np.ndarray, K_33: np.ndarray) -> bytes:
|
||||
return encode_request_raw(image_uint8_hwc, K_33)
|
||||
|
||||
|
||||
class _Tensorlike:
|
||||
"""Mimics CoreMLArray to avoid a hard import on multihmr_coreml."""
|
||||
__slots__ = ("_arr",)
|
||||
|
||||
def __init__(self, arr: np.ndarray) -> None:
|
||||
self._arr = arr
|
||||
|
||||
def detach(self) -> "_Tensorlike":
|
||||
return self
|
||||
|
||||
def cpu(self) -> "_Tensorlike":
|
||||
return self
|
||||
|
||||
def numpy(self) -> np.ndarray:
|
||||
return self._arr
|
||||
|
||||
def item(self) -> float:
|
||||
return float(self._arr.reshape(-1)[0])
|
||||
|
||||
@property
|
||||
def shape(self) -> tuple[int, ...]:
|
||||
return tuple(self._arr.shape)
|
||||
|
||||
|
||||
def _humans_from_arrays(v3d: np.ndarray, transl: np.ndarray,
|
||||
scores: np.ndarray, betas: np.ndarray,
|
||||
expr: np.ndarray, det_thresh: float
|
||||
) -> list[dict[str, Any]]:
|
||||
humans: list[dict[str, Any]] = []
|
||||
for k in range(N_PERSONS_FIXED):
|
||||
sc = float(scores[k])
|
||||
if sc < det_thresh:
|
||||
continue
|
||||
humans.append({
|
||||
"v3d": _Tensorlike(v3d[k]),
|
||||
"transl_pelvis": _Tensorlike(transl[k]),
|
||||
"scores": _Tensorlike(np.array([sc], dtype=np.float32)),
|
||||
"shape": _Tensorlike(betas[k]),
|
||||
"expression": _Tensorlike(expr[k]),
|
||||
})
|
||||
return humans
|
||||
|
||||
|
||||
class MultiHMRRemoteBackend:
|
||||
"""TCP client backend mirroring ``MultiHMRCoreMLBackend.infer`` API.
|
||||
|
||||
JPEG compression and async double-buffering are toggleable via env
|
||||
(``MULTIHMR_REMOTE_JPEG``, ``MULTIHMR_REMOTE_ASYNC``).
|
||||
"""
|
||||
|
||||
def __init__(self, host: str = "192.168.0.175", port: int = 57140,
|
||||
connect_timeout: float = 3.0,
|
||||
io_timeout: float = 5.0) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.connect_timeout = connect_timeout
|
||||
self.io_timeout = io_timeout
|
||||
self._sock: socket.socket | None = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
self.use_jpeg = _cfg.multihmr_remote_jpeg
|
||||
self.jpeg_quality = _cfg.multihmr_remote_jpeg_quality
|
||||
self.use_async = _cfg.multihmr_remote_async
|
||||
|
||||
# Async pipeline state.
|
||||
# Multi-buffer queues (2 in / 3 out) absorb jitter without
|
||||
# stalling capture. Drop-oldest semantics on overflow.
|
||||
self._in_q: queue.Queue[tuple[bytes, float, float]] = queue.Queue(
|
||||
maxsize=2)
|
||||
self._out_q: queue.Queue[
|
||||
tuple[list[dict[str, Any]], dict[str, float]]
|
||||
] = queue.Queue(maxsize=3)
|
||||
self._stop = threading.Event()
|
||||
self._async_det_thresh = 0.3
|
||||
self._worker_thread: threading.Thread | None = None
|
||||
self._last_stats: dict[str, float] = {}
|
||||
|
||||
if self.use_jpeg:
|
||||
try:
|
||||
import cv2 # noqa: F401
|
||||
except ImportError:
|
||||
LOG.warning("cv2 unavailable client-side, disabling JPEG")
|
||||
self.use_jpeg = False
|
||||
|
||||
if self.use_async:
|
||||
self._start_worker()
|
||||
LOG.info(
|
||||
"MultiHMRRemoteBackend %s:%d (jpeg=%s q=%d, async=%s)",
|
||||
host, port, self.use_jpeg, self.jpeg_quality, self.use_async)
|
||||
|
||||
# -- connection management -------------------------------------------
|
||||
|
||||
def _connect(self) -> socket.socket:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(self.connect_timeout)
|
||||
sock.connect((self.host, self.port))
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
sock.settimeout(self.io_timeout)
|
||||
LOG.info("connected to %s:%d", self.host, self.port)
|
||||
return sock
|
||||
|
||||
def _ensure_sock(self) -> socket.socket:
|
||||
if self._sock is None:
|
||||
self._sock = self._connect()
|
||||
return self._sock
|
||||
|
||||
def _drop_sock(self) -> None:
|
||||
if self._sock is not None:
|
||||
try:
|
||||
self._sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
self._sock = None
|
||||
|
||||
@staticmethod
|
||||
def is_available(host: str | None = None, port: int | None = None
|
||||
) -> bool:
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
host = host or _cfg.multihmr_remote_host
|
||||
port = port or _cfg.multihmr_remote_port
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(1.0)
|
||||
s.connect((host, port))
|
||||
s.close()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
# -- request encoding ------------------------------------------------
|
||||
|
||||
def _encode_request_from_chw(
|
||||
self, image_chw_float32: np.ndarray, K_33: np.ndarray
|
||||
) -> tuple[bytes, float]:
|
||||
"""Return (request bytes, encode_ms)."""
|
||||
img = np.asarray(image_chw_float32, dtype=np.float32)
|
||||
if img.ndim == 4 and img.shape[0] == 1:
|
||||
img = img[0]
|
||||
if img.shape != (3, IMG_SIZE, IMG_SIZE):
|
||||
raise ValueError(
|
||||
f"image shape {img.shape} != (3,{IMG_SIZE},{IMG_SIZE})")
|
||||
img_hwc = np.clip(img.transpose(1, 2, 0) * 255.0, 0.0, 255.0
|
||||
).astype(np.uint8)
|
||||
K = np.asarray(K_33, dtype=np.float32)
|
||||
if K.ndim == 3 and K.shape[0] == 1:
|
||||
K = K[0]
|
||||
if K.shape != (3, 3):
|
||||
raise ValueError(f"K shape {K.shape} != (3,3)")
|
||||
|
||||
t0 = time.monotonic()
|
||||
if self.use_jpeg:
|
||||
import cv2 # local import to keep optional
|
||||
# cv2.imencode wants BGR for nicest JPEG perceptually but the
|
||||
# server decodes back to RGB ; encode RGB->BGR once for parity.
|
||||
bgr = cv2.cvtColor(img_hwc, cv2.COLOR_RGB2BGR)
|
||||
ok, enc = cv2.imencode(
|
||||
".jpg", bgr,
|
||||
[int(cv2.IMWRITE_JPEG_QUALITY), self.jpeg_quality])
|
||||
if not ok:
|
||||
raise RuntimeError("cv2.imencode failed")
|
||||
req = encode_request_jpeg(bytes(enc), K)
|
||||
else:
|
||||
req = encode_request_raw(img_hwc, K)
|
||||
enc_ms = (time.monotonic() - t0) * 1e3
|
||||
return req, enc_ms
|
||||
|
||||
# -- synchronous fallback -------------------------------------------
|
||||
|
||||
def _send_recv(self, req: bytes) -> bytes:
|
||||
attempts = 0
|
||||
last_err: Exception | None = None
|
||||
while attempts < 2:
|
||||
attempts += 1
|
||||
try:
|
||||
sock = self._ensure_sock()
|
||||
sock.sendall(req)
|
||||
len_buf = _recv_exact(sock, 4)
|
||||
payload_len = struct.unpack("<I", len_buf)[0]
|
||||
if payload_len != RSP_PAYLOAD_LEN:
|
||||
raise ValueError(
|
||||
f"unexpected rsp len {payload_len}")
|
||||
return _recv_exact(sock, payload_len)
|
||||
except (ConnectionError, BrokenPipeError, OSError,
|
||||
socket.timeout) as e:
|
||||
LOG.warning("rpc failed (try %d): %s", attempts, e)
|
||||
self._drop_sock()
|
||||
last_err = e
|
||||
raise RuntimeError(f"remote inference failed: {last_err}")
|
||||
|
||||
# -- async worker ---------------------------------------------------
|
||||
|
||||
def _start_worker(self) -> None:
|
||||
self._worker_thread = threading.Thread(
|
||||
target=self._async_loop, name="multihmr-remote",
|
||||
daemon=True)
|
||||
self._worker_thread.start()
|
||||
|
||||
def _async_loop(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
req, t_submit, det_thresh = self._in_q.get(timeout=0.5)
|
||||
except queue.Empty:
|
||||
continue
|
||||
t_send = time.monotonic()
|
||||
try:
|
||||
with self._lock:
|
||||
payload = self._send_recv(req)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("async send_recv failed: %s", e)
|
||||
continue
|
||||
t_recv = time.monotonic()
|
||||
try:
|
||||
v3d, transl, scores, betas, expr, status = decode_response(
|
||||
payload)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("decode_response failed: %s", e)
|
||||
continue
|
||||
if status != 0:
|
||||
humans: list[dict[str, Any]] = []
|
||||
else:
|
||||
humans = _humans_from_arrays(
|
||||
v3d, transl, scores, betas, expr, det_thresh)
|
||||
stats = {
|
||||
"queue_wait_ms": (t_send - t_submit) * 1e3,
|
||||
"rpc_ms": (t_recv - t_send) * 1e3,
|
||||
}
|
||||
# Drop any pending stale output before pushing.
|
||||
try:
|
||||
self._out_q.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
try:
|
||||
self._out_q.put_nowait((humans, stats))
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
# -- public API -----------------------------------------------------
|
||||
|
||||
def infer(
|
||||
self,
|
||||
image_chw_float32: np.ndarray,
|
||||
K_33: np.ndarray,
|
||||
det_thresh: float = 0.3,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""In sync mode returns the humans list (possibly empty).
|
||||
|
||||
In async mode, submits the new frame (non-blocking, drop-newest
|
||||
if previous frame still in flight) and returns whatever output
|
||||
is ready in the out-queue. Returns ``None`` if nothing is ready
|
||||
yet — caller must reuse its last humans list.
|
||||
"""
|
||||
req, _enc_ms = self._encode_request_from_chw(
|
||||
image_chw_float32, K_33)
|
||||
|
||||
if not self.use_async:
|
||||
with self._lock:
|
||||
payload = self._send_recv(req)
|
||||
v3d, transl, scores, betas, expr, status = decode_response(
|
||||
payload)
|
||||
if status != 0:
|
||||
return []
|
||||
return _humans_from_arrays(
|
||||
v3d, transl, scores, betas, expr, det_thresh)
|
||||
|
||||
# Async path.
|
||||
self._async_det_thresh = det_thresh
|
||||
# drop-newest semantics: keep the freshest pending frame
|
||||
try:
|
||||
self._in_q.get_nowait()
|
||||
except queue.Empty:
|
||||
pass
|
||||
try:
|
||||
self._in_q.put_nowait((req, time.monotonic(), det_thresh))
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
try:
|
||||
humans, stats = self._out_q.get_nowait()
|
||||
except queue.Empty:
|
||||
return None
|
||||
self._last_stats = stats
|
||||
return humans
|
||||
|
||||
def close(self) -> None:
|
||||
self._stop.set()
|
||||
with self._lock:
|
||||
self._drop_sock()
|
||||
@@ -10,6 +10,7 @@ Routes emises :
|
||||
/pose/head <pid> <x> <y> <c> position du nez (visage)
|
||||
/pose/sho_span <pid> <dx> ecart epaules (estime distance camera)
|
||||
/pose/limb_span <pid> <span> envergure brassse (poignet a poignet)
|
||||
/pose/mouth <pid> <open> mouth openness 0..1
|
||||
/face/count <n> nombre de visages detectes
|
||||
/face/kp <pid> <idx> <x> <y> <z> <c> 68-pt subset (dlib mapping)
|
||||
/hand/count <n_left> <n_right> nombre de mains gauche / droite
|
||||
@@ -85,10 +86,27 @@ class PoseSoundBridge:
|
||||
self._client = SimpleUDPClient(sclang_host, sclang_port)
|
||||
# Broadcast secondaire vers AV-Live-Body (Swift) pour overlay
|
||||
# skeleton dans la fenetre RealityKit. Silent si pas connecte.
|
||||
self._avbody = SimpleUDPClient("127.0.0.1", 57126)
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
self._avbody = SimpleUDPClient(_cfg.avbody_host, 57126)
|
||||
# Optional VDMX (VJ) OSC mirror. Off by default; when VDMX_OSC_HOST is
|
||||
# set, the VJ-useful pose signals (hands/kin/center/action/count) are
|
||||
# forwarded to VDMX's OSC input so the body can drive any VDMX param.
|
||||
self._vdmx = (
|
||||
SimpleUDPClient(_cfg.vdmx_osc_host, _cfg.vdmx_osc_port)
|
||||
if _cfg.vdmx_osc_host else None
|
||||
)
|
||||
self._period = 1.0 / max(1.0, throttle_hz)
|
||||
self._last_t = 0.0
|
||||
|
||||
def _vj(self, addr: str, args: list) -> None:
|
||||
"""Mirror a pose message to VDMX if configured (silent if not)."""
|
||||
if getattr(self, "_vdmx", None) is not None:
|
||||
try:
|
||||
self._vdmx.send_message(addr, args)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def send(self, persons_body: list, persons_body_ids: list, t_now: float,
|
||||
*,
|
||||
persons_face: Sequence[Sequence[Any]] | None = None,
|
||||
@@ -109,6 +127,7 @@ class PoseSoundBridge:
|
||||
self._client.send_message("/pose/count", [int(n)])
|
||||
try: self._avbody.send_message("/pose/count", [int(n)])
|
||||
except OSError: pass
|
||||
self._vj("/pose/count", [int(n)])
|
||||
except OSError:
|
||||
return # SC pas la, on continue silencieusement
|
||||
|
||||
@@ -139,6 +158,7 @@ class PoseSoundBridge:
|
||||
cli.send_message("/pose/center", [pid, float(cx), float(cy)])
|
||||
try: self._avbody.send_message("/pose/center", [pid, float(cx), float(cy)])
|
||||
except OSError: pass
|
||||
self._vj("/pose/center", [pid, float(cx), float(cy)])
|
||||
|
||||
# Nez (visage) — important pour piloter une voix
|
||||
if len(body) > NOSE and body[NOSE].c > 0.3:
|
||||
@@ -174,6 +194,17 @@ class PoseSoundBridge:
|
||||
try: self._avbody.send_message("/pose/limb_span", [pid, float(span)])
|
||||
except OSError: pass
|
||||
|
||||
if len(body) >= 29:
|
||||
def _xy(i):
|
||||
kp = body[i]
|
||||
return [float(getattr(kp, "x", 0.0)), float(getattr(kp, "y", 0.0))]
|
||||
skel = [pid]
|
||||
for idx in (0, 11, 12, 23, 24, 25, 26, 27, 28): # nose, sh L/R, hip L/R, knee L/R, ank L/R
|
||||
skel += _xy(idx)
|
||||
cli.send_message("/pose/skel", skel)
|
||||
try: self._avbody.send_message("/pose/skel", skel)
|
||||
except OSError: pass
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def send_face(self, persons_face: Sequence[Sequence[Any]],
|
||||
persons_face_ids: Sequence[int], t_now: float,
|
||||
@@ -305,6 +336,7 @@ class PoseSoundBridge:
|
||||
return
|
||||
p = [float(probs[0]), float(probs[1]), float(probs[2])]
|
||||
self._client.send_message("/pose/action", [int(pid), int(label_idx), *p])
|
||||
self._vj("/pose/action", [int(pid), int(label_idx), *p])
|
||||
|
||||
def send_kin(self, pid: int, kin,
|
||||
t_now: float, force: bool = False) -> None:
|
||||
@@ -314,10 +346,64 @@ class PoseSoundBridge:
|
||||
"""
|
||||
if not force and (t_now - self._last_t) < self._period:
|
||||
return
|
||||
self._client.send_message(
|
||||
"/pose/kin",
|
||||
[int(pid), float(kin[0]), float(kin[1]), float(kin[2])],
|
||||
)
|
||||
kin_args = [int(pid), float(kin[0]), float(kin[1]), float(kin[2])]
|
||||
self._client.send_message("/pose/kin", kin_args)
|
||||
self._vj("/pose/kin", kin_args)
|
||||
|
||||
def send_mouth(self, pid: int, mouth_open: float) -> None:
|
||||
"""Send mouth openness via /pose/mouth OSC route.
|
||||
|
||||
Sends: [pid (int), mouth_open (float)] — mouth_open is 0..1.
|
||||
"""
|
||||
args = [int(pid), float(mouth_open)]
|
||||
self._client.send_message("/pose/mouth", args)
|
||||
self._vj("/pose/mouth", args)
|
||||
|
||||
@staticmethod
|
||||
def _hand_slot(h):
|
||||
if not h:
|
||||
return [0.0, 0.0, 0.0, 0.0]
|
||||
return [float(h["cx"]), float(h["cy"]),
|
||||
float(h["openness"]), float(h["speed"])]
|
||||
|
||||
def send_hands(self, feats, t):
|
||||
"""Emit /pose/hands [0, lx,ly,lopen,lspeed, rx,ry,ropen,rspeed, dist]."""
|
||||
args = [0]
|
||||
args += self._hand_slot(feats.get("L"))
|
||||
args += self._hand_slot(feats.get("R"))
|
||||
args.append(float(feats.get("dist", 0.0)))
|
||||
try:
|
||||
self._client.send_message("/pose/hands", args)
|
||||
except OSError:
|
||||
pass
|
||||
self._vj("/pose/hands", args)
|
||||
|
||||
def send_finger(self, ev) -> None:
|
||||
"""Emit one air-piano strike event. Not throttled: the detector's
|
||||
refractory window already rate-limits these."""
|
||||
args = [
|
||||
0, # pid (single performer)
|
||||
int(ev.hand),
|
||||
int(ev.finger),
|
||||
float(ev.strike_speed),
|
||||
float(ev.z),
|
||||
float(ev.tipx),
|
||||
float(ev.tipy),
|
||||
]
|
||||
try:
|
||||
self._client.send_message("/pose/finger", args)
|
||||
except OSError:
|
||||
pass
|
||||
self._vj("/pose/finger", args)
|
||||
|
||||
def send_pinch(self, ev) -> None:
|
||||
"""Emit a thumb-to-finger pinch edge (state 1=engage, 0=release)."""
|
||||
args = [0, int(ev.hand), int(ev.finger), int(getattr(ev, "state", 1))]
|
||||
try:
|
||||
self._client.send_message("/pose/pinch", args)
|
||||
except OSError:
|
||||
pass
|
||||
self._vj("/pose/pinch", args)
|
||||
|
||||
def send_enter(self, pid: int) -> None:
|
||||
"""Send lifecycle event when person enters frame."""
|
||||
|
||||
@@ -457,7 +457,8 @@ class IKConstraints:
|
||||
# --------------------------- chain wrapper ------------------------------
|
||||
|
||||
def _parse_env_stages() -> tuple[str, ...]:
|
||||
raw = os.environ.get("POSE_FILTER")
|
||||
from .config import VizConfig
|
||||
raw = VizConfig.from_env().pose_filter
|
||||
if raw is None:
|
||||
return DEFAULT_STAGES
|
||||
raw = raw.strip().lower()
|
||||
@@ -709,7 +710,8 @@ class ArkitFuse:
|
||||
# (median + Kalman 2D + lookahead) — no IK, no spring.
|
||||
|
||||
def _parse_env_face_stages() -> tuple[str, ...]:
|
||||
raw = os.environ.get("POSE_FILTER_FACE")
|
||||
from .config import VizConfig
|
||||
raw = VizConfig.from_env().pose_filter_face
|
||||
if raw is None:
|
||||
return ("median", "kalman", "lookahead")
|
||||
raw = raw.strip().lower()
|
||||
@@ -720,7 +722,8 @@ def _parse_env_face_stages() -> tuple[str, ...]:
|
||||
|
||||
|
||||
def _parse_env_hand_stages() -> tuple[str, ...]:
|
||||
raw = os.environ.get("POSE_FILTER_HAND")
|
||||
from .config import VizConfig
|
||||
raw = VizConfig.from_env().pose_filter_hand
|
||||
if raw is None:
|
||||
return ("median", "kalman", "lookahead")
|
||||
raw = raw.strip().lower()
|
||||
|
||||
@@ -73,6 +73,10 @@ multihmr = [
|
||||
# via git submodule third_party/SMPLer-X (fork electron-rare).
|
||||
# mmcv-lite suffit pour Config (le repo vendorise sa propre mmpose).
|
||||
# Le detector mmdet est remplace par YOLO Ultralytics (extras pose).
|
||||
iphone-usb = [
|
||||
"av>=12.0",
|
||||
"opencv-python>=4.10",
|
||||
]
|
||||
smplerx = [
|
||||
"torch>=2.4",
|
||||
"torchvision>=0.19",
|
||||
|
||||
+270
-24
@@ -15,6 +15,7 @@ sous lock a chaque frame (60 fps).
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -42,14 +43,29 @@ from Metal import (
|
||||
# pyobjc detecte automatiquement l'implementation par signature.
|
||||
from MetalKit import MTKView # noqa: F401 (utilise par d'autres modules)
|
||||
|
||||
from .arkit_skeleton import arkit_segments, finger_joint_mask
|
||||
from .hand_display import (
|
||||
HandPersistenceGate,
|
||||
arkit_2d_fresh,
|
||||
gauge_segments,
|
||||
hand_plausible,
|
||||
hand_size,
|
||||
segment_ok,
|
||||
panel_frame,
|
||||
panel_segments,
|
||||
)
|
||||
from .hand_slots import route_hands
|
||||
from .mesh_topology import (
|
||||
BODY_TRIANGLES,
|
||||
FACE_TRIANGLES,
|
||||
HAND_TRIANGLES,
|
||||
build_face_triangles_dynamic,
|
||||
)
|
||||
from .state import State
|
||||
|
||||
# Draw the full ARKit 91-joint body (topology-driven) instead of the
|
||||
# MP33-reduced body. Set ARKIT_FULL_SKELETON=0 to fall back to MP33.
|
||||
ARKIT_FULL = os.environ.get("ARKIT_FULL_SKELETON", "1") != "0"
|
||||
|
||||
LOG = logging.getLogger("renderer")
|
||||
|
||||
# Triangle primitive constant (Metal MTLPrimitiveType.triangle = 3)
|
||||
@@ -119,8 +135,8 @@ MESH_MAX_VERTS = 10475 # SMPL-X is the larger family; SMPL (6890) fits inside
|
||||
# de 16, regle Metal). On y stocke (time, rms, kp_norm, netz_dev,
|
||||
# lightning_flash, flare, wind_norm, bz_norm, social_rate, pose_alive,
|
||||
# pose_count, width, height, viz_mode, _pad0, _pad1).
|
||||
UNIFORM_FLOATS = 20 # +4 floats : hand_l_x/y, hand_r_x/y
|
||||
UNIFORM_SIZE = UNIFORM_FLOATS * 4 # 80 octets, aligne 16
|
||||
UNIFORM_FLOATS = 24 # +4 floats : hand_l_x/y, hand_r_x/y + hand_height, hand_openness, hand_speed, hand_dist
|
||||
UNIFORM_SIZE = UNIFORM_FLOATS * 4 # 96 octets, aligne 16
|
||||
|
||||
|
||||
class MetalRenderer(NSObject):
|
||||
@@ -145,6 +161,17 @@ class MetalRenderer(NSObject):
|
||||
self._mesh_buf = self._device.newBufferWithLength_options_(
|
||||
MESH_MAX_TRIS * 3 * MESH_VERT_FLOATS * 4, MTLResourceStorageModeShared)
|
||||
self._mp_bones = _mediapipe_bones() # None si pas dispo
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
self._hand_conf_min = _cfg.hand_conf_min
|
||||
self._arkit_bone_max = _cfg.arkit_bone_max
|
||||
self._hand_gate = HandPersistenceGate(min_frames=_cfg.hand_persist_frames, grace=_cfg.hand_persist_grace)
|
||||
# Safety knob: invert Vision chirality interpretation.
|
||||
# Chirality validated correct live 2026-07-02; keep False unless a
|
||||
# future iPhone app build flips it.
|
||||
self._hand_swap_lr = _cfg.hand_swap_lr
|
||||
self._arkit_full = _cfg.arkit_full_skeleton
|
||||
self._skel_width = max(1, int(_cfg.skel_line_width))
|
||||
self._init_skel_cpu_buffer()
|
||||
self._init_mesh_cpu_buffer()
|
||||
self._build_pipelines()
|
||||
@@ -262,6 +289,14 @@ class MetalRenderer(NSObject):
|
||||
hly = 1 - (lh_wrist.y if lh_wrist else 0.5) * 2
|
||||
hrx = (rh_wrist.x if rh_wrist else 0.5) * 2 - 1
|
||||
hry = 1 - (rh_wrist.y if rh_wrist else 0.5) * 2
|
||||
# Expressive hand features from HandFeatureExtractor
|
||||
hf = getattr(s, "hand_feats", None) or {}
|
||||
hl, hr = hf.get("L"), hf.get("R")
|
||||
ys = [h["cy"] for h in (hl, hr) if h]
|
||||
hand_height = (1.0 - min(ys)) if ys else 0.0
|
||||
hand_open = max([h["openness"] for h in (hl, hr) if h] or [0.0])
|
||||
hand_speed = max([h["speed"] for h in (hl, hr) if h] or [0.0])
|
||||
hand_dist = float(hf.get("dist", 0.0))
|
||||
uniforms = struct.pack(
|
||||
f"{UNIFORM_FLOATS}f",
|
||||
s.elapsed(), # 1
|
||||
@@ -278,8 +313,10 @@ class MetalRenderer(NSObject):
|
||||
float(s.width), # 12
|
||||
float(s.height), # 13
|
||||
float(s.viz_mode), # 14
|
||||
hlx, hly, hrx, hry, # 15-18 (mains)
|
||||
0.0, 0.0, # 19-20 pad
|
||||
hlx, hly, hrx, hry, # 15-18 wrist xy (kept)
|
||||
hand_height, hand_open, # 19-20
|
||||
hand_speed, hand_dist, # 21-22
|
||||
0.0, 0.0, # 23-24 pad (16-byte align)
|
||||
)
|
||||
n_segs = self._update_skeleton(s)
|
||||
n_tris = self._update_mesh(s)
|
||||
@@ -304,11 +341,19 @@ class MetalRenderer(NSObject):
|
||||
|
||||
Priorise MediaPipe (body 33 + face 478 + 2 mains 21) si disponible
|
||||
et present ; sinon fallback COCO 17 keypoints YOLO."""
|
||||
if not s.pose_alive():
|
||||
return 0
|
||||
_arkit_now = time.perf_counter()
|
||||
use_arkit = (getattr(self, '_arkit_full', ARKIT_FULL) and bool(s.arkit_parents)
|
||||
and arkit_2d_fresh(s.persons_arkit_2d_t, _arkit_now))
|
||||
# NO early-return when nothing is detected: the panel frames and the
|
||||
# X/Y gauges are permanent screen furniture (user request: indicators
|
||||
# must stay visible with no hands). The overlay loops below iterate
|
||||
# empty collections at negligible cost.
|
||||
|
||||
buf = self._skel_cpu_buf
|
||||
segs = 0
|
||||
# Mirror overlays in X to match the (already mirrored) iPhone video
|
||||
# background. Display-only: gesture data is untouched.
|
||||
mirror = bool(getattr(s, "mirror_2d", False))
|
||||
|
||||
def push(A, B, conf, pid):
|
||||
"""Empile un segment (2 verts) dans le buffer CPU prealloque."""
|
||||
@@ -317,12 +362,93 @@ class MetalRenderer(NSObject):
|
||||
return False
|
||||
ax = A.x * 2.0 - 1.0; ay = 1.0 - A.y * 2.0
|
||||
bx = B.x * 2.0 - 1.0; by = 1.0 - B.y * 2.0
|
||||
if mirror:
|
||||
ax = -ax; bx = -bx
|
||||
i = segs * 10
|
||||
buf[i+0] = ax; buf[i+1] = ay; buf[i+2] = float(A.z); buf[i+3] = conf; buf[i+4] = float(pid)
|
||||
buf[i+5] = bx; buf[i+6] = by; buf[i+7] = float(B.z); buf[i+8] = conf; buf[i+9] = float(pid)
|
||||
segs += 1
|
||||
return True
|
||||
|
||||
def push_seg(ax, ay, bx, by, conf, pid):
|
||||
"""Like push() but takes raw normalized [0,1] coords."""
|
||||
nonlocal segs
|
||||
if segs >= SKEL_MAX_SEGS:
|
||||
return False
|
||||
cax = ax * 2.0 - 1.0; cay = 1.0 - ay * 2.0
|
||||
cbx = bx * 2.0 - 1.0; cby = 1.0 - by * 2.0
|
||||
if mirror:
|
||||
cax = -cax; cbx = -cbx
|
||||
i = segs * 10
|
||||
buf[i+0] = cax; buf[i+1] = cay; buf[i+2] = 0.0
|
||||
buf[i+3] = conf; buf[i+4] = float(pid)
|
||||
buf[i+5] = cbx; buf[i+6] = cby; buf[i+7] = 0.0
|
||||
buf[i+8] = conf; buf[i+9] = float(pid)
|
||||
segs += 1
|
||||
return True
|
||||
|
||||
def push_panel(ax, ay, bx, by, conf, pid):
|
||||
"""Like push_seg but NEVER applies mirror.
|
||||
|
||||
Panel coords are final screen coords [0,1] — they must not be
|
||||
flipped again even when the video background is mirrored.
|
||||
"""
|
||||
nonlocal segs
|
||||
if segs >= SKEL_MAX_SEGS:
|
||||
return False
|
||||
cax = ax * 2.0 - 1.0; cay = 1.0 - ay * 2.0
|
||||
cbx = bx * 2.0 - 1.0; cby = 1.0 - by * 2.0
|
||||
i = segs * 10
|
||||
buf[i+0] = cax; buf[i+1] = cay; buf[i+2] = 0.0
|
||||
buf[i+3] = conf; buf[i+4] = float(pid)
|
||||
buf[i+5] = cbx; buf[i+6] = cby; buf[i+7] = 0.0
|
||||
buf[i+8] = conf; buf[i+9] = float(pid)
|
||||
segs += 1
|
||||
return True
|
||||
|
||||
# Gate called once per frame on the full persons_hands list.
|
||||
# Used for both the overlay loop and the side panels below.
|
||||
# getattr: unit tests build the renderer via __new__ (no Metal init).
|
||||
_gate = getattr(self, "_hand_gate", None)
|
||||
_hand_draw_flags = (_gate.step(s.persons_hands) if _gate is not None
|
||||
else [True] * len(s.persons_hands))
|
||||
|
||||
if use_arkit:
|
||||
parents = s.arkit_parents
|
||||
# Hide ARKit's per-finger joints ("false hands"): the display
|
||||
# shows the filtered Vision hands instead. Mask cached until the
|
||||
# topology changes.
|
||||
fmask = getattr(self, "_finger_mask", None)
|
||||
if fmask is None or len(fmask) != len(s.arkit_joint_names):
|
||||
fmask = np.asarray(
|
||||
finger_joint_mask(s.arkit_joint_names), dtype=bool)
|
||||
self._finger_mask = fmask
|
||||
for pid, arr2d in s.persons_arkit_2d.items():
|
||||
ts = s.persons_arkit_2d_t.get(pid)
|
||||
if ts is None or (_arkit_now - ts) >= 1.0:
|
||||
continue # skip stale or unknown pid
|
||||
valid = s.persons_arkit_2d_valid.get(pid)
|
||||
if len(fmask):
|
||||
if valid is None:
|
||||
valid = ~fmask
|
||||
elif len(valid) == len(fmask):
|
||||
valid = valid & ~fmask
|
||||
# Bold strokes: Metal lines are 1px, so each bone is drawn
|
||||
# in concentric offset passes (SKEL_LINE_WIDTH approx px).
|
||||
_w = getattr(self, "_skel_width", 1)
|
||||
_o = 1.0 / max(1, s.height or 720)
|
||||
_offsets = [(0.0, 0.0)]
|
||||
for _r in range(1, _w):
|
||||
_d = _r * _o
|
||||
_offsets += [(_d, 0.0), (-_d, 0.0), (0.0, _d), (0.0, -_d)]
|
||||
for (ax, ay, bx, by) in arkit_segments(
|
||||
arr2d, valid, parents, max_len=self._arkit_bone_max
|
||||
):
|
||||
for _dx, _dy in _offsets:
|
||||
if not push_seg(ax + _dx, ay + _dy,
|
||||
bx + _dx, by + _dy, 1.0, pid):
|
||||
break
|
||||
|
||||
if self._mp_bones is not None and (
|
||||
s.persons_body or s.persons_face or s.persons_hands or
|
||||
s.body_present or s.face_present or s.hands_present
|
||||
@@ -333,7 +459,7 @@ class MetalRenderer(NSObject):
|
||||
ids_b = s.persons_body_ids or list(range(len(s.persons_body)))
|
||||
ids_f = s.persons_face_ids or list(range(len(s.persons_face)))
|
||||
ids_h = s.persons_hands_ids or list(range(len(s.persons_hands)))
|
||||
for i, body_kp in enumerate(s.persons_body):
|
||||
for i, body_kp in enumerate([] if use_arkit else s.persons_body):
|
||||
pid = ids_b[i] if i < len(ids_b) else i
|
||||
for a, b in body_bones:
|
||||
if a >= len(body_kp) or b >= len(body_kp): continue
|
||||
@@ -346,15 +472,35 @@ class MetalRenderer(NSObject):
|
||||
if a >= len(face_kp) or b >= len(face_kp): continue
|
||||
if not push(face_kp[a], face_kp[b], 1.0, pid): break
|
||||
if segs >= SKEL_MAX_SEGS: break
|
||||
# Bold hand strokes: same concentric-offset technique as the
|
||||
# ARKit body (SKEL_LINE_WIDTH), one step thinner so small hands
|
||||
# do not blob.
|
||||
_hw = max(1, getattr(self, "_skel_width", 1) - 1)
|
||||
_ho = 1.0 / max(1, s.height or 720)
|
||||
_hoffs = [(0.0, 0.0)]
|
||||
for _r in range(1, _hw):
|
||||
_hd = _r * _ho
|
||||
_hoffs += [(_hd, 0.0), (-_hd, 0.0), (0.0, _hd), (0.0, -_hd)]
|
||||
for i, hand_kp in enumerate(s.persons_hands):
|
||||
if not _hand_draw_flags[i]:
|
||||
continue
|
||||
pid = ids_h[i] if i < len(ids_h) else i
|
||||
if not hand_plausible(hand_kp, conf_min=self._hand_conf_min):
|
||||
continue
|
||||
size = hand_size(hand_kp)
|
||||
for a, b in lhand_bones:
|
||||
if a >= len(hand_kp) or b >= len(hand_kp): continue
|
||||
A = hand_kp[a]; B = hand_kp[b]
|
||||
if not segment_ok(A, B, size, conf_min=self._hand_conf_min): continue
|
||||
# Decalage palette mains (+5) pour les distinguer
|
||||
if not push(hand_kp[a], hand_kp[b], 1.0, pid + 5): break
|
||||
_c = min(A.c, B.c)
|
||||
for _dx, _dy in _hoffs:
|
||||
if not push_seg(A.x + _dx, A.y + _dy,
|
||||
B.x + _dx, B.y + _dy, _c, pid + 5):
|
||||
break
|
||||
# ----- FALLBACK single-person si persons_* vides -----
|
||||
if not (s.persons_body or s.persons_face or s.persons_hands):
|
||||
if s.body_present:
|
||||
if s.body_present and not use_arkit:
|
||||
for a, b in body_bones:
|
||||
if a >= len(s.body_kp) or b >= len(s.body_kp): continue
|
||||
A = s.body_kp[a]; B = s.body_kp[b]
|
||||
@@ -368,9 +514,14 @@ class MetalRenderer(NSObject):
|
||||
for kp_list in (s.left_hand_kp, s.right_hand_kp):
|
||||
if not any(p.x != 0.0 or p.y != 0.0 for p in kp_list):
|
||||
continue
|
||||
if not hand_plausible(kp_list, conf_min=self._hand_conf_min):
|
||||
continue
|
||||
size = hand_size(kp_list)
|
||||
for a, b in lhand_bones:
|
||||
if a >= len(kp_list) or b >= len(kp_list): continue
|
||||
if not push(kp_list[a], kp_list[b], 1.0, 0): break
|
||||
A = kp_list[a]; B = kp_list[b]
|
||||
if not segment_ok(A, B, size, conf_min=self._hand_conf_min): continue
|
||||
if not push(A, B, min(A.c, B.c), 0): break
|
||||
else:
|
||||
# Fallback COCO 17 (YOLO legacy)
|
||||
for a, b in COCO_BONES:
|
||||
@@ -378,6 +529,105 @@ class MetalRenderer(NSObject):
|
||||
if A.c < 0.2 or B.c < 0.2: continue
|
||||
if not push(A, B, min(A.c, B.c), 0): break
|
||||
|
||||
# ---- SIDE PANELS: left/right hand front-view zoomed wireframe ----
|
||||
# Gated on self._mp_bones (panels need _mp_bones for hand bone topology,
|
||||
# even when use_arkit is True). Hands from s.persons_hands only.
|
||||
if self._mp_bones is not None:
|
||||
_body_b, _face_b, lhand_bones_p, _ = self._mp_bones
|
||||
# Filter hands by persistence gate + plausibility before routing
|
||||
# to panels (a ghost hand must not steal a slot from a real one).
|
||||
keep = [
|
||||
ok and hand_plausible(h, conf_min=self._hand_conf_min)
|
||||
for h, ok in zip(s.persons_hands, _hand_draw_flags)
|
||||
]
|
||||
gated_hands = [h for h, k in zip(s.persons_hands, keep) if k]
|
||||
chir_src = getattr(s, "persons_hands_chirality", None) or []
|
||||
# Keep chirality aligned with the gated hands (same filter mask).
|
||||
if chir_src and len(chir_src) == len(s.persons_hands):
|
||||
gated_chir = [c for c, k in zip(chir_src, keep) if k]
|
||||
else:
|
||||
gated_chir = None
|
||||
asp = float(s.width) / float(s.height) if s.width and s.height else 1.0
|
||||
|
||||
# Route to L/R panels. mirror= passed for cx fallback path;
|
||||
# no near_min — panels show far hands as user proximity feedback.
|
||||
slotted = route_hands(
|
||||
gated_hands, gated_chir,
|
||||
mirror=mirror,
|
||||
swap=self._hand_swap_lr,
|
||||
)
|
||||
left_kp, right_kp = slotted
|
||||
|
||||
# Panel frame: status drives hue (pid 7/8/9), quality drives
|
||||
# brightness (0.25+0.75*q) and thickness (1..3 stroke passes).
|
||||
# pid 7 conf=0.25+0.75*q → status 0: absent (q=0)
|
||||
# → status 1: detected (q≥0.30)
|
||||
# pid 8 conf=0.25+0.75*q → status 2: armed (near+facing)
|
||||
# pid 9 conf=1.0 (q=1.0) → status 3: pinch engaged
|
||||
# Thickness: pass 1 always; pass 2 at q≥0.50; pass 3 at q≥0.85 or status==3.
|
||||
_slot_status = getattr(s, "gesture_slot_status", [0, 0])
|
||||
_slot_quality = getattr(s, "gesture_slot_quality", [0.0, 0.0])
|
||||
_hf_all = getattr(s, "hand_feats", None) or {}
|
||||
for _si, (side_name, h_kp, pid_s) in enumerate((
|
||||
("left", left_kp, 5),
|
||||
("right", right_kp, 6),
|
||||
)):
|
||||
_st = _slot_status[_si] if _si < len(_slot_status) else 0
|
||||
_q = float(_slot_quality[_si]) if _si < len(_slot_quality) else 0.0
|
||||
_fconf = 0.25 + 0.75 * _q
|
||||
# Status drives hue (pid selection)
|
||||
_fpid = 9 if _st == 3 else (8 if _st == 2 else 7)
|
||||
_frame_segs = panel_frame(side_name, asp)
|
||||
# Pass 1: always
|
||||
for seg in _frame_segs:
|
||||
push_panel(*seg, _fconf, _fpid)
|
||||
# Pass 2: thicker when quality ≥ 0.50
|
||||
if _q >= 0.5:
|
||||
for ax, ay, bx, by in _frame_segs:
|
||||
push_panel(ax + 0.001, ay, bx + 0.001, by, _fconf, _fpid)
|
||||
# Pass 3: boldest when quality ≥ 0.85 or pinch engaged
|
||||
if _q >= 0.85 or _st == 3:
|
||||
for ax, ay, bx, by in _frame_segs:
|
||||
push_panel(ax + 0.002, ay, bx + 0.002, by, _fconf, _fpid)
|
||||
# Position gauges (X below / Y outer-side). SC's mod cache
|
||||
# HOLDS the last received value when a hand vanishes, so the
|
||||
# markers must stay visible at that last value too (user:
|
||||
# "X et Y toujours visibles") — live = full conf, held = dim.
|
||||
_slot_key = "L" if side_name == "left" else "R"
|
||||
_slot_hf = _hf_all.get(_slot_key)
|
||||
if not hasattr(self, "_gauge_hold"):
|
||||
self._gauge_hold = {"L": (None, None), "R": (None, None)}
|
||||
_live = _slot_hf is not None
|
||||
if _live:
|
||||
self._gauge_hold[_slot_key] = (_slot_hf["cx"],
|
||||
_slot_hf["cy"])
|
||||
_g_cx, _g_cy = self._gauge_hold[_slot_key]
|
||||
_mk_conf = 1.0 if _live else 0.45
|
||||
for ax, ay, bx, by, gconf, gpid in gauge_segments(
|
||||
_g_cx, _g_cy, side_name, asp, mirror,
|
||||
content_pid=pid_s,
|
||||
):
|
||||
push_panel(ax, ay, bx, by,
|
||||
min(gconf, _mk_conf) if gpid == pid_s
|
||||
else gconf, gpid)
|
||||
# Wrist marker ON the video: the X/Y voice mods are anchored
|
||||
# to the wrist, so its control point stays visible whenever
|
||||
# a value exists (held at the last position when the hand
|
||||
# drops, matching what SC hears). push_seg applies the
|
||||
# video mirror like the overlay.
|
||||
if _g_cx is not None and _g_cy is not None:
|
||||
_mx = 0.015 / asp
|
||||
push_seg(_g_cx - _mx, _g_cy, _g_cx + _mx, _g_cy,
|
||||
_mk_conf, pid_s)
|
||||
push_seg(_g_cx, _g_cy - 0.015, _g_cx, _g_cy + 0.015,
|
||||
_mk_conf, pid_s)
|
||||
if h_kp is not None:
|
||||
for seg in panel_segments(
|
||||
h_kp, side_name, lhand_bones_p, asp,
|
||||
mirror=mirror, conf_min=self._hand_conf_min,
|
||||
):
|
||||
push_panel(*seg, 1.0, pid_s)
|
||||
|
||||
if segs == 0:
|
||||
return 0
|
||||
data = self._skel_cpu_buf[: segs * 2 * SKEL_VERT_FLOATS].tobytes()
|
||||
@@ -386,7 +636,7 @@ class MetalRenderer(NSObject):
|
||||
return segs
|
||||
|
||||
def _update_mesh(self, s: State) -> int:
|
||||
"""Remplit self._mesh_buf avec des triangles face/hand/body.
|
||||
"""Remplit self._mesh_buf avec des triangles face/body.
|
||||
|
||||
Retourne le nombre de triangles ecrits (chacun = 3 vertices).
|
||||
Filtre les triangles dont au moins un sommet a confiance < 0.3.
|
||||
@@ -436,10 +686,15 @@ class MetalRenderer(NSObject):
|
||||
|
||||
ids_b = s.persons_body_ids or list(range(len(s.persons_body)))
|
||||
ids_f = s.persons_face_ids or list(range(len(s.persons_face)))
|
||||
ids_h = s.persons_hands_ids or list(range(len(s.persons_hands)))
|
||||
|
||||
# Body
|
||||
for i, body_kp in enumerate(s.persons_body):
|
||||
# Body — SKIPPED whenever ARKit is the body source (iphone-usb: the
|
||||
# ARKit topology arrived). persons_body is then the ARKit->MP33
|
||||
# conversion with c forced to 1.0, so garbage/stale joints would pass
|
||||
# the confidence filter and paint solid triangles anywhere on screen
|
||||
# (the "blue blob"). The body is the gated ARKit wireframe instead.
|
||||
arkit_body = (getattr(self, "_arkit_full", ARKIT_FULL)
|
||||
and bool(s.arkit_parents))
|
||||
for i, body_kp in enumerate([] if arkit_body else s.persons_body):
|
||||
pid = ids_b[i] if i < len(ids_b) else i
|
||||
for a, b, c in BODY_TRIANGLES:
|
||||
if not push_tri(body_kp, a, b, c, pid):
|
||||
@@ -459,15 +714,6 @@ class MetalRenderer(NSObject):
|
||||
if n_verts >= MESH_MAX_VERTS:
|
||||
break
|
||||
|
||||
# Hands — decalage palette +5 comme dans le skel
|
||||
for i, hand_kp in enumerate(s.persons_hands):
|
||||
pid = (ids_h[i] if i < len(ids_h) else i) + 5
|
||||
for a, b, c in HAND_TRIANGLES:
|
||||
if not push_tri(hand_kp, a, b, c, pid):
|
||||
break
|
||||
if n_verts >= MESH_MAX_VERTS:
|
||||
break
|
||||
|
||||
if n_verts == 0:
|
||||
return 0
|
||||
# Slice is exact — stale floats beyond n_verts*MESH_VERT_FLOATS never reach the GPU.
|
||||
|
||||
@@ -14,7 +14,10 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
CACHE = Path.home() / ".cache" / "av-live-multihmr"
|
||||
CKPT = CACHE / "checkpoints" / "multiHMR_672_S.pt"
|
||||
_CKPT_NAME = os.environ.get("MULTIHMR_CKPT_NAME", "multiHMR_672_S.pt")
|
||||
CKPT = CACHE / "checkpoints" / _CKPT_NAME
|
||||
_OUT_NAME = os.environ.get("MULTIHMR_OUT_NAME",
|
||||
_CKPT_NAME.replace(".pt", ".mlpackage").lower())
|
||||
MULTIHMR_REPO = CACHE / "multi-hmr"
|
||||
|
||||
sys.path.insert(0, str(MULTIHMR_REPO))
|
||||
@@ -276,7 +279,8 @@ class TracedMHMR(nn.Module):
|
||||
zeros_p = torch.zeros(K_PERSONS, 3)
|
||||
zeros_s = torch.zeros(K_PERSONS)
|
||||
zeros_b = torch.zeros(K_PERSONS, 10)
|
||||
return zeros, zeros_p, zeros_s, zeros_b, zeros_b
|
||||
zeros_j = torch.zeros(K_PERSONS, 127, 3)
|
||||
return zeros, zeros_p, zeros_s, zeros_b, zeros_b, zeros_j
|
||||
v3d = torch.stack([h["v3d"] for h in humans])
|
||||
transl = torch.stack([h["transl_pelvis"] for h in humans])
|
||||
scores = torch.stack([
|
||||
@@ -285,13 +289,15 @@ class TracedMHMR(nn.Module):
|
||||
]).squeeze(-1)
|
||||
shape = torch.stack([h["shape"] for h in humans])
|
||||
expr = torch.stack([h["expression"] for h in humans])
|
||||
# NOTE: CoreML mlprogram conversion currently produces all-NaN
|
||||
# outputs for v3d and transl while PyTorch eager produces valid
|
||||
# finite values from the same trace. nan_to_num here masks the
|
||||
# symptom but yields all-zero meshes (no information). Leave
|
||||
# raw outputs and let downstream decide; investigation tracked
|
||||
# in task #2 (op-by-op bisection needed).
|
||||
return v3d, transl, scores, shape, expr
|
||||
# Joints (SMPL-X). smplx.create(use_pca=False) populates
|
||||
# output.joints of shape (B, 127, 3) which is then carried as
|
||||
# 'j3d' in smpl_layer.py:148 already in camera space (same
|
||||
# transl_up applied as v3d). The first 55 are the standard
|
||||
# SMPL-X joints (22 body + jaw + 2 eyes + 30 fingers); the
|
||||
# remaining 72 are face/landmark anchors. Downstream code can
|
||||
# slice [..., :55, :] if it only needs the skeleton.
|
||||
j3d = torch.stack([h["j3d"] for h in humans])
|
||||
return v3d, transl, scores, shape, expr, j3d
|
||||
|
||||
|
||||
wrapper = TracedMHMR(model).eval()
|
||||
@@ -306,10 +312,10 @@ example_x = torch.rand(1, 3, IMG_SIZE, IMG_SIZE)
|
||||
|
||||
print("==> Sanity forward")
|
||||
with torch.no_grad():
|
||||
v3d, transl, scores, shape, expr = wrapper(example_x, example_K)
|
||||
v3d, transl, scores, shape, expr, joints = wrapper(example_x, example_K)
|
||||
print(f" v3d: {tuple(v3d.shape)}, transl: {tuple(transl.shape)},")
|
||||
print(f" scores: {tuple(scores.shape)}, shape: {tuple(shape.shape)},")
|
||||
print(f" expr: {tuple(expr.shape)}")
|
||||
print(f" expr: {tuple(expr.shape)}, joints: {tuple(joints.shape)}")
|
||||
|
||||
print("==> torch.jit.trace")
|
||||
try:
|
||||
@@ -529,9 +535,19 @@ try:
|
||||
# teste 2026-05-14 : aucun gain sur GPU compute-bound.
|
||||
compute_precision=ct.precision.FLOAT32,
|
||||
)
|
||||
out_path = "/tmp/multihmr_full_672_s.mlpackage"
|
||||
out_path = f"/tmp/{_OUT_NAME}"
|
||||
mlmodel.save(out_path)
|
||||
print(f" CONVERT OK -> {out_path}")
|
||||
# Dump output names + shapes so we can wire OUT_* constants.
|
||||
try:
|
||||
spec = mlmodel.get_spec()
|
||||
print("==> mlpackage outputs:")
|
||||
for o in spec.description.output:
|
||||
mt = o.type.multiArrayType
|
||||
shape = list(mt.shape) if mt is not None else []
|
||||
print(f" {o.name} shape={shape}")
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" spec dump failed: {e}")
|
||||
except Exception as e:
|
||||
print(f" CONVERT FAILED: {type(e).__name__}: {e}")
|
||||
raise
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
"""iPhone ARBodyTracker USB skeleton -> OSC /body3d/kp bridge.
|
||||
|
||||
The ARBodyTracker iOS app serves its ARKit 91-joint skeleton over the
|
||||
device's TCP :7000 in the AVLiveWire framing (it does NOT publish OSC).
|
||||
AVLiveBody (Swift) consumes that over usbmuxd; data_only_viz instead
|
||||
listens for OSC /body3d/kp on UDP :57128 and fuses it via the `arkit_fuse`
|
||||
POSE_FILTER stage. This script is the missing link: it tunnels to the
|
||||
device's :7000 through usbmuxd, demuxes the AVLiveWire stream, decodes the
|
||||
skeleton frames, and republishes each valid joint as OSC.
|
||||
|
||||
Run it on the Mac the iPhone is plugged into, alongside:
|
||||
POSE_FILTER=median+kalman+lookahead+ik+arkit_fuse \
|
||||
python -m data_only_viz.main --pose
|
||||
|
||||
Wire formats (reverse-engineered from shared/AVLiveWire + avlivebody-mac):
|
||||
- usbmux packet: 16-byte LE header (length=16+body, version=1, message=8,
|
||||
tag) + XML plist. ListDevices -> DeviceList[].DeviceID; Connect with
|
||||
PortNumber byte-swapped to big-endian, Number==0 means success.
|
||||
- AVLiveWire frame header (19 B, big-endian): magic "AVL1", tag u8
|
||||
(skeleton=1), pid i16, timestamp f64, length u32.
|
||||
- SkeletonPayload (1183 B): 91 joints * (x,y,z) big-endian f32 interleaved,
|
||||
then 91 validity bytes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import plistlib
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
from pythonosc.udp_client import SimpleUDPClient
|
||||
|
||||
USBMUXD_PATH = "/var/run/usbmuxd"
|
||||
DEVICE_PORT = 7000
|
||||
OSC_HOST = "127.0.0.1"
|
||||
OSC_PORT = 57128
|
||||
|
||||
MAGIC = b"AVL1"
|
||||
HEADER_LEN = 19
|
||||
TAG_SKELETON = 1
|
||||
JOINT_COUNT = 91
|
||||
SKEL_FLOAT_BYTES = JOINT_COUNT * 3 * 4 # 1092
|
||||
SKEL_BYTES = SKEL_FLOAT_BYTES + JOINT_COUNT # 1183
|
||||
TAG_SKELETON2D = 6 # wire: skeleton2D (face=5 taken)
|
||||
SKEL2D_FLOAT_BYTES = JOINT_COUNT * 2 * 4 # 728
|
||||
SKEL2D_BYTES = SKEL2D_FLOAT_BYTES + JOINT_COUNT # 819
|
||||
MAX_PAYLOAD = 8 * 1024 * 1024
|
||||
|
||||
LOG = logging.getLogger("iphone_usb_bridge")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# usbmux client
|
||||
# --------------------------------------------------------------------------
|
||||
def _recv_exact(sock: socket.socket, n: int) -> bytes | None:
|
||||
buf = bytearray()
|
||||
while len(buf) < n:
|
||||
chunk = sock.recv(n - len(buf))
|
||||
if not chunk:
|
||||
return None
|
||||
buf += chunk
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def _usbmux_send(sock: socket.socket, plist: dict, tag: int) -> None:
|
||||
body = plistlib.dumps(plist, fmt=plistlib.FMT_XML)
|
||||
sock.sendall(struct.pack("<IIII", 16 + len(body), 1, 8, tag) + body)
|
||||
|
||||
|
||||
def _usbmux_recv(sock: socket.socket) -> dict | None:
|
||||
head = _recv_exact(sock, 4)
|
||||
if head is None:
|
||||
return None
|
||||
(length,) = struct.unpack("<I", head)
|
||||
if length < 16:
|
||||
return None
|
||||
rest = _recv_exact(sock, length - 4)
|
||||
if rest is None:
|
||||
return None
|
||||
packet = head + rest # full `length` bytes
|
||||
return plistlib.loads(packet[16:]) # plist body after 16-byte header
|
||||
|
||||
|
||||
def connect_device() -> socket.socket | None:
|
||||
"""usbmux ListDevices + Connect to the first device's :7000.
|
||||
|
||||
On success returns a socket whose stream is the tunneled device bytes.
|
||||
"""
|
||||
try:
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.connect(USBMUXD_PATH)
|
||||
except OSError as e:
|
||||
LOG.warning("no usbmuxd (%s)", e)
|
||||
return None
|
||||
try:
|
||||
_usbmux_send(sock, {"MessageType": "ListDevices"}, 1)
|
||||
reply = _usbmux_recv(sock)
|
||||
devices = [d.get("DeviceID") for d in (reply or {}).get("DeviceList", [])]
|
||||
devices = [d for d in devices if isinstance(d, int)]
|
||||
if not devices:
|
||||
LOG.warning("no iOS device attached")
|
||||
sock.close()
|
||||
return None
|
||||
device_id = devices[0]
|
||||
swapped = ((DEVICE_PORT << 8) | (DEVICE_PORT >> 8)) & 0xFFFF
|
||||
_usbmux_send(
|
||||
sock,
|
||||
{"MessageType": "Connect", "DeviceID": device_id,
|
||||
"PortNumber": swapped},
|
||||
2,
|
||||
)
|
||||
reply = _usbmux_recv(sock)
|
||||
if not reply or reply.get("Number") != 0:
|
||||
LOG.warning("Connect to :%d refused (%s)", DEVICE_PORT, reply)
|
||||
sock.close()
|
||||
return None
|
||||
LOG.info("connected to device %d port %d", device_id, DEVICE_PORT)
|
||||
return sock
|
||||
except OSError as e:
|
||||
LOG.warning("usbmux handshake failed: %s", e)
|
||||
sock.close()
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# AVLiveWire stream demux + skeleton decode
|
||||
# --------------------------------------------------------------------------
|
||||
def iter_frames(sock: socket.socket):
|
||||
"""Yield (tag, pid, payload) for each complete AVLiveWire frame."""
|
||||
buf = bytearray()
|
||||
while True:
|
||||
chunk = sock.recv(65536)
|
||||
if not chunk:
|
||||
return
|
||||
buf += chunk
|
||||
while True:
|
||||
idx = buf.find(MAGIC)
|
||||
if idx < 0:
|
||||
if len(buf) > 3: # keep a partial trailing magic
|
||||
del buf[:-3]
|
||||
break
|
||||
if idx > 0:
|
||||
del buf[:idx]
|
||||
if len(buf) < HEADER_LEN:
|
||||
break
|
||||
tag = buf[4]
|
||||
(length,) = struct.unpack(">I", buf[15:19])
|
||||
if length > MAX_PAYLOAD: # corrupt header, skip the magic
|
||||
del buf[:len(MAGIC)]
|
||||
continue
|
||||
total = HEADER_LEN + length
|
||||
if len(buf) < total:
|
||||
break
|
||||
(pid,) = struct.unpack(">h", buf[5:7])
|
||||
payload = bytes(buf[HEADER_LEN:total])
|
||||
del buf[:total]
|
||||
yield tag, pid, payload
|
||||
|
||||
|
||||
def decode_skeleton(payload: bytes):
|
||||
"""Return [(x, y, z, valid), ...] of length 91, or None if malformed."""
|
||||
if len(payload) != SKEL_BYTES:
|
||||
return None
|
||||
floats = struct.unpack(">" + "f" * (JOINT_COUNT * 3), payload[:SKEL_FLOAT_BYTES])
|
||||
valid = payload[SKEL_FLOAT_BYTES:]
|
||||
return [
|
||||
(floats[i * 3], floats[i * 3 + 1], floats[i * 3 + 2], valid[i] != 0)
|
||||
for i in range(JOINT_COUNT)
|
||||
]
|
||||
|
||||
|
||||
def decode_skeleton2D(payload: bytes):
|
||||
"""Return [(x, y, valid), ...] of length 91 (normalized 0..1), or None.
|
||||
|
||||
Wire layout: 91 × (x f32 BE, y f32 BE) = 728 bytes, then 91 validity bytes.
|
||||
Total: SKEL2D_BYTES = 819.
|
||||
"""
|
||||
if len(payload) != SKEL2D_BYTES:
|
||||
return None
|
||||
floats = struct.unpack(">" + "f" * (JOINT_COUNT * 2), payload[:SKEL2D_FLOAT_BYTES])
|
||||
valid = payload[SKEL2D_FLOAT_BYTES:]
|
||||
return [(floats[i * 2], floats[i * 2 + 1], valid[i] != 0) for i in range(JOINT_COUNT)]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# main loop
|
||||
# --------------------------------------------------------------------------
|
||||
def main() -> int:
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s — %(message)s",
|
||||
)
|
||||
osc = SimpleUDPClient(OSC_HOST, OSC_PORT)
|
||||
LOG.info("iphone_usb_bridge: usbmuxd :%d -> OSC %s:%d",
|
||||
DEVICE_PORT, OSC_HOST, OSC_PORT)
|
||||
frames = 0
|
||||
last_log = time.monotonic()
|
||||
while True:
|
||||
sock = connect_device()
|
||||
if sock is None:
|
||||
time.sleep(1.0)
|
||||
continue
|
||||
try:
|
||||
for tag, pid, payload in iter_frames(sock):
|
||||
if tag != TAG_SKELETON:
|
||||
continue
|
||||
joints = decode_skeleton(payload)
|
||||
if joints is None:
|
||||
continue
|
||||
n = 0
|
||||
for i, (x, y, z, v) in enumerate(joints):
|
||||
if v:
|
||||
osc.send_message("/body3d/kp", [pid, i, x, y, z])
|
||||
n += 1
|
||||
osc.send_message("/body3d/count", [1 if n else 0])
|
||||
frames += 1
|
||||
now = time.monotonic()
|
||||
if now - last_log > 3.0:
|
||||
LOG.info("forwarding skeleton frames (last: pid=%d, %d/%d valid joints)",
|
||||
pid, n, JOINT_COUNT)
|
||||
last_log = now
|
||||
except (OSError, struct.error) as e:
|
||||
LOG.warning("stream error: %s — reconnecting", e)
|
||||
finally:
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,607 @@
|
||||
"""Multi-HMR inference server (TCP, coremltools backend).
|
||||
|
||||
Runs on a remote Mac (macm1 in the AV-Live cluster), loads the
|
||||
mlpackage via coremltools (Python 3.12), and serves frames over TCP.
|
||||
|
||||
Protocol (little-endian, persistent connection):
|
||||
|
||||
Request:
|
||||
[4 bytes uint32 payload_len]
|
||||
[4 bytes magic "REQ\\x01"]
|
||||
[1 byte uint8 format_id] # 1 = raw RGB uint8 HWC, 2 = JPEG
|
||||
[3 bytes padding]
|
||||
[variable image bytes] # IMG_BYTES if format=1, else JPEG bytes
|
||||
[9 float32 LE = 36 bytes K] # always last 36 bytes
|
||||
|
||||
Response:
|
||||
[4 bytes uint32 payload_len]
|
||||
[4 bytes magic "RSP\\x01"]
|
||||
[4 bytes int32 status] # 0 = OK, 1 = error
|
||||
[v3d: 4*10475*3 float32]
|
||||
[transl: 4*1*3 float32]
|
||||
[scores: 4 float32]
|
||||
[betas: 4*10 float32]
|
||||
[expr: 4*10 float32]
|
||||
|
||||
Connection handler runs a 3-thread pipeline: reader -> worker -> writer.
|
||||
While the worker predicts frame N, the reader has already buffered frame
|
||||
N+1 so the next predict can start the instant the previous response is
|
||||
handed to the writer. Queue depth is 2 to absorb network jitter.
|
||||
|
||||
Bench mode (--bench): synthetic frames against the loaded backend.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
LOG = logging.getLogger("multihmr_server")
|
||||
|
||||
IMG_SIZE = 672
|
||||
N_PERSONS_FIXED = 4
|
||||
N_VERTS = 10475
|
||||
|
||||
MAGIC_REQ = b"REQ\x01"
|
||||
MAGIC_RSP = b"RSP\x01"
|
||||
|
||||
FORMAT_RAW = 1
|
||||
FORMAT_JPEG = 2
|
||||
|
||||
IMG_BYTES = IMG_SIZE * IMG_SIZE * 3 # 1_354_752
|
||||
K_BYTES = 9 * 4 # 36
|
||||
REQ_HEADER = 4 + 1 + 3 # magic + fmt u8 + 3 pad
|
||||
|
||||
V3D_BYTES = N_PERSONS_FIXED * N_VERTS * 3 * 4
|
||||
TRANSL_BYTES = N_PERSONS_FIXED * 1 * 3 * 4
|
||||
SCORES_BYTES = N_PERSONS_FIXED * 4
|
||||
BETAS_BYTES = N_PERSONS_FIXED * 10 * 4
|
||||
EXPR_BYTES = N_PERSONS_FIXED * 10 * 4
|
||||
RSP_HEADER = 4 + 4
|
||||
RSP_PAYLOAD_LEN = (RSP_HEADER + V3D_BYTES + TRANSL_BYTES
|
||||
+ SCORES_BYTES + BETAS_BYTES + EXPR_BYTES)
|
||||
|
||||
|
||||
DEFAULT_MLPACKAGE = Path(
|
||||
os.environ.get("MULTIHMR_MLPACKAGE")
|
||||
or str(Path.home() / ".cache" / "av-live-multihmr"
|
||||
/ "multihmr_full_672_s.mlpackage"))
|
||||
|
||||
OUT_V3D = "var_2420"
|
||||
OUT_TRANSL = "var_2423"
|
||||
OUT_SCORES = "var_2436"
|
||||
OUT_BETAS = "var_2439"
|
||||
OUT_EXPR = "var_2442"
|
||||
OUT_JOINTS = "var_2445" # (4, 127, 3) SMPL-X joints incl fingers
|
||||
N_JOINTS = 127
|
||||
|
||||
|
||||
def recv_exact(sock: socket.socket, n: int) -> bytes:
|
||||
buf = bytearray(n)
|
||||
view = memoryview(buf)
|
||||
pos = 0
|
||||
while pos < n:
|
||||
got = sock.recv_into(view[pos:])
|
||||
if got == 0:
|
||||
raise ConnectionError("peer closed")
|
||||
pos += got
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def encode_response(v3d: np.ndarray, transl: np.ndarray,
|
||||
scores: np.ndarray, betas: np.ndarray,
|
||||
expr: np.ndarray, status: int = 0) -> bytes:
|
||||
parts = [
|
||||
struct.pack("<I", RSP_PAYLOAD_LEN),
|
||||
MAGIC_RSP,
|
||||
struct.pack("<i", status),
|
||||
np.ascontiguousarray(v3d, dtype=np.float32).tobytes(),
|
||||
np.ascontiguousarray(transl, dtype=np.float32).tobytes(),
|
||||
np.ascontiguousarray(scores, dtype=np.float32).tobytes(),
|
||||
np.ascontiguousarray(betas, dtype=np.float32).tobytes(),
|
||||
np.ascontiguousarray(expr, dtype=np.float32).tobytes(),
|
||||
]
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def decode_request(payload: bytes) -> tuple[np.ndarray, np.ndarray, float]:
|
||||
"""Decode a request payload (without the leading 4-byte length).
|
||||
|
||||
Returns (image_uint8_hwc, K_33_f32, decode_ms_overhead).
|
||||
"""
|
||||
if len(payload) < REQ_HEADER + K_BYTES:
|
||||
raise ValueError(f"req payload too short: {len(payload)}")
|
||||
magic = payload[:4]
|
||||
if magic != MAGIC_REQ:
|
||||
raise ValueError(f"bad magic {magic!r}")
|
||||
fmt = payload[4]
|
||||
# payload[5:8] reserved.
|
||||
img_end = len(payload) - K_BYTES
|
||||
img_bytes = payload[REQ_HEADER:img_end]
|
||||
K = np.frombuffer(payload, dtype="<f4", count=9,
|
||||
offset=img_end).reshape(3, 3).astype(np.float32)
|
||||
t0 = time.monotonic()
|
||||
if fmt == FORMAT_RAW:
|
||||
if len(img_bytes) != IMG_BYTES:
|
||||
raise ValueError(
|
||||
f"raw img bytes {len(img_bytes)} != {IMG_BYTES}")
|
||||
img = np.frombuffer(img_bytes, dtype=np.uint8).reshape(
|
||||
IMG_SIZE, IMG_SIZE, 3)
|
||||
elif fmt == FORMAT_JPEG:
|
||||
import cv2
|
||||
arr = np.frombuffer(img_bytes, dtype=np.uint8)
|
||||
bgr = cv2.imdecode(arr, cv2.IMREAD_COLOR)
|
||||
if bgr is None:
|
||||
raise ValueError("cv2.imdecode failed")
|
||||
if bgr.shape[:2] != (IMG_SIZE, IMG_SIZE):
|
||||
bgr = cv2.resize(bgr, (IMG_SIZE, IMG_SIZE))
|
||||
img = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
|
||||
else:
|
||||
raise ValueError(f"unknown format_id {fmt}")
|
||||
decode_ms = (time.monotonic() - t0) * 1e3
|
||||
return img, K, decode_ms
|
||||
|
||||
|
||||
ML_DTYPE_FLOAT16 = 65552
|
||||
ML_DTYPE_FLOAT32 = 65568
|
||||
ML_DTYPE_DOUBLE = 65600
|
||||
|
||||
|
||||
def _np_to_mlarray(arr: np.ndarray, MLMultiArray):
|
||||
"""Create a contiguous float32 MLMultiArray from a numpy array."""
|
||||
import ctypes
|
||||
arr = np.ascontiguousarray(arr, dtype=np.float32)
|
||||
shape = [int(s) for s in arr.shape]
|
||||
ml = MLMultiArray.alloc().initWithShape_dataType_error_(
|
||||
shape, ML_DTYPE_FLOAT32, None)
|
||||
if ml is None:
|
||||
raise RuntimeError("MLMultiArray alloc failed")
|
||||
ptr = ml.dataPointer()
|
||||
addr = int(ptr) if isinstance(ptr, int) else ctypes.cast(
|
||||
ptr, ctypes.c_void_p).value
|
||||
if addr is None:
|
||||
raise RuntimeError("MLMultiArray dataPointer null")
|
||||
ctypes.memmove(addr, arr.ctypes.data, arr.nbytes)
|
||||
return ml
|
||||
|
||||
|
||||
def _mlarray_to_np(ml) -> np.ndarray:
|
||||
"""Copy an MLMultiArray (FLOAT16/32/64) to numpy float32."""
|
||||
import ctypes
|
||||
shape = tuple(int(s) for s in ml.shape())
|
||||
dtype_id = int(ml.dataType())
|
||||
count = 1
|
||||
for s in shape:
|
||||
count *= s
|
||||
ptr = ml.dataPointer()
|
||||
addr = int(ptr) if isinstance(ptr, int) else ctypes.cast(
|
||||
ptr, ctypes.c_void_p).value
|
||||
if addr is None:
|
||||
raise RuntimeError("MLMultiArray dataPointer null")
|
||||
if dtype_id == ML_DTYPE_FLOAT16:
|
||||
raw = (ctypes.c_uint16 * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).view(np.float16).astype(np.float32)
|
||||
elif dtype_id == ML_DTYPE_FLOAT32:
|
||||
raw = (ctypes.c_float * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).copy()
|
||||
elif dtype_id == ML_DTYPE_DOUBLE:
|
||||
raw = (ctypes.c_double * count).from_address(addr)
|
||||
arr = np.ctypeslib.as_array(raw).astype(np.float32)
|
||||
else:
|
||||
raise RuntimeError(f"unsupported MLMultiArray dtype {dtype_id}")
|
||||
return arr.reshape(shape)
|
||||
|
||||
|
||||
class CoreMLModel:
|
||||
"""pyobjc-direct CoreML wrapper. Drops the ~30 ms coremltools.MLModel.predict
|
||||
overhead by using CoreML.framework directly (MLDictionaryFeatureProvider
|
||||
+ MLMultiArray ctypes memcpy). Fallback to coremltools if pyobjc missing,
|
||||
via MULTIHMR_SERVER_BACKEND=coremltools env."""
|
||||
|
||||
def __init__(self, mlpackage_path: Path) -> None:
|
||||
self.path = Path(mlpackage_path)
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"mlpackage missing: {self.path}")
|
||||
backend = os.environ.get(
|
||||
"MULTIHMR_SERVER_BACKEND", "pyobjc").strip().lower()
|
||||
cu_env = os.environ.get(
|
||||
"COREML_COMPUTE_UNITS", "cpu_and_gpu").strip().lower()
|
||||
if backend == "pyobjc":
|
||||
self._use_pyobjc = True
|
||||
self._init_pyobjc(cu_env)
|
||||
else:
|
||||
self._use_pyobjc = False
|
||||
self._init_coremltools(cu_env)
|
||||
|
||||
def _init_pyobjc(self, cu_env: str) -> None:
|
||||
import objc
|
||||
from Foundation import NSURL
|
||||
ns: dict = {}
|
||||
objc.loadBundle("CoreML", ns,
|
||||
"/System/Library/Frameworks/CoreML.framework")
|
||||
cu_map = {"cpu_only": 0, "cpu_and_gpu": 1, "all": 2,
|
||||
"cpu_and_ne": 3}
|
||||
cu = cu_map.get(cu_env, 1)
|
||||
MLModel = ns["MLModel"]
|
||||
MLModelConfiguration = ns["MLModelConfiguration"]
|
||||
cfg = MLModelConfiguration.alloc().init()
|
||||
try:
|
||||
cfg.setComputeUnits_(cu)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
url = NSURL.fileURLWithPath_(str(self.path))
|
||||
compiled_url = MLModel.compileModelAtURL_error_(url, None)
|
||||
if compiled_url is None:
|
||||
raise RuntimeError(f"compileModelAtURL failed for {self.path}")
|
||||
model = MLModel.modelWithContentsOfURL_configuration_error_(
|
||||
compiled_url, cfg, None)
|
||||
if model is None:
|
||||
raise RuntimeError(f"MLModel load failed for {compiled_url}")
|
||||
self._model = model
|
||||
self._ns = ns
|
||||
LOG.info("loading mlpackage %s via pyobjc (computeUnit=%s)",
|
||||
self.path.name, cu_env)
|
||||
|
||||
def _init_coremltools(self, cu_env: str) -> None:
|
||||
import coremltools as ct
|
||||
from coremltools.models import MLModel as CTMLModel
|
||||
cu_map = {
|
||||
"cpu_only": ct.ComputeUnit.CPU_ONLY,
|
||||
"cpu_and_gpu": ct.ComputeUnit.CPU_AND_GPU,
|
||||
"all": ct.ComputeUnit.ALL,
|
||||
"cpu_and_ne": ct.ComputeUnit.CPU_AND_NE,
|
||||
}
|
||||
cu = cu_map.get(cu_env, ct.ComputeUnit.CPU_AND_GPU)
|
||||
LOG.info("loading mlpackage %s via coremltools (computeUnit=%s)",
|
||||
self.path.name, cu_env)
|
||||
self.model = CTMLModel(str(self.path), compute_units=cu)
|
||||
|
||||
def predict(self, image_uint8_hwc: np.ndarray, K_33: np.ndarray
|
||||
) -> dict[str, np.ndarray]:
|
||||
img_chw = image_uint8_hwc.transpose(2, 0, 1).astype(np.float32) / 255.0
|
||||
img4 = img_chw[np.newaxis, ...]
|
||||
K = K_33.astype(np.float32)
|
||||
if K.ndim == 2:
|
||||
K = K[np.newaxis, ...]
|
||||
if self._use_pyobjc:
|
||||
return self._predict_pyobjc(img4, K)
|
||||
return self.model.predict({"image": img4, "cam_K": K})
|
||||
|
||||
def _predict_pyobjc(self, image_4d: np.ndarray, K_33: np.ndarray
|
||||
) -> dict[str, np.ndarray]:
|
||||
ns = self._ns
|
||||
MLMultiArray = ns["MLMultiArray"]
|
||||
MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"]
|
||||
MLFeatureValue = ns["MLFeatureValue"]
|
||||
img_ml = _np_to_mlarray(image_4d, MLMultiArray)
|
||||
k_ml = _np_to_mlarray(K_33, MLMultiArray)
|
||||
feats = {
|
||||
"image": MLFeatureValue.featureValueWithMultiArray_(img_ml),
|
||||
"cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml),
|
||||
}
|
||||
provider = MLDictionaryFeatureProvider.alloc(
|
||||
).initWithDictionary_error_(feats, None)
|
||||
if provider is None:
|
||||
raise RuntimeError("MLDictionaryFeatureProvider alloc failed")
|
||||
out = self._model.predictionFromFeatures_error_(provider, None)
|
||||
if out is None:
|
||||
raise RuntimeError("MLModel predict failed")
|
||||
names = [str(n) for n in out.featureNames()]
|
||||
result: dict[str, np.ndarray] = {}
|
||||
for n in names:
|
||||
fv = out.featureValueForName_(n)
|
||||
if fv is None:
|
||||
continue
|
||||
ml = fv.multiArrayValue()
|
||||
if ml is None:
|
||||
continue
|
||||
result[n] = _mlarray_to_np(ml)
|
||||
return result
|
||||
|
||||
|
||||
def _zero_outputs() -> tuple[np.ndarray, ...]:
|
||||
return (
|
||||
np.zeros((N_PERSONS_FIXED, N_VERTS, 3), dtype=np.float32),
|
||||
np.zeros((N_PERSONS_FIXED, 1, 3), dtype=np.float32),
|
||||
np.zeros((N_PERSONS_FIXED,), dtype=np.float32),
|
||||
np.zeros((N_PERSONS_FIXED, 10), dtype=np.float32),
|
||||
np.zeros((N_PERSONS_FIXED, 10), dtype=np.float32),
|
||||
)
|
||||
|
||||
|
||||
def _extract_outputs(raw: dict[str, np.ndarray]
|
||||
) -> tuple[np.ndarray, ...]:
|
||||
v3d = np.asarray(raw[OUT_V3D], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED, N_VERTS, 3)
|
||||
transl = np.asarray(raw[OUT_TRANSL], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED, 1, 3)
|
||||
scores = np.asarray(raw[OUT_SCORES], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED)
|
||||
betas = np.asarray(raw[OUT_BETAS], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED, 10)
|
||||
expr = np.asarray(raw[OUT_EXPR], dtype=np.float32).reshape(
|
||||
N_PERSONS_FIXED, 10)
|
||||
return v3d, transl, scores, betas, expr
|
||||
|
||||
|
||||
class Server:
|
||||
def __init__(self, model: CoreMLModel, host: str, port: int) -> None:
|
||||
self.model = model
|
||||
self.host = host
|
||||
self.port = port
|
||||
self._stop = threading.Event()
|
||||
self._sock: socket.socket | None = None
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
if self._sock is not None:
|
||||
try:
|
||||
self._sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def serve(self) -> None:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
sock.bind((self.host, self.port))
|
||||
sock.listen(4)
|
||||
sock.settimeout(1.0)
|
||||
self._sock = sock
|
||||
LOG.info("listening %s:%d", self.host, self.port)
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
conn, addr = sock.accept()
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
break
|
||||
LOG.info("client connected %s", addr)
|
||||
try:
|
||||
self._handle_pipelined(conn)
|
||||
except (ConnectionError, BrokenPipeError, OSError) as e:
|
||||
LOG.info("client disconnected: %s", e)
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
LOG.info("server stopped")
|
||||
|
||||
# -- pipelined per-connection handler -----------------------------
|
||||
|
||||
def _handle_pipelined(self, conn: socket.socket) -> None:
|
||||
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
conn_stop = threading.Event()
|
||||
|
||||
# raw requests in, encoded responses out.
|
||||
req_q: queue.Queue[bytes] = queue.Queue(maxsize=2)
|
||||
rsp_q: queue.Queue[bytes] = queue.Queue(maxsize=2)
|
||||
|
||||
# stats
|
||||
served = {"n": 0, "t0": time.monotonic(),
|
||||
"sum_decode": 0.0, "sum_pred": 0.0,
|
||||
"sum_encode": 0.0}
|
||||
|
||||
def reader() -> None:
|
||||
try:
|
||||
while not conn_stop.is_set() and not self._stop.is_set():
|
||||
len_buf = recv_exact(conn, 4)
|
||||
payload_len = struct.unpack("<I", len_buf)[0]
|
||||
if payload_len > 8 * 1024 * 1024:
|
||||
raise ValueError(f"reqlen too big {payload_len}")
|
||||
payload = recv_exact(conn, payload_len)
|
||||
req_q.put(payload)
|
||||
except (ConnectionError, BrokenPipeError, OSError) as e:
|
||||
LOG.info("reader exit: %s", e)
|
||||
finally:
|
||||
conn_stop.set()
|
||||
# poison-pill the worker
|
||||
try:
|
||||
req_q.put_nowait(b"")
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
def worker() -> None:
|
||||
try:
|
||||
while not conn_stop.is_set() and not self._stop.is_set():
|
||||
try:
|
||||
payload = req_q.get(timeout=0.5)
|
||||
except queue.Empty:
|
||||
continue
|
||||
if payload == b"":
|
||||
break
|
||||
try:
|
||||
img, K, decode_ms = decode_request(payload)
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("decode failed: %s", e)
|
||||
v3d, transl, scores, betas, expr = _zero_outputs()
|
||||
rsp_q.put(encode_response(
|
||||
v3d, transl, scores, betas, expr, status=1))
|
||||
continue
|
||||
t_pred = time.monotonic()
|
||||
try:
|
||||
raw = self.model.predict(img, K)
|
||||
v3d, transl, scores, betas, expr = _extract_outputs(
|
||||
raw)
|
||||
status = 0
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("predict failed: %s", e)
|
||||
v3d, transl, scores, betas, expr = _zero_outputs()
|
||||
status = 1
|
||||
t_pred_end = time.monotonic()
|
||||
t_enc = time.monotonic()
|
||||
rsp = encode_response(
|
||||
v3d, transl, scores, betas, expr, status=status)
|
||||
t_enc_end = time.monotonic()
|
||||
pred_ms = (t_pred_end - t_pred) * 1e3
|
||||
encode_ms = (t_enc_end - t_enc) * 1e3
|
||||
served["n"] += 1
|
||||
served["sum_decode"] += decode_ms
|
||||
served["sum_pred"] += pred_ms
|
||||
served["sum_encode"] += encode_ms
|
||||
rsp_q.put(rsp)
|
||||
now = time.monotonic()
|
||||
if served["n"] % 30 == 0:
|
||||
dt = now - served["t0"]
|
||||
fps = served["n"] / max(1e-6, dt)
|
||||
LOG.info(
|
||||
"served %d frames at %.1f fps over %.1f s "
|
||||
"(decode=%.1fms pred=%.1fms encode=%.1fms)",
|
||||
served["n"], fps, dt,
|
||||
served["sum_decode"] / served["n"],
|
||||
served["sum_pred"] / served["n"],
|
||||
served["sum_encode"] / served["n"])
|
||||
finally:
|
||||
conn_stop.set()
|
||||
try:
|
||||
rsp_q.put_nowait(b"")
|
||||
except queue.Full:
|
||||
pass
|
||||
|
||||
def writer() -> None:
|
||||
try:
|
||||
while not conn_stop.is_set() and not self._stop.is_set():
|
||||
try:
|
||||
rsp = rsp_q.get(timeout=0.5)
|
||||
except queue.Empty:
|
||||
continue
|
||||
if rsp == b"":
|
||||
break
|
||||
conn.sendall(rsp)
|
||||
except (ConnectionError, BrokenPipeError, OSError) as e:
|
||||
LOG.info("writer exit: %s", e)
|
||||
finally:
|
||||
conn_stop.set()
|
||||
|
||||
t_r = threading.Thread(target=reader, name="srv-reader", daemon=True)
|
||||
t_w = threading.Thread(target=worker, name="srv-worker", daemon=True)
|
||||
t_x = threading.Thread(target=writer, name="srv-writer", daemon=True)
|
||||
t_r.start()
|
||||
t_w.start()
|
||||
t_x.start()
|
||||
t_r.join()
|
||||
t_w.join()
|
||||
t_x.join()
|
||||
dt = time.monotonic() - served["t0"]
|
||||
if served["n"] > 0:
|
||||
LOG.info("connection closed: served %d frames at %.1f fps "
|
||||
"over %.1f s", served["n"],
|
||||
served["n"] / max(1e-6, dt), dt)
|
||||
|
||||
|
||||
def run_bench(model: CoreMLModel, n: int = 30) -> None:
|
||||
"""Local synthetic bench (no socket)."""
|
||||
rng = np.random.default_rng(0)
|
||||
K = np.array([[672.0, 0.0, 336.0],
|
||||
[0.0, 672.0, 336.0],
|
||||
[0.0, 0.0, 1.0]], dtype=np.float32)
|
||||
img0 = rng.integers(0, 256, (IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)
|
||||
model.predict(img0, K)
|
||||
times = []
|
||||
for _ in range(n):
|
||||
img = rng.integers(0, 256, (IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)
|
||||
t0 = time.monotonic()
|
||||
model.predict(img, K)
|
||||
times.append((time.monotonic() - t0) * 1e3)
|
||||
ts = sorted(times)
|
||||
median = ts[len(ts) // 2]
|
||||
mean = sum(times) / len(times)
|
||||
p90 = ts[int(len(ts) * 0.9)]
|
||||
LOG.info("bench n=%d median=%.1fms mean=%.1fms p90=%.1fms (%.1f fps)",
|
||||
n, median, mean, p90, 1000.0 / median)
|
||||
|
||||
|
||||
def run_bench_async(model: CoreMLModel, host: str, port: int,
|
||||
n: int = 60) -> None:
|
||||
"""End-to-end pipeline bench via real socket loopback."""
|
||||
import threading
|
||||
server = Server(model, host, port)
|
||||
th = threading.Thread(target=server.serve, daemon=True)
|
||||
th.start()
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
from data_only_viz.multihmr_remote import MultiHMRRemoteBackend
|
||||
except ImportError:
|
||||
# When the server runs standalone, the client package may not be
|
||||
# importable. Skip with a friendly message.
|
||||
LOG.warning("data_only_viz package not importable on this host, "
|
||||
"skipping --bench-async")
|
||||
server.stop()
|
||||
return
|
||||
os.environ.setdefault("MULTIHMR_REMOTE_HOST", host)
|
||||
os.environ.setdefault("MULTIHMR_REMOTE_PORT", str(port))
|
||||
be = MultiHMRRemoteBackend(host=host, port=port)
|
||||
rng = np.random.default_rng(0)
|
||||
K = np.array([[672.0, 0.0, 336.0],
|
||||
[0.0, 672.0, 336.0],
|
||||
[0.0, 0.0, 1.0]], dtype=np.float32)
|
||||
t0 = time.monotonic()
|
||||
got = 0
|
||||
for _ in range(n):
|
||||
img = (rng.random((3, IMG_SIZE, IMG_SIZE), dtype=np.float32))
|
||||
out = be.infer(img, K)
|
||||
if out is not None:
|
||||
got += 1
|
||||
time.sleep(0.01)
|
||||
dt = time.monotonic() - t0
|
||||
LOG.info("bench-async submitted=%d got=%d in %.2fs (%.1f fps submit)",
|
||||
n, got, dt, n / max(1e-6, dt))
|
||||
be.close()
|
||||
server.stop()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(description="Multi-HMR TCP server")
|
||||
ap.add_argument("--mlpackage", type=Path, default=DEFAULT_MLPACKAGE)
|
||||
ap.add_argument("--host", default=os.environ.get(
|
||||
"MULTIHMR_SERVER_HOST", "0.0.0.0"))
|
||||
ap.add_argument("--port", type=int, default=int(os.environ.get(
|
||||
"MULTIHMR_SERVER_PORT", "57140")))
|
||||
ap.add_argument("--bench", action="store_true",
|
||||
help="local synthetic bench, no socket")
|
||||
ap.add_argument("--bench-async", action="store_true",
|
||||
help="loopback pipeline bench through real sockets")
|
||||
ap.add_argument("--bench-n", type=int, default=30)
|
||||
ap.add_argument("--log-level", default="INFO")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
logging.basicConfig(
|
||||
level=args.log_level.upper(),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
|
||||
model = CoreMLModel(args.mlpackage)
|
||||
|
||||
if args.bench:
|
||||
run_bench(model, n=args.bench_n)
|
||||
return 0
|
||||
if args.bench_async:
|
||||
run_bench_async(model, "127.0.0.1", args.port, n=args.bench_n)
|
||||
return 0
|
||||
|
||||
server = Server(model, args.host, args.port)
|
||||
|
||||
def _sigint(*_a):
|
||||
LOG.info("SIGINT received, stopping")
|
||||
server.stop()
|
||||
|
||||
signal.signal(signal.SIGINT, _sigint)
|
||||
signal.signal(signal.SIGTERM, _sigint)
|
||||
try:
|
||||
server.serve()
|
||||
except KeyboardInterrupt:
|
||||
server.stop()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
# Push the Multi-HMR mlpackage + server to macm1 (M1 Max, 32-core GPU)
|
||||
# and launch the inference server in the background.
|
||||
#
|
||||
# Prereqs on macm1 :
|
||||
# * passwordless ssh (Tailscale alias 'macm1' or LAN)
|
||||
# * uv installed
|
||||
# * Python 3.12 available via uv (uv pulls it)
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/setup_remote_macm1.sh
|
||||
# MACM1_HOST=clems@192.168.0.175 ./scripts/setup_remote_macm1.sh
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${MACM1_HOST:-macm1}"
|
||||
PORT="${MULTIHMR_SERVER_PORT:-57140}"
|
||||
MLPACKAGE_LOCAL="${MLPACKAGE_LOCAL:-$HOME/.cache/av-live-multihmr/multihmr_full_672_s.mlpackage}"
|
||||
REMOTE_TMP="/tmp/av-live-multihmr"
|
||||
REMOTE_VENV="/tmp/av-live-multihmr/venv"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
echo "==> Target host : $HOST"
|
||||
echo "==> mlpackage : $MLPACKAGE_LOCAL"
|
||||
|
||||
if [ ! -d "$MLPACKAGE_LOCAL" ]; then
|
||||
echo "ERROR: mlpackage missing at $MLPACKAGE_LOCAL" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Creating remote tmp dir"
|
||||
ssh "$HOST" "mkdir -p $REMOTE_TMP"
|
||||
|
||||
echo "==> rsync mlpackage (~70 MB, may take a moment first time)"
|
||||
rsync -a --delete \
|
||||
"$MLPACKAGE_LOCAL/" \
|
||||
"$HOST:$REMOTE_TMP/multihmr_full_672_s.mlpackage/"
|
||||
|
||||
echo "==> rsync multihmr_server.py"
|
||||
rsync -a "$SCRIPT_DIR/multihmr_server.py" \
|
||||
"$HOST:$REMOTE_TMP/multihmr_server.py"
|
||||
|
||||
echo "==> Provision Python 3.12 venv with uv (idempotent)"
|
||||
ssh "$HOST" "bash -lc 'set -e
|
||||
if [ ! -x $REMOTE_VENV/bin/python ]; then
|
||||
uv venv --python 3.12 $REMOTE_VENV --quiet
|
||||
fi
|
||||
uv pip install --python $REMOTE_VENV/bin/python --quiet \
|
||||
coremltools numpy opencv-python-headless \
|
||||
pyobjc-core pyobjc-framework-Cocoa pyobjc-framework-CoreML
|
||||
'"
|
||||
|
||||
echo "==> Killing any stale server on :$PORT"
|
||||
ssh "$HOST" "bash -lc 'pkill -f multihmr_server.py 2>/dev/null || true; sleep 0.3'"
|
||||
|
||||
echo "==> Launching server (background)"
|
||||
ssh "$HOST" "bash -lc 'cd $REMOTE_TMP && \
|
||||
MULTIHMR_SERVER_PORT=$PORT \
|
||||
nohup $REMOTE_VENV/bin/python multihmr_server.py \
|
||||
--mlpackage $REMOTE_TMP/multihmr_full_672_s.mlpackage \
|
||||
--port $PORT \
|
||||
>> $REMOTE_TMP/server.log 2>&1 &
|
||||
echo \$! > $REMOTE_TMP/server.pid
|
||||
disown || true'"
|
||||
|
||||
echo "==> Waiting for server to be ready"
|
||||
REMOTE_ADDR=$(ssh "$HOST" 'echo $SSH_CONNECTION' | awk '{print $3}')
|
||||
# Fallback to host alias if SSH_CONNECTION trick fails.
|
||||
if [ -z "${REMOTE_ADDR:-}" ]; then REMOTE_ADDR="$HOST"; fi
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
if ssh "$HOST" "bash -lc 'nc -z 127.0.0.1 $PORT 2>/dev/null'"; then
|
||||
echo "==> Server up on $HOST:$PORT (probed via localhost on host)"
|
||||
echo "==> Reachable from this Mac at $REMOTE_ADDR:$PORT"
|
||||
ssh "$HOST" "tail -n 20 $REMOTE_TMP/server.log" || true
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "ERROR: server did not come up within 30s. Last log lines:" >&2
|
||||
ssh "$HOST" "tail -n 60 $REMOTE_TMP/server.log" || true
|
||||
exit 1
|
||||
@@ -34,6 +34,10 @@ struct SceneUniforms {
|
||||
float hand_l_y;
|
||||
float hand_r_x;
|
||||
float hand_r_y;
|
||||
float hand_height;
|
||||
float hand_openness;
|
||||
float hand_speed;
|
||||
float hand_dist;
|
||||
float _pad0;
|
||||
float _pad1;
|
||||
};
|
||||
@@ -454,6 +458,10 @@ fragment float4 bg_fragment(VsOut in [[stage_in]],
|
||||
p.x *= U.width / U.height;
|
||||
|
||||
int mode = int(U.viz_mode + 0.5);
|
||||
|
||||
// Mains ecartees -> leger zoom (sauf mode 8 qui pilote sa propre camera).
|
||||
if (mode != 8) { p *= (1.0 - U.hand_dist * 0.2); }
|
||||
|
||||
float3 color;
|
||||
if (mode == 1) color = mode_tunnel(p, U);
|
||||
else if (mode == 2) color = mode_plasma(p, U);
|
||||
@@ -466,6 +474,11 @@ fragment float4 bg_fragment(VsOut in [[stage_in]],
|
||||
else if (mode == 9) color = mode_openpos(p, U);
|
||||
else color = mode_storm(p, U);
|
||||
|
||||
// Modulation expressive pilotee par les mains (subtile, tous les modes).
|
||||
color *= (1.0 + U.hand_height * 0.6); // luminosite
|
||||
color += U.hand_openness * 0.5 * color; // bloom / dispersion
|
||||
color += U.hand_speed * 0.05 * sin(U.time * 8.0 + p.x * 10.0); // turbulence
|
||||
|
||||
// Flash global + vignette
|
||||
color += float3(U.lightning_flash * 1.2);
|
||||
color *= vignette(p);
|
||||
|
||||
@@ -45,8 +45,13 @@ class SMPLXTCPSender:
|
||||
def __init__(self, state: State, host: str = "127.0.0.1",
|
||||
port: int = PORT, target_fps: float = 30.0,
|
||||
enable_rigging: bool = True) -> None:
|
||||
import os as _os
|
||||
from .config import VizConfig as _VizConfig
|
||||
_cfg = _VizConfig.from_env()
|
||||
self.state = state
|
||||
self.host = host
|
||||
# AVBODY_HOST env wins; otherwise honor the caller-supplied host
|
||||
# (pre-VizConfig contract: environ.get("AVBODY_HOST", host)).
|
||||
self.host = _cfg.avbody_host if "AVBODY_HOST" in _os.environ else host
|
||||
self.port = port
|
||||
self.period = 1.0 / max(1.0, target_fps)
|
||||
self._stop = threading.Event()
|
||||
@@ -56,7 +61,7 @@ class SMPLXTCPSender:
|
||||
# on translate le mesh via le delta pelvis Apple Vision (30 fps).
|
||||
# MULTIHMR_REID: 'dino' (try DINOv2 + IoU fusion, fallback IoU) /
|
||||
# 'iou' (pure IoU). Default: 'dino' if mlpackage exists.
|
||||
reid_mode = os.environ.get("MULTIHMR_REID", "dino").lower()
|
||||
reid_mode = _cfg.multihmr_reid
|
||||
dino = None
|
||||
if enable_rigging and reid_mode == "dino" and DinoReid is not None:
|
||||
try:
|
||||
@@ -69,7 +74,7 @@ class SMPLXTCPSender:
|
||||
except Exception as e: # noqa: BLE001
|
||||
LOG.warning("MeshRigger: dino load failed (%s), IoU only", e)
|
||||
dino = None
|
||||
dino_weight = float(os.environ.get("MULTIHMR_REID_ALPHA", "0.5"))
|
||||
dino_weight = _cfg.multihmr_reid_alpha
|
||||
self._rigger = MeshRigger(
|
||||
state, dino_weight=dino_weight,
|
||||
dino_reid=dino) if enable_rigging else None
|
||||
|
||||
@@ -102,6 +102,16 @@ class State:
|
||||
persons_body: list[list[PoseKp]] = field(default_factory=list)
|
||||
persons_face: list[list[PoseKp]] = field(default_factory=list)
|
||||
persons_hands: list[list[PoseKp]] = field(default_factory=list)
|
||||
# iPhone Vision hands (on-device, 21 kp MediaPipe order, .right upright,
|
||||
# y already top-left/down). Stored separately from MediaPipe persons_hands
|
||||
# so the air-piano can prefer this stabler, rotation-invariant source.
|
||||
persons_hands_iphone: list[list[PoseKp]] = field(default_factory=list)
|
||||
persons_hands_iphone_t: float = 0.0
|
||||
# Chirality for each entry in persons_hands_iphone: 0=left, 1=right.
|
||||
# Aligned 1:1 with persons_hands_iphone (same length after each TAG_HANDS update).
|
||||
# Not valid for MediaPipe-written persons_hands (stays empty on that path).
|
||||
persons_hands_chirality: list[int] = field(default_factory=list)
|
||||
hand_feats: dict | None = None
|
||||
# MediaPipe pose_world_landmarks per person : 33 keypoints in meters,
|
||||
# relative to the hip-center. Optional companion of persons_body
|
||||
# (image-space xy). Empty if no detection or backend doesn't emit it.
|
||||
@@ -138,6 +148,15 @@ class State:
|
||||
# (metres, hip-centered). Fresh = updated within < 1 s.
|
||||
persons_arkit_joints: dict = field(default_factory=dict)
|
||||
persons_arkit_last_t: dict = field(default_factory=dict)
|
||||
# ARKit 2D projected skeleton (Task 3): 91×2 float32 arrays (normalized
|
||||
# screen coords 0..1) per pid. Updated by IphoneUSBSource on TAG_SKELETON2D.
|
||||
persons_arkit_2d: dict[int, "np.ndarray"] = field(default_factory=dict)
|
||||
persons_arkit_2d_t: dict[int, float] = field(default_factory=dict)
|
||||
persons_arkit_2d_valid: dict = field(default_factory=dict)
|
||||
arkit_joint_names: list = field(default_factory=list)
|
||||
arkit_parents: list = field(default_factory=list)
|
||||
# iPhone video is mirrored (CONCERT_MIRROR); renderer mirrors overlays to match.
|
||||
mirror_2d: bool = False
|
||||
|
||||
# ---- LiDAR / ICP mesh fusion (Task 8 - 2026-05-14) ----
|
||||
# Set by the LidarTCPReader poller; consumed by FusionWorker.run_once.
|
||||
@@ -157,6 +176,13 @@ class State:
|
||||
latest_bgr_id: int = 0
|
||||
latest_bgr_t: float = 0.0
|
||||
|
||||
# Gesture slot status per hand slot (written by action_head_pub, read by renderer):
|
||||
# 0=absent, 1=detected(plausible+established not armed), 2=armed(near+facing), 3=pinch engaged
|
||||
gesture_slot_status: list[int] = field(default_factory=lambda: [0, 0])
|
||||
# Continuous quality score per slot ∈ [0, 1] (written alongside gesture_slot_status).
|
||||
# Drives panel frame brightness (0.25+0.75*q) and stroke thickness.
|
||||
gesture_slot_quality: list[float] = field(default_factory=lambda: [0.0, 0.0])
|
||||
|
||||
# Renderer
|
||||
width: int = 1280
|
||||
height: int = 720
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
"""ActionHeadPublisher emits /pose/finger when FINGER_PIANO is enabled."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from data_only_viz.finger_strike import FINGERTIPS, FINGER_BASES
|
||||
|
||||
|
||||
class _FakeState:
|
||||
def __init__(self) -> None:
|
||||
self.persons_hands = []
|
||||
self.mirror_2d = False
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def lock(self):
|
||||
return self._lock
|
||||
|
||||
|
||||
class _Kp:
|
||||
"""Landmark with .x .y .c attrs and index fallback for _coord."""
|
||||
__slots__ = ("x", "y", "z", "c")
|
||||
|
||||
def __init__(self, x: float, y: float, z: float = 0.0, c: float = 1.0) -> None:
|
||||
self.x, self.y, self.z, self.c = x, y, z, c
|
||||
|
||||
def __getitem__(self, i: int) -> float:
|
||||
return (self.x, self.y, self.z)[i]
|
||||
|
||||
def __len__(self) -> int:
|
||||
return 3
|
||||
|
||||
|
||||
def _hand(index_tip_y: float, base_y: float = 0.4, c: float = 1.0):
|
||||
lm = [_Kp(0.3, base_y, 0.0, c) for _ in range(21)]
|
||||
lm[0] = _Kp(0.3, base_y + 0.15, 0.0, c) # wrist 0.15 below center -> hand_size 0.15
|
||||
lm[9] = _Kp(0.3, base_y, 0.0, c) # middle-MCP at center -> size 0.15 > near_min
|
||||
lm[FINGERTIPS[1]] = _Kp(0.3, index_tip_y, 0.0, c)
|
||||
lm[FINGER_BASES[1]] = _Kp(0.3, base_y, 0.0, c)
|
||||
# Spread index-MCP(5) and pinky-MCP(17) AFTER FINGER_BASES[1]=5 assignment
|
||||
# so lm[5] is not overwritten. facing = 0.12/0.15 = 0.8 >= face_min=0.5.
|
||||
# Only x changes; y=base_y is preserved so FingerStrikeDetector base_y is correct.
|
||||
lm[5] = _Kp(0.3 - 0.06, base_y, 0.0, c)
|
||||
lm[17] = _Kp(0.3 + 0.06, base_y, 0.0, c)
|
||||
return lm
|
||||
|
||||
|
||||
def _pub(monkeypatch, enabled: bool):
|
||||
monkeypatch.setenv("FINGER_PIANO", "1" if enabled else "0")
|
||||
monkeypatch.setenv("FINGER_STRIKE_VEL", "0.02")
|
||||
monkeypatch.setenv("FINGER_STRIKE_REFRACTORY_MS", "120")
|
||||
# Instant establishment so legacy strike/pinch tests need no warmup ticks.
|
||||
monkeypatch.setenv("HAND_PERSIST_FRAMES", "1")
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
# bypass __init__ heavy ckpt load: construct then set only what we touch
|
||||
pub = ActionHeadPublisher.__new__(ActionHeadPublisher)
|
||||
pub.state = _FakeState()
|
||||
pub.bridge = MagicMock()
|
||||
pub._init_finger_piano() # helper that reads env + builds detector
|
||||
return pub
|
||||
|
||||
|
||||
def test_emit_fingers_fires_when_enabled(monkeypatch):
|
||||
pub = _pub(monkeypatch, enabled=True)
|
||||
pub.state.persons_hands = [_hand(0.40)]
|
||||
pub._emit_fingers(0.00) # prime
|
||||
pub.state.persons_hands = [_hand(0.46)] # downward strike
|
||||
pub._emit_fingers(0.04)
|
||||
assert pub.bridge.send_finger.called
|
||||
|
||||
|
||||
def test_emit_fingers_noop_when_disabled(monkeypatch):
|
||||
pub = _pub(monkeypatch, enabled=False)
|
||||
pub.state.persons_hands = [_hand(0.40)]
|
||||
pub._emit_fingers(0.00)
|
||||
pub.state.persons_hands = [_hand(0.46)]
|
||||
pub._emit_fingers(0.04)
|
||||
assert not pub.bridge.send_finger.called
|
||||
|
||||
|
||||
def _pub_envs(monkeypatch, finger_piano: str, pinch_enable: str):
|
||||
"""Build a publisher with both gates set independently."""
|
||||
monkeypatch.setenv("FINGER_PIANO", finger_piano)
|
||||
monkeypatch.setenv("PINCH_ENABLE", pinch_enable)
|
||||
monkeypatch.setenv("FINGER_STRIKE_VEL", "0.02")
|
||||
monkeypatch.setenv("FINGER_STRIKE_REFRACTORY_MS", "120")
|
||||
# Instant establishment so legacy tests need no warmup ticks.
|
||||
monkeypatch.setenv("HAND_PERSIST_FRAMES", "1")
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
pub = ActionHeadPublisher.__new__(ActionHeadPublisher)
|
||||
pub.state = _FakeState()
|
||||
pub.bridge = MagicMock()
|
||||
pub._init_finger_piano()
|
||||
return pub
|
||||
|
||||
|
||||
def test_emit_pinch_fires_when_pinch_enabled(monkeypatch):
|
||||
# PINCH_ENABLE on, FINGER_PIANO off -> pinches still emit (decoupled).
|
||||
pub = _pub_envs(monkeypatch, finger_piano="0", pinch_enable="1")
|
||||
pub._pinch_det = MagicMock()
|
||||
pub._pinch_det.step.return_value = [MagicMock(hand=0, finger=1)]
|
||||
pub.state.persons_hands = [_hand(0.40)]
|
||||
pub._emit_pinch(0.0)
|
||||
assert pub.bridge.send_pinch.called
|
||||
|
||||
|
||||
def test_emit_pinch_off_when_pinch_disabled_even_with_finger_piano(monkeypatch):
|
||||
# Decouple: FINGER_PIANO on but PINCH_ENABLE off -> no pinch emitted.
|
||||
pub = _pub_envs(monkeypatch, finger_piano="1", pinch_enable="0")
|
||||
pub._pinch_det = MagicMock()
|
||||
pub._pinch_det.step.return_value = [MagicMock(hand=0, finger=1)]
|
||||
pub.state.persons_hands = [_hand(0.40)]
|
||||
pub._emit_pinch(0.0)
|
||||
assert not pub.bridge.send_pinch.called
|
||||
|
||||
|
||||
def test_emit_fingers_off_when_only_pinch_enabled(monkeypatch):
|
||||
# Decouple: PINCH_ENABLE on, FINGER_PIANO off -> air-piano stays silent.
|
||||
pub = _pub_envs(monkeypatch, finger_piano="0", pinch_enable="1")
|
||||
pub.state.persons_hands = [_hand(0.40)]
|
||||
pub._emit_fingers(0.00)
|
||||
pub.state.persons_hands = [_hand(0.46)]
|
||||
pub._emit_fingers(0.04)
|
||||
assert not pub.bridge.send_finger.called
|
||||
|
||||
|
||||
def test_pinch_gate_env_plumbing(monkeypatch):
|
||||
monkeypatch.setenv("PINCH_EXT_RATIO", "1.5")
|
||||
monkeypatch.setenv("PINCH_EXT_MIN", "3")
|
||||
monkeypatch.setenv("PINCH_DEBOUNCE_FRAMES", "5")
|
||||
pub = _pub_envs(monkeypatch, finger_piano="0", pinch_enable="1")
|
||||
assert pub._pinch_det.ext_ratio == 1.5
|
||||
assert pub._pinch_det.ext_min == 3
|
||||
assert pub._pinch_det.debounce_frames == 5
|
||||
|
||||
|
||||
def test_pinch_gate_env_defaults_are_strict(monkeypatch):
|
||||
# Live defaults: gate ON (2 extended fingers) + 3-frame debounce.
|
||||
for k in ("PINCH_EXT_RATIO", "PINCH_EXT_MIN", "PINCH_DEBOUNCE_FRAMES"):
|
||||
monkeypatch.delenv(k, raising=False)
|
||||
pub = _pub_envs(monkeypatch, finger_piano="0", pinch_enable="1")
|
||||
assert pub._pinch_det.ext_ratio == 1.35
|
||||
assert pub._pinch_det.ext_min == 1 # 1 since 2026-07-02 (inner-finger fix)
|
||||
assert pub._pinch_det.debounce_frames == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Item 1a: mirror_2d coherence — _emit_hands passes mirror to extractor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_hand_row(cx: float, c: float = 1.0) -> list:
|
||||
"""21 _Kp landmarks with enough geometry for features."""
|
||||
lm = [_Kp(cx, 0.5, 0.0, c) for _ in range(21)]
|
||||
lm[0] = _Kp(cx, 0.6, 0.0, c) # wrist
|
||||
lm[9] = _Kp(cx, 0.45, 0.0, c) # middle-MCP -> hand_size 0.15 > near threshold
|
||||
return lm
|
||||
|
||||
|
||||
def test_emit_hands_mirror_2d_flips_cx_slot_assignment(monkeypatch):
|
||||
"""With state.mirror_2d=True, the cx fallback in _emit_hands assigns
|
||||
the RIGHT-screen hand to the L slot — coherent with the mirrored renderer."""
|
||||
from data_only_viz.hand_features import HandFeatureExtractor
|
||||
pub = _pub(monkeypatch, enabled=False)
|
||||
pub._hand_ext = HandFeatureExtractor() # not set by _pub/_init_finger_piano
|
||||
|
||||
h_low_cx = _make_hand_row(0.2) # screen-left
|
||||
h_high_cx = _make_hand_row(0.8) # screen-right
|
||||
pub.state.persons_hands = [h_low_cx, h_high_cx]
|
||||
# No chirality -> cx fallback path
|
||||
# (persons_hands_chirality absent -> getattr returns None -> [] -> None)
|
||||
|
||||
# Without mirror: screen-left (cx=0.2) -> slot L
|
||||
pub.state.mirror_2d = False
|
||||
pub._emit_hands(0.0)
|
||||
feats_plain = pub.state.hand_feats
|
||||
assert feats_plain["L"] is not None
|
||||
assert feats_plain["L"]["cx"] == pytest.approx(0.2, abs=0.05)
|
||||
|
||||
# With mirror: cx fallback flips -> screen-right (cx=0.8) -> slot L
|
||||
pub.state.mirror_2d = True
|
||||
pub._emit_hands(0.0)
|
||||
feats_mirror = pub.state.hand_feats
|
||||
assert feats_mirror["L"] is not None
|
||||
assert feats_mirror["L"]["cx"] == pytest.approx(0.8, abs=0.05)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gesture input validation tests (spec: a, b, c, e, f)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _pub_gate(monkeypatch, persist_frames: int = 3, conf_min: str = "0.45"):
|
||||
"""Build a publisher with explicit gate params, finger + pinch enabled."""
|
||||
monkeypatch.setenv("FINGER_PIANO", "1")
|
||||
monkeypatch.setenv("PINCH_ENABLE", "1")
|
||||
monkeypatch.setenv("FINGER_STRIKE_VEL", "0.001") # very sensitive
|
||||
monkeypatch.setenv("FINGER_STRIKE_REFRACTORY_MS", "0")
|
||||
monkeypatch.setenv("HAND_PERSIST_FRAMES", str(persist_frames))
|
||||
monkeypatch.setenv("HAND_CONF_MIN", conf_min)
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
pub = ActionHeadPublisher.__new__(ActionHeadPublisher)
|
||||
pub.state = _FakeState()
|
||||
pub.bridge = MagicMock()
|
||||
pub._init_finger_piano()
|
||||
return pub
|
||||
|
||||
|
||||
def test_gesture_gate_blocks_low_confidence_hand(monkeypatch):
|
||||
"""(a) c=0.1 < HAND_CONF_MIN=0.45: plausibility gate blocks hand before detectors."""
|
||||
pub = _pub_gate(monkeypatch, persist_frames=1, conf_min="0.45")
|
||||
low_hand = _hand(0.40, c=0.1)
|
||||
for _ in range(10):
|
||||
pub.state.persons_hands = [low_hand]
|
||||
pub._emit_fingers(0.0)
|
||||
pub._emit_pinch(0.0)
|
||||
assert not pub.bridge.send_finger.called
|
||||
assert not pub.bridge.send_pinch.called
|
||||
|
||||
|
||||
def test_gesture_gate_plausible_hand_needs_min_frames(monkeypatch):
|
||||
"""(b) Plausible hand feeds gestures only after HAND_PERSIST_FRAMES ticks."""
|
||||
pub = _pub_gate(monkeypatch, persist_frames=3)
|
||||
good = _hand(0.40) # c=1.0, plausible
|
||||
|
||||
# Ticks 1 and 2: gate count < min_frames → hand not established → no slot
|
||||
for _ in range(2):
|
||||
pub.state.persons_hands = [good]
|
||||
pub._emit_fingers(0.0)
|
||||
assert not pub.bridge.send_finger.called
|
||||
|
||||
# Tick 3: established; move finger to trigger a strike
|
||||
pub.state.persons_hands = [good]
|
||||
pub._emit_fingers(0.0) # prime with hand in slot
|
||||
pub.state.persons_hands = [_hand(0.50)]
|
||||
pub._emit_fingers(0.1)
|
||||
assert pub.bridge.send_finger.called
|
||||
|
||||
|
||||
def test_gesture_gate_1_frame_ghost_never_fires(monkeypatch):
|
||||
"""(c) A confident ghost appearing for only 1-2 frames never triggers anything."""
|
||||
pub = _pub_gate(monkeypatch, persist_frames=3)
|
||||
ghost = _hand(0.40) # c=1.0 but short-lived
|
||||
|
||||
# One-frame flash then absence
|
||||
pub.state.persons_hands = [ghost]
|
||||
pub._emit_fingers(0.0)
|
||||
pub.state.persons_hands = []
|
||||
pub._emit_fingers(0.1)
|
||||
assert not pub.bridge.send_finger.called
|
||||
|
||||
# Two-frame flash then absence
|
||||
pub.state.persons_hands = [ghost]
|
||||
pub._emit_fingers(0.2)
|
||||
pub._emit_fingers(0.3) # count=2 still < 3
|
||||
pub.state.persons_hands = []
|
||||
pub._emit_fingers(0.4)
|
||||
assert not pub.bridge.send_finger.called
|
||||
|
||||
|
||||
def test_gesture_gate_mediapipe_hand_passes_through(monkeypatch):
|
||||
"""(e) MediaPipe c=1.0 hand passes plausibility gate and fires gestures normally."""
|
||||
pub = _pub_gate(monkeypatch, persist_frames=1, conf_min="0.45")
|
||||
mp_hand = _hand(0.40) # c=1.0, models MediaPipe c=1.0 landmark
|
||||
pub.state.persons_hands = [mp_hand]
|
||||
pub._emit_fingers(0.00) # prime (established on first tick: min_frames=1)
|
||||
pub.state.persons_hands = [_hand(0.50)]
|
||||
pub._emit_fingers(0.10) # strike
|
||||
assert pub.bridge.send_finger.called
|
||||
|
||||
|
||||
def test_gesture_gate_chirality_aligned_after_plausibility_mask(monkeypatch):
|
||||
"""(f) Invalid(idx=0) + valid(idx=1, chir=1) → only right slot filled."""
|
||||
import time as _time
|
||||
monkeypatch.setenv("HAND_PERSIST_FRAMES", "1")
|
||||
monkeypatch.setenv("HAND_CONF_MIN", "0.45")
|
||||
monkeypatch.setenv("FINGER_SOURCE", "iphone")
|
||||
monkeypatch.setenv("FINGER_PIANO", "0")
|
||||
monkeypatch.setenv("PINCH_ENABLE", "0")
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
|
||||
class _FakeStateIphone(_FakeState):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.persons_hands_iphone: list = []
|
||||
self.persons_hands_iphone_t: float = 0.0
|
||||
self.persons_hands_chirality: list = []
|
||||
|
||||
pub = ActionHeadPublisher.__new__(ActionHeadPublisher)
|
||||
pub.state = _FakeStateIphone()
|
||||
pub.bridge = MagicMock()
|
||||
pub._init_finger_piano()
|
||||
|
||||
invalid = _hand(0.40, c=0.1) # low-conf → filtered by plausibility gate
|
||||
valid = _hand(0.40, c=1.0) # passes gate
|
||||
|
||||
pub.state.persons_hands_iphone = [invalid, valid]
|
||||
pub.state.persons_hands_chirality = [0, 1] # invalid=left, valid=right
|
||||
pub.state.persons_hands_iphone_t = _time.perf_counter()
|
||||
|
||||
slotted = pub._step_stab()
|
||||
|
||||
# invalid (chir=0) was removed before route_hands; valid (chir=1) → slot R
|
||||
assert slotted[0] is None # left slot empty
|
||||
assert slotted[1] is not None # right slot has valid hand
|
||||
@@ -28,10 +28,22 @@ class _FakeState:
|
||||
|
||||
|
||||
def _make_smplx_person(pid: int, seed: int = 0) -> dict:
|
||||
"""Build a legacy dict-format person (backward-compat test fixture)."""
|
||||
rng = np.random.default_rng(seed)
|
||||
return {"pid": pid, "v3d": rng.normal(size=(10475, 3)).astype(np.float32)}
|
||||
|
||||
|
||||
def _make_real_smplx_person(pid: int, seed: int = 0):
|
||||
"""Build a real SMPLXPerson dataclass (as emitted by multi_hmr_worker)."""
|
||||
from data_only_viz.state import SMPLXPerson
|
||||
rng = np.random.default_rng(seed)
|
||||
return SMPLXPerson(
|
||||
pid=pid,
|
||||
vertices_3d=rng.normal(size=(10475, 3)).astype(np.float32),
|
||||
expression=np.zeros(10, dtype=np.float32),
|
||||
)
|
||||
|
||||
|
||||
def test_publisher_smplx_source_emits_osc() -> None:
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
@@ -152,3 +164,224 @@ def test_publisher_passes_hands_kp_to_step() -> None:
|
||||
assert len(captured_hands) == 1
|
||||
assert captured_hands[0] is not None
|
||||
assert captured_hands[0].shape == (42, 3)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task-3 tests: _emit_hands wired into _tick
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _fake_hand_21(seed: int = 0, c: float = 1.0):
|
||||
"""Return a list of 21 landmark-like objects with .x .y .c attrs.
|
||||
|
||||
Wrist (slot 0) at (0.3, 0.6) and middle-MCP (slot 9) at (0.3, 0.45)
|
||||
so hand_plausible passes: size=0.15 > 0.02, c=1.0 >= 0.45, in-frame.
|
||||
"""
|
||||
import types
|
||||
rng = __import__("random").Random(seed)
|
||||
lm = [
|
||||
types.SimpleNamespace(x=rng.random() * 0.4 + 0.2,
|
||||
y=rng.random() * 0.4 + 0.2, c=c)
|
||||
for _ in range(21)
|
||||
]
|
||||
lm[0] = types.SimpleNamespace(x=0.3, y=0.6, c=c) # wrist in frame
|
||||
lm[9] = types.SimpleNamespace(x=0.3, y=0.45, c=c) # middle-MCP, size=0.15
|
||||
return lm
|
||||
|
||||
|
||||
def test_emit_hands_called_once_per_tick_with_hands(monkeypatch) -> None:
|
||||
"""`bridge.send_hands` must be called once per tick with L/R/dist keys."""
|
||||
monkeypatch.setenv("HAND_PERSIST_FRAMES", "1") # instant establishment
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
|
||||
state.persons_hands = [_fake_hand_21(0)]
|
||||
state.persons_smplx = [_make_smplx_person(0)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
|
||||
bridge.send_hands.assert_called_once()
|
||||
feats, _t = bridge.send_hands.call_args[0]
|
||||
assert set(feats.keys()) >= {"L", "R", "dist"}
|
||||
# One hand present → L is populated, R is None
|
||||
assert feats["L"] is not None
|
||||
assert feats["R"] is None
|
||||
assert isinstance(feats["dist"], float)
|
||||
|
||||
|
||||
def test_emit_hands_called_once_per_tick_empty() -> None:
|
||||
"""`bridge.send_hands` still called once when persons_hands is empty."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
|
||||
state.persons_hands = []
|
||||
state.persons_smplx = [_make_smplx_person(0)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
|
||||
bridge.send_hands.assert_called_once()
|
||||
feats, _t = bridge.send_hands.call_args[0]
|
||||
assert set(feats.keys()) >= {"L", "R", "dist"}
|
||||
assert feats["L"] is None
|
||||
assert feats["R"] is None
|
||||
assert feats["dist"] == 0.0
|
||||
|
||||
|
||||
def test_emit_hands_single_person_fallback_sends_once() -> None:
|
||||
"""Single-person fallback (hands_present=True + left/right_hand_kp) → send_hands once."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
|
||||
# No per-person hands list; exercise the single-person fallback branch.
|
||||
state.persons_hands = []
|
||||
state.hands_present = True
|
||||
state.left_hand_kp = _fake_hand_21(0)
|
||||
state.right_hand_kp = _fake_hand_21(1)
|
||||
state.persons_smplx = [_make_smplx_person(0)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
|
||||
bridge.send_hands.assert_called_once()
|
||||
|
||||
|
||||
def test_emit_hands_once_for_two_pids() -> None:
|
||||
"""Two pids in persons_smplx → send_hands called exactly once, not once per pid."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
|
||||
state.persons_smplx = [_make_smplx_person(0), _make_smplx_person(1)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
|
||||
assert bridge.send_hands.call_count == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# B3 contract: real SMPLXPerson dataclass (multi_hmr_worker output)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_publisher_smplx_real_dataclass_emits_osc() -> None:
|
||||
"""SMPLXPerson dataclass (vertices_3d, not v3d) must not raise AttributeError."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
state.persons_smplx = [_make_real_smplx_person(7)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
actions = bridge.send_action.call_args_list
|
||||
assert len(actions) == 1
|
||||
assert actions[0].kwargs.get("pid", actions[0].args[0]) == 7
|
||||
bridge.send_enter.assert_called_with(pid=7)
|
||||
|
||||
|
||||
def test_publisher_smplx_real_dataclass_expression_used() -> None:
|
||||
"""Expression from SMPLXPerson.expression (ndarray) must not crash."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
p = _make_real_smplx_person(0)
|
||||
p.expression = np.ones(10, dtype=np.float32) * 0.5
|
||||
state.persons_smplx = [p]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
bridge.send_action.assert_called_once()
|
||||
|
||||
|
||||
def test_publisher_dict_format_still_works() -> None:
|
||||
"""Legacy dict format {"pid": ..., "v3d": ...} must remain supported."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
state.persons_smplx = [_make_smplx_person(99)]
|
||||
state.smplx_last_t = 1.0
|
||||
pub._tick(t_now=0.0)
|
||||
actions = bridge.send_action.call_args_list
|
||||
assert len(actions) == 1
|
||||
assert actions[0].kwargs.get("pid", actions[0].args[0]) == 99
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gesture validation: feature path requires established hands (spec test d)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_emit_hands_features_require_establishment(monkeypatch) -> None:
|
||||
"""(d) _emit_hands yields None for hands not yet established, non-None after.
|
||||
|
||||
Ghost = plausible hand present for 1-2 ticks only (never established).
|
||||
Real = same plausible hand held for HAND_PERSIST_FRAMES ticks.
|
||||
"""
|
||||
monkeypatch.setenv("HAND_PERSIST_FRAMES", "3")
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
|
||||
good_hand = _fake_hand_21(0)
|
||||
state.persons_hands = [good_hand]
|
||||
state.persons_smplx = [_make_smplx_person(0)]
|
||||
|
||||
# Ticks 1-2: hand count < 3 → not yet established → features are None
|
||||
for t in range(1, 3):
|
||||
state.smplx_last_t = float(t)
|
||||
pub._tick(t_now=float(t))
|
||||
feats_early, _ = bridge.send_hands.call_args_list[-1][0]
|
||||
assert feats_early["L"] is None # not established yet
|
||||
|
||||
# Tick 3: count reaches min_frames → established → features populated
|
||||
state.smplx_last_t = 3.0
|
||||
pub._tick(t_now=3.0)
|
||||
feats_est, _ = bridge.send_hands.call_args_list[-1][0]
|
||||
assert feats_est["L"] is not None # now established
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gesture slot status
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_gesture_status_defaults_to_zero_with_no_hands() -> None:
|
||||
"""With no hands, _update_gesture_slot_status() writes [0, 0]."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
state.gesture_slot_status = [99, 99] # sentinel: should be overwritten
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
pub._update_gesture_slot_status()
|
||||
assert state.gesture_slot_status == [0, 0]
|
||||
|
||||
|
||||
def test_gesture_status_armed_when_stab_active() -> None:
|
||||
"""gesture_slot_status[slot]=2 when stabilizer output is non-None (arm state)."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
state.gesture_slot_status = [0, 0]
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
# Directly mark slot 0 as active in the stabilizer's output cache
|
||||
pub._stab._out[0] = object()
|
||||
pub._update_gesture_slot_status()
|
||||
assert state.gesture_slot_status[0] == 2, "armed slot must report status 2"
|
||||
assert state.gesture_slot_status[1] == 0, "idle slot must report status 0"
|
||||
|
||||
|
||||
def test_gesture_status_engaged_when_pinch_active() -> None:
|
||||
"""gesture_slot_status[slot]=3 when a pinch is engaged in that slot."""
|
||||
from data_only_viz.action_head_pub import ActionHeadPublisher
|
||||
state = _FakeState()
|
||||
state.gesture_slot_status = [0, 0]
|
||||
bridge = MagicMock()
|
||||
pub = ActionHeadPublisher(state, bridge, ckpt_path=None)
|
||||
pub._stab._out[1] = object() # slot 1 armed
|
||||
pub._pinch_det._state[1][0].engaged = True # slot 1 finger engaged
|
||||
pub._update_gesture_slot_status()
|
||||
assert state.gesture_slot_status[0] == 0, "slot 0: idle"
|
||||
assert state.gesture_slot_status[1] == 3, "slot 1: engaged pinch = status 3"
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Task 3: decode_skeleton2D round-trip tests.
|
||||
|
||||
Wire layout (TAG_SKELETON2D = 6):
|
||||
91 joints * (x: f32 BE, y: f32 BE) = 728 bytes of floats
|
||||
+ 91 validity bytes
|
||||
= 819 bytes total
|
||||
"""
|
||||
import struct
|
||||
|
||||
from data_only_viz.scripts.iphone_usb_bridge import decode_skeleton2D, JOINT_COUNT
|
||||
|
||||
|
||||
def test_decode_skeleton2d_roundtrip():
|
||||
floats = []
|
||||
for i in range(JOINT_COUNT):
|
||||
floats += [i * 0.001, i * 0.002]
|
||||
payload = struct.pack(">" + "f" * (JOINT_COUNT * 2), *floats)
|
||||
payload += bytes(1 if i % 2 == 0 else 0 for i in range(JOINT_COUNT))
|
||||
out = decode_skeleton2D(payload)
|
||||
assert out is not None and len(out) == JOINT_COUNT
|
||||
assert abs(out[10][0] - 0.010) < 1e-5 and out[10][2] is True
|
||||
assert out[11][2] is False
|
||||
|
||||
|
||||
def test_decode_skeleton2d_wrong_length_returns_none():
|
||||
assert decode_skeleton2D(b"\x00" * 10) is None
|
||||
assert decode_skeleton2D(b"\x00" * 818) is None
|
||||
assert decode_skeleton2D(b"\x00" * 820) is None
|
||||
assert decode_skeleton2D(b"") is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 4: ARKit → MediaPipe-33 body builders
|
||||
# ---------------------------------------------------------------------------
|
||||
import numpy as np
|
||||
from data_only_viz.arkit_joint_map import arkit_body_2d, ARKIT91_TO_MP33
|
||||
|
||||
|
||||
def test_arkit_body_2d_maps_slots():
|
||||
arr = np.zeros((91, 2), dtype=np.float32)
|
||||
for ai, _mp in ARKIT91_TO_MP33:
|
||||
arr[ai] = [0.3 + ai * 0.001, 0.4]
|
||||
body = arkit_body_2d(arr)
|
||||
assert len(body) == 33
|
||||
# a mapped slot is confident; an unmapped face slot is not
|
||||
mapped_mp = ARKIT91_TO_MP33[0][1]
|
||||
assert body[mapped_mp].c > 0.5
|
||||
assert body[1].c < 0.5 # MP slot 1 (eye) has no ARKit source
|
||||
|
||||
|
||||
def test_arkit_body_3d_maps_slots():
|
||||
from data_only_viz.arkit_joint_map import arkit_body_3d
|
||||
arr = np.zeros((91, 3), dtype=np.float32)
|
||||
for ai, _mp in ARKIT91_TO_MP33:
|
||||
arr[ai] = [float(ai) * 0.01, float(ai) * 0.02, float(ai) * 0.03]
|
||||
body = arkit_body_3d(arr)
|
||||
assert len(body) == 33
|
||||
mapped_mp = ARKIT91_TO_MP33[0][1]
|
||||
assert body[mapped_mp].c > 0.5
|
||||
assert body[1].c < 0.5
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Pure ARKit skeleton -> line segment builder."""
|
||||
from data_only_viz.arkit_skeleton import arkit_segments, bones_from_parents
|
||||
|
||||
|
||||
def test_bones_skip_root():
|
||||
parents = [-1, 0, 1]
|
||||
assert bones_from_parents(parents) == [(1, 0), (2, 1)]
|
||||
|
||||
|
||||
def test_bones_skip_out_of_range():
|
||||
parents = [-1, 0, 99] # 99 has no joint
|
||||
assert bones_from_parents(parents) == [(1, 0)]
|
||||
|
||||
|
||||
def test_segments_basic():
|
||||
# Realistic bone lengths < 0.5 (horizontal spine-like chain)
|
||||
arr2d = [(0.0, 0.0), (0.2, 0.0), (0.4, 0.0)]
|
||||
valid = [True, True, True]
|
||||
parents = [-1, 0, 1]
|
||||
segs = arkit_segments(arr2d, valid, parents)
|
||||
assert segs == [(0.2, 0.0, 0.0, 0.0), (0.4, 0.0, 0.2, 0.0)]
|
||||
|
||||
|
||||
def test_segments_drop_invalid_endpoint():
|
||||
arr2d = [(0.0, 0.0), (0.2, 0.0), (0.4, 0.0)]
|
||||
valid = [True, False, True] # joint 1 invalid
|
||||
parents = [-1, 0, 1]
|
||||
# bone (1,0) drops (1 invalid); bone (2,1) drops (1 invalid)
|
||||
assert arkit_segments(arr2d, valid, parents) == []
|
||||
|
||||
|
||||
def test_segments_valid_none_keeps_all():
|
||||
arr2d = [(0.0, 0.0), (0.3, 0.0)] # bone length 0.3 < max_len=0.5
|
||||
parents = [-1, 0]
|
||||
assert arkit_segments(arr2d, None, parents) == [(0.3, 0.0, 0.0, 0.0)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# New filter tests (bounds + max_len)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_segments_out_of_bounds_endpoint_skipped():
|
||||
"""Endpoint outside [-bounds, 1+bounds] → bone skipped."""
|
||||
# Joint 1 at x=-0.2 is outside [-0.1, 1.1]
|
||||
arr2d = [(0.5, 0.5), (-0.2, 0.5), (0.7, 0.5)]
|
||||
valid = [True, True, True]
|
||||
parents = [-1, 0, 1]
|
||||
# bone (1,0): joint 1 out of bounds → skip
|
||||
# bone (2,1): joint 1 out of bounds → skip
|
||||
segs = arkit_segments(arr2d, valid, parents, bounds=0.1)
|
||||
assert segs == []
|
||||
|
||||
|
||||
def test_segments_overlong_bone_skipped():
|
||||
"""Bone length > max_len → bone skipped."""
|
||||
# Bone from (0.0, 0.5) to (1.0, 0.5): length=1.0 > max_len=0.5
|
||||
arr2d = [(0.0, 0.5), (1.0, 0.5)]
|
||||
valid = [True, True]
|
||||
parents = [-1, 0]
|
||||
segs = arkit_segments(arr2d, valid, parents, max_len=0.5)
|
||||
assert segs == []
|
||||
|
||||
|
||||
def test_segments_normal_skeleton_passes_with_defaults():
|
||||
"""Realistic bones (length < 0.5, coords in [0,1]) pass all filters."""
|
||||
arr2d = [(0.5, 0.3), (0.5, 0.6), (0.5, 0.8)] # vertical spine-like
|
||||
valid = [True, True, True]
|
||||
parents = [-1, 0, 1]
|
||||
segs = arkit_segments(arr2d, valid, parents)
|
||||
# Both bones length = 0.3 < max_len=0.5; all coords in [0,1] ⊂ [-0.1,1.1]
|
||||
assert len(segs) == 2
|
||||
assert segs[0] == (0.5, 0.6, 0.5, 0.3)
|
||||
assert segs[1] == (0.5, 0.8, 0.5, 0.6)
|
||||
|
||||
|
||||
def test_segments_in_bounds_endpoint_passes():
|
||||
"""Endpoint exactly at boundary (0.0, 1.0) is kept."""
|
||||
arr2d = [(0.5, 0.5), (0.5, 0.2)] # bone length 0.3 < 0.5
|
||||
valid = [True, True]
|
||||
parents = [-1, 0]
|
||||
segs = arkit_segments(arr2d, valid, parents, bounds=0.1)
|
||||
assert len(segs) == 1
|
||||
|
||||
|
||||
def test_segments_legacy_diagonal_bone_now_filtered():
|
||||
"""Regression pin: the pre-filter test fixtures used a full-diagonal
|
||||
bone (0,0)->(0.5,0.5), length ~0.707. The max_len=0.5 default now
|
||||
REJECTS it — this is the intended behavior change of the v2 filter,
|
||||
made explicit here instead of silently retuning old fixtures."""
|
||||
arr2d = [(0.0, 0.0), (0.5, 0.5)]
|
||||
valid = [True, True]
|
||||
parents = [-1, 0]
|
||||
assert arkit_segments(arr2d, valid, parents) == []
|
||||
# ...and the env-tunable knob restores it for close-up shots.
|
||||
assert len(arkit_segments(arr2d, valid, parents, max_len=0.8)) == 1
|
||||
|
||||
|
||||
def test_finger_joint_mask_hides_fingers_keeps_wrist():
|
||||
from data_only_viz.arkit_skeleton import finger_joint_mask
|
||||
names = ["root", "left_hand_joint", "left_handIndexStart_joint",
|
||||
"left_handThumb1_joint", "right_handMid2_joint",
|
||||
"right_handPinkyEnd_joint", "head_joint", "right_hand_joint",
|
||||
"left_handRing3_joint"]
|
||||
mask = finger_joint_mask(names)
|
||||
assert mask == [False, False, True, True, True, True, False, False, True]
|
||||
@@ -0,0 +1,33 @@
|
||||
"""AVLiveWire topology payload round-trip (mirror of Swift TopologyPayload)."""
|
||||
from data_only_viz.arkit_topology import decode_topology, encode_topology
|
||||
|
||||
|
||||
def test_round_trip():
|
||||
names = ["root_joint", "hips_joint", "left_arm_joint"]
|
||||
parents = [-1, 0, 1]
|
||||
payload = encode_topology(names, parents)
|
||||
out = decode_topology(payload)
|
||||
assert out is not None
|
||||
got_names, got_parents = out
|
||||
assert got_names == names
|
||||
assert got_parents == parents
|
||||
|
||||
|
||||
def test_empty():
|
||||
payload = encode_topology([], [])
|
||||
out = decode_topology(payload)
|
||||
assert out == ([], [])
|
||||
|
||||
|
||||
def test_rejects_truncated():
|
||||
# Claims 3 joints, provides no body.
|
||||
assert decode_topology(b"\x00\x03") is None
|
||||
|
||||
|
||||
def test_rejects_too_short():
|
||||
assert decode_topology(b"\x00") is None
|
||||
|
||||
|
||||
def test_rejects_truncated_parents():
|
||||
# 1 joint "a", parents section missing.
|
||||
assert decode_topology(b"\x00\x01\x01a") is None
|
||||
@@ -0,0 +1,14 @@
|
||||
"""State carries ARKit topology + 2D validity for the renderer."""
|
||||
from data_only_viz.state import State
|
||||
|
||||
|
||||
def test_state_has_topology_fields():
|
||||
s = State()
|
||||
assert s.arkit_joint_names == []
|
||||
assert s.arkit_parents == []
|
||||
assert s.persons_arkit_2d_valid == {}
|
||||
|
||||
|
||||
def test_topology_tag_constant():
|
||||
from data_only_viz.iphone_usb_source import TAG_TOPOLOGY
|
||||
assert TAG_TOPOLOGY == 7
|
||||
@@ -0,0 +1,432 @@
|
||||
"""Tests for data_only_viz.config — VizConfig defaults, bool conventions, round-trip."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from data_only_viz.config import VizConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default values
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_defaults_mediapipe():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.mediapipe_delegate == "gpu"
|
||||
assert cfg.video_rotate == "none"
|
||||
|
||||
|
||||
def test_defaults_pose_discrimination():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.pose_ghost_min_visible == 10
|
||||
assert cfg.pose_ghost_min_conf == 0.5
|
||||
assert cfg.pose_hand_min_visible == 15
|
||||
assert cfg.pose_face_min_visible == 50
|
||||
assert cfg.pose_nms_iou == 0.7
|
||||
|
||||
|
||||
def test_defaults_pose_filter():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.pose_filter is None
|
||||
assert cfg.pose_filter_face is None
|
||||
assert cfg.pose_filter_hand is None
|
||||
|
||||
|
||||
def test_defaults_finger_piano():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.finger_piano is False
|
||||
assert cfg.finger_strike_vel == pytest.approx(0.02)
|
||||
assert cfg.finger_strike_refractory_ms == pytest.approx(120.0)
|
||||
assert cfg.finger_source == "auto"
|
||||
assert cfg.finger_debug is False
|
||||
|
||||
|
||||
def test_defaults_pinch():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.pinch_enable is False
|
||||
assert cfg.pinch_ratio_on == pytest.approx(0.50)
|
||||
assert cfg.pinch_ratio_off == pytest.approx(0.65)
|
||||
assert cfg.pinch_refractory_ms == pytest.approx(250.0)
|
||||
assert cfg.pinch_margin == pytest.approx(0.05)
|
||||
assert cfg.pinch_ext_ratio == pytest.approx(1.35)
|
||||
assert cfg.pinch_ext_min == 1
|
||||
assert cfg.pinch_debounce_frames == 3
|
||||
|
||||
|
||||
def test_defaults_hand_display():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.hand_conf_min == pytest.approx(0.45)
|
||||
assert cfg.hand_persist_frames == 3
|
||||
# HAND_SWAP_LR validated correct 2026-07-02: default OFF
|
||||
assert cfg.hand_swap_lr is False
|
||||
assert cfg.hand_near_min == pytest.approx(0.10)
|
||||
assert cfg.arkit_bone_max == pytest.approx(0.5)
|
||||
assert cfg.arkit_full_skeleton is True
|
||||
# detection v3 stabilizer knobs
|
||||
assert cfg.hand_persist_grace == 2
|
||||
assert cfg.hand_near_off == pytest.approx(0.08)
|
||||
assert cfg.hand_hold_frames == 2
|
||||
|
||||
|
||||
def test_defaults_misc():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.viz_hud is False
|
||||
assert cfg.iphone_osc_port == 57128
|
||||
assert cfg.concert_mirror is True
|
||||
|
||||
|
||||
def test_defaults_icp():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.icp_fusion is False
|
||||
assert cfg.icp_lidar_host is None
|
||||
assert cfg.icp_lidar_port == 5500
|
||||
assert cfg.icp_lidar_extrinsic is None
|
||||
|
||||
|
||||
def test_defaults_multihmr():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.multihmr_backend == "pytorch"
|
||||
assert cfg.multihmr_loop_fps == pytest.approx(30.0)
|
||||
assert cfg.multihmr_autocast is False
|
||||
assert cfg.multihmr_remote_host == "192.168.0.175"
|
||||
assert cfg.multihmr_remote_port == 57140
|
||||
assert cfg.multihmr_remote_jpeg is True
|
||||
assert cfg.multihmr_remote_jpeg_quality == 80
|
||||
assert cfg.multihmr_remote_async is True
|
||||
assert cfg.multihmr_reid == "dino"
|
||||
assert cfg.multihmr_reid_alpha == pytest.approx(0.5)
|
||||
|
||||
|
||||
def test_defaults_osc():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.avbody_host == "127.0.0.1"
|
||||
assert cfg.vdmx_osc_host is None
|
||||
assert cfg.vdmx_osc_port == 1234
|
||||
|
||||
|
||||
def test_defaults_worker_selection():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.av_live_mediapipe is True
|
||||
assert cfg.av_live_apple_vision is True
|
||||
assert cfg.av_live_coreml is True
|
||||
assert cfg.av_live_detrpose is False
|
||||
assert cfg.av_live_parallel_pose == "both"
|
||||
assert cfg.mesh_rig is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bool convention: standard (not in ("0","","false","False") → True)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("0", False),
|
||||
("", False),
|
||||
("false", False),
|
||||
("False", False),
|
||||
("1", True),
|
||||
("true", True),
|
||||
("True", True),
|
||||
("yes", True),
|
||||
])
|
||||
def test_bool_std_finger_piano(val, expected):
|
||||
cfg = VizConfig.from_env({"FINGER_PIANO": val})
|
||||
assert cfg.finger_piano is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("0", False),
|
||||
("", False),
|
||||
("false", False),
|
||||
("False", False),
|
||||
("1", True),
|
||||
])
|
||||
def test_bool_std_pinch_enable(val, expected):
|
||||
cfg = VizConfig.from_env({"PINCH_ENABLE": val})
|
||||
assert cfg.pinch_enable is expected
|
||||
|
||||
|
||||
# HAND_SWAP_LR: standard bool, default False
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("0", False),
|
||||
("", False),
|
||||
("false", False),
|
||||
("False", False),
|
||||
("1", True),
|
||||
])
|
||||
def test_bool_std_hand_swap_lr(val, expected):
|
||||
cfg = VizConfig.from_env({"HAND_SWAP_LR": val})
|
||||
assert cfg.hand_swap_lr is expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bool convention: ne0 — != "0" (empty string is True)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("0", False),
|
||||
("1", True),
|
||||
("", True), # ne0: "" != "0" = True
|
||||
("false", True), # ne0: "false" != "0" = True
|
||||
])
|
||||
def test_bool_ne0_concert_mirror(val, expected):
|
||||
cfg = VizConfig.from_env({"CONCERT_MIRROR": val})
|
||||
assert cfg.concert_mirror is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("0", False),
|
||||
("1", True),
|
||||
("", True), # ne0: enabled by default when not "0"
|
||||
])
|
||||
def test_bool_ne0_av_live_mediapipe(val, expected):
|
||||
cfg = VizConfig.from_env({"AV_LIVE_MEDIAPIPE": val})
|
||||
assert cfg.av_live_mediapipe is expected
|
||||
|
||||
|
||||
# When AV_LIVE_MEDIAPIPE is not set, default = True
|
||||
def test_bool_ne0_av_live_unset():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.av_live_mediapipe is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bool convention: eq1 — only "1" is True
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("1", True),
|
||||
("0", False),
|
||||
("true", False), # eq1: only exact "1"
|
||||
("", False),
|
||||
])
|
||||
def test_bool_eq1_icp_fusion(val, expected):
|
||||
cfg = VizConfig.from_env({"ICP_FUSION": val})
|
||||
assert cfg.icp_fusion is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("1", True),
|
||||
("0", False),
|
||||
("true", False),
|
||||
])
|
||||
def test_bool_eq1_multihmr_autocast(val, expected):
|
||||
cfg = VizConfig.from_env({"MULTIHMR_AUTOCAST": val})
|
||||
assert cfg.multihmr_autocast is expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bool convention: flag — in ("1","true","yes","on") → True
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("1", True),
|
||||
("true", True),
|
||||
("yes", True),
|
||||
("on", True),
|
||||
("0", False),
|
||||
("false", False),
|
||||
("no", False),
|
||||
("off", False),
|
||||
("", False),
|
||||
])
|
||||
def test_bool_flag_multihmr_remote_async(val, expected):
|
||||
cfg = VizConfig.from_env({"MULTIHMR_REMOTE_ASYNC": val})
|
||||
assert cfg.multihmr_remote_async is expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Numeric casts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_int_cast():
|
||||
cfg = VizConfig.from_env({"POSE_GHOST_MIN_VISIBLE": "7"})
|
||||
assert cfg.pose_ghost_min_visible == 7
|
||||
|
||||
|
||||
def test_float_cast():
|
||||
cfg = VizConfig.from_env({"POSE_GHOST_MIN_CONF": "0.8"})
|
||||
assert cfg.pose_ghost_min_conf == pytest.approx(0.8)
|
||||
|
||||
|
||||
def test_int_cast_fallback_on_invalid():
|
||||
cfg = VizConfig.from_env({"POSE_GHOST_MIN_VISIBLE": "notanumber"})
|
||||
assert cfg.pose_ghost_min_visible == 10 # falls back to default
|
||||
|
||||
|
||||
def test_float_cast_fallback_on_invalid():
|
||||
cfg = VizConfig.from_env({"POSE_GHOST_MIN_CONF": "bad"})
|
||||
assert cfg.pose_ghost_min_conf == pytest.approx(0.5)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Monkeypatched environ round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_roundtrip_via_monkeypatch(monkeypatch):
|
||||
monkeypatch.setenv("FINGER_PIANO", "1")
|
||||
monkeypatch.setenv("FINGER_STRIKE_VEL", "0.05")
|
||||
monkeypatch.setenv("PINCH_EXT_RATIO", "1.5")
|
||||
monkeypatch.setenv("PINCH_EXT_MIN", "4")
|
||||
monkeypatch.setenv("HAND_SWAP_LR", "1")
|
||||
cfg = VizConfig.from_env()
|
||||
assert cfg.finger_piano is True
|
||||
assert cfg.finger_strike_vel == pytest.approx(0.05)
|
||||
assert cfg.pinch_ext_ratio == pytest.approx(1.5)
|
||||
assert cfg.pinch_ext_min == 4
|
||||
assert cfg.hand_swap_lr is True
|
||||
|
||||
|
||||
def test_roundtrip_uses_live_environ(monkeypatch):
|
||||
"""from_env() without args reads os.environ — picks up monkeypatches."""
|
||||
monkeypatch.setenv("POSE_NMS_IOU", "0.55")
|
||||
cfg = VizConfig.from_env()
|
||||
assert cfg.pose_nms_iou == pytest.approx(0.55)
|
||||
|
||||
|
||||
def test_custom_mapping_does_not_pollute_os_environ(monkeypatch):
|
||||
"""Passing an explicit mapping leaves os.environ untouched."""
|
||||
monkeypatch.delenv("FINGER_PIANO", raising=False)
|
||||
cfg_custom = VizConfig.from_env({"FINGER_PIANO": "1"})
|
||||
cfg_env = VizConfig.from_env()
|
||||
assert cfg_custom.finger_piano is True
|
||||
assert cfg_env.finger_piano is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Part B: MultiWorker with iphone_usb=True does not call landmarker builders
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_iphone_usb_skips_ensure_model():
|
||||
"""_ensure_model must never be called when iphone_usb=True.
|
||||
|
||||
When iphone_usb is set, the entire MediaPipe block (import + model
|
||||
download + landmarker construction) is gated out. We verify by
|
||||
patching _ensure_model to raise on call, then running _run with
|
||||
iphone_usb=True and a fake IphoneUSBSource that fails to start
|
||||
(so _run returns early without entering the capture loop).
|
||||
"""
|
||||
import data_only_viz.multi as multi_mod
|
||||
from data_only_viz.multi import MultiWorker
|
||||
from data_only_viz.state import State
|
||||
|
||||
ensure_calls: list[str] = []
|
||||
|
||||
def fail_ensure(name: str) -> Path:
|
||||
ensure_calls.append(name)
|
||||
raise AssertionError(
|
||||
f"_ensure_model({name!r}) called in iphone_usb mode — "
|
||||
"MediaPipe must not be loaded under --iphone-usb"
|
||||
)
|
||||
|
||||
# Minimal MultiWorker without calling __init__ (avoids ActionHead load)
|
||||
w = object.__new__(MultiWorker)
|
||||
w.iphone_usb = True
|
||||
w._stop = threading.Event()
|
||||
w.state = MagicMock(spec=State)
|
||||
w.state.lock.return_value = MagicMock(__enter__=MagicMock(return_value=None),
|
||||
__exit__=MagicMock(return_value=False))
|
||||
w.period = 0.033
|
||||
w.num_persons = 1
|
||||
w.min_conf = 0.4
|
||||
w.camera_index = 0
|
||||
|
||||
# fake_cap.start() returns False → _run returns after source setup
|
||||
fake_cap = MagicMock()
|
||||
fake_cap.start.return_value = False
|
||||
|
||||
with patch.object(multi_mod, "_ensure_model", fail_ensure):
|
||||
# Patch the inline `from .iphone_usb_source import IphoneUSBSource`
|
||||
# by pre-populating sys.modules so the local import resolves to our stub.
|
||||
import sys
|
||||
fake_src_mod = MagicMock()
|
||||
fake_src_mod.IphoneUSBSource.return_value = fake_cap
|
||||
with patch.dict(sys.modules,
|
||||
{"data_only_viz.iphone_usb_source": fake_src_mod}):
|
||||
multi_mod._run = None # guard against any indirect recursion
|
||||
try:
|
||||
w._run()
|
||||
except Exception:
|
||||
pass # early return from failed cap.start() is expected
|
||||
|
||||
assert ensure_calls == [], (
|
||||
f"_ensure_model was called under iphone_usb=True: {ensure_calls}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("val,expected", [
|
||||
("0", False),
|
||||
("1", True),
|
||||
("", True), # ne0: "" != "0" = True (drift to _bool_std would break this)
|
||||
])
|
||||
def test_bool_ne0_mesh_rig(val, expected):
|
||||
cfg = VizConfig.from_env({"MESH_RIG": val})
|
||||
assert cfg.mesh_rig is expected
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Item 2: MultiWorker runs at 30 Hz under iphone-usb
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_multiworker_period_formula_for_iphone_usb():
|
||||
"""MultiWorker stores period = 1 / max(1, target_fps).
|
||||
|
||||
Frame-based gesture calibrations (debounce/hold/grace/persist) assume
|
||||
30 Hz. When --iphone-usb is active (camera decode + ARKit conversion,
|
||||
no MediaPipe on the Mac), target_fps=30 is passed; webcam keeps 18.
|
||||
This test verifies the __init__ period arithmetic directly, sidestepping
|
||||
heavy ML/AppKit imports that are unavailable in CI.
|
||||
"""
|
||||
from data_only_viz.multi import MultiWorker
|
||||
import math
|
||||
# Verify the formula for both paths
|
||||
for target_fps, expected in [(30.0, 1.0 / 30.0), (18.0, 1.0 / 18.0)]:
|
||||
# Same expression as MultiWorker.__init__: self.period = 1.0 / max(1.0, target_fps)
|
||||
period = 1.0 / max(1.0, target_fps)
|
||||
assert math.isclose(period, expected, rel_tol=1e-6), (
|
||||
f"target_fps={target_fps}: expected period {expected:.6f}, got {period:.6f}"
|
||||
)
|
||||
# Verify the constructor signature accepts the kwarg (compilation check)
|
||||
import inspect
|
||||
sig = inspect.signature(MultiWorker.__init__)
|
||||
assert "target_fps" in sig.parameters, (
|
||||
"MultiWorker.__init__ must accept target_fps kwarg"
|
||||
)
|
||||
|
||||
|
||||
def test_hand_face_min_default():
|
||||
"""VizConfig.from_env({}) exposes hand_face_min with a default of 0.5."""
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.hand_face_min == 0.5
|
||||
|
||||
|
||||
def test_hand_face_min_env_override():
|
||||
"""HAND_FACE_MIN env var is parsed as float and stored in hand_face_min."""
|
||||
cfg = VizConfig.from_env({"HAND_FACE_MIN": "0.3"})
|
||||
assert cfg.hand_face_min == pytest.approx(0.3)
|
||||
|
||||
|
||||
def test_viz_source_keys_default_off():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.viz_source_keys is False
|
||||
assert VizConfig.from_env({"VIZ_SOURCE_KEYS": "1"}).viz_source_keys is True
|
||||
|
||||
|
||||
def test_viz_audio_keys_default_off():
|
||||
cfg = VizConfig.from_env({})
|
||||
assert cfg.viz_audio_keys is False
|
||||
assert VizConfig.from_env({"VIZ_AUDIO_KEYS": "1"}).viz_audio_keys is True
|
||||
|
||||
|
||||
def test_viz_default_mode():
|
||||
assert VizConfig.from_env({}).viz_default_mode == "auto"
|
||||
assert VizConfig.from_env(
|
||||
{"VIZ_DEFAULT_MODE": "voronoi"}).viz_default_mode == "voronoi"
|
||||
|
||||
|
||||
def test_skel_line_width_default():
|
||||
assert VizConfig.from_env({}).skel_line_width == 4
|
||||
assert VizConfig.from_env({"SKEL_LINE_WIDTH": "1"}).skel_line_width == 1
|
||||
@@ -0,0 +1,365 @@
|
||||
"""Tests for FingerStrikeDetector (air-piano strike detection)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from data_only_viz.finger_strike import (
|
||||
FingerStrikeDetector,
|
||||
PinchDetector,
|
||||
StrikeEvent,
|
||||
FINGERTIPS,
|
||||
FINGER_BASES,
|
||||
)
|
||||
|
||||
|
||||
def _hand(tip_y_by_finger: dict[int, float], base_y: float = 0.4,
|
||||
cx: float = 0.3) -> list[list[float]]:
|
||||
"""Build a 21-landmark hand. Every base knuckle sits at base_y; each
|
||||
fingertip sits at base_y unless overridden in tip_y_by_finger (keyed by
|
||||
finger 0..4). x is set near cx so L/R slotting is deterministic."""
|
||||
lm = [[cx, base_y, 0.0] for _ in range(21)]
|
||||
for f, base_idx in enumerate(FINGER_BASES):
|
||||
lm[base_idx] = [cx, base_y, 0.0]
|
||||
for f, tip_idx in enumerate(FINGERTIPS):
|
||||
ty = tip_y_by_finger.get(f, base_y)
|
||||
lm[tip_idx] = [cx, ty, 0.0]
|
||||
return lm
|
||||
|
||||
|
||||
def test_downward_spike_fires_exactly_one_strike():
|
||||
det = FingerStrikeDetector(vel_thresh=0.02, refractory_ms=120.0)
|
||||
# frame 0: neutral (primes prev), index tip level with base
|
||||
det.step([_hand({1: 0.40}), None], t_now=0.00)
|
||||
# frame 1: index tip drops 0.06 below -> downward velocity 0.06 > thresh
|
||||
e1 = det.step([_hand({1: 0.46}), None], t_now=0.04)
|
||||
# frame 2: tip stays down -> velocity ~0, must NOT refire
|
||||
e2 = det.step([_hand({1: 0.46}), None], t_now=0.08)
|
||||
strikes = e1 + e2
|
||||
assert len(strikes) == 1
|
||||
assert strikes[0].finger == 1
|
||||
assert strikes[0].hand == 0
|
||||
assert strikes[0].strike_speed > 0.0
|
||||
|
||||
|
||||
def test_whole_hand_translation_does_not_fire():
|
||||
det = FingerStrikeDetector(vel_thresh=0.02)
|
||||
out = []
|
||||
# tip and base move down together each frame -> relative y constant
|
||||
for i, by in enumerate((0.40, 0.50, 0.60, 0.70)):
|
||||
out += det.step([_hand({1: by}, base_y=by), None], t_now=i * 0.04)
|
||||
assert out == []
|
||||
|
||||
|
||||
def test_refractory_blocks_second_strike():
|
||||
det = FingerStrikeDetector(vel_thresh=0.02, refractory_ms=120.0)
|
||||
det.step([_hand({1: 0.40}), None], t_now=0.00) # prime
|
||||
a = det.step([_hand({1: 0.46}), None], t_now=0.02) # strike 1
|
||||
det.step([_hand({1: 0.40}), None], t_now=0.04) # lift -> rearm
|
||||
b = det.step([_hand({1: 0.46}), None], t_now=0.06) # within 120 ms -> blocked
|
||||
assert len(a) == 1
|
||||
assert b == []
|
||||
|
||||
|
||||
def test_strike_speed_scales_with_velocity():
|
||||
soft = FingerStrikeDetector(vel_thresh=0.02, speed_scale=0.10)
|
||||
hard = FingerStrikeDetector(vel_thresh=0.02, speed_scale=0.10)
|
||||
soft.step([_hand({1: 0.40}), None], t_now=0.0)
|
||||
hard.step([_hand({1: 0.40}), None], t_now=0.0)
|
||||
s = soft.step([_hand({1: 0.44}), None], t_now=0.04) # delta 0.04
|
||||
h = hard.step([_hand({1: 0.50}), None], t_now=0.04) # delta 0.10
|
||||
assert h[0].strike_speed > s[0].strike_speed
|
||||
|
||||
|
||||
def _pinch_hand(thumb_xy, index_xy):
|
||||
"""21-kp hand with fixed wrist & middle-MCP (hand size = 0.3). Middle/
|
||||
ring/pinky tips are parked far from the thumb so only the thumb-index
|
||||
pair can pinch."""
|
||||
lm = [[0.5, 0.5, 0.0] for _ in range(21)]
|
||||
lm[0] = [0.5, 0.8, 0.0] # WRIST
|
||||
lm[9] = [0.5, 0.5, 0.0] # MIDDLE_MCP -> size 0.3
|
||||
lm[4] = [thumb_xy[0], thumb_xy[1], 0.0] # THUMB_TIP
|
||||
lm[8] = [index_xy[0], index_xy[1], 0.0] # INDEX_TIP
|
||||
lm[12] = [0.9, 0.5, 0.0] # MIDDLE_TIP (far)
|
||||
lm[16] = [0.9, 0.6, 0.0] # RING_TIP (far)
|
||||
lm[20] = [0.9, 0.7, 0.0] # LITTLE_TIP (far)
|
||||
return lm
|
||||
|
||||
|
||||
_OPEN = ((0.2, 0.5), (0.8, 0.5)) # thumb-index dist 0.6 -> ratio 2.0
|
||||
_PINCH = ((0.5, 0.5), (0.52, 0.5)) # dist 0.02 -> ratio 0.067
|
||||
|
||||
|
||||
def test_pinch_fires_on_thumb_index_contact():
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65)
|
||||
assert det.step([_pinch_hand(*_OPEN), None], 0.0) == []
|
||||
ev = det.step([_pinch_hand(*_PINCH), None], 0.1)
|
||||
assert len(ev) == 1
|
||||
assert ev[0].finger == 1 # index = finger 1
|
||||
assert ev[0].hand == 0
|
||||
|
||||
|
||||
def test_pinch_does_not_refire_while_held():
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0)
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.0)
|
||||
a = det.step([_pinch_hand(*_PINCH), None], 0.1) # fire
|
||||
b = det.step([_pinch_hand(*_PINCH), None], 0.2) # still held -> no refire
|
||||
assert len(a) == 1
|
||||
assert b == []
|
||||
|
||||
|
||||
def test_pinch_rearms_after_release():
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0)
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.0)
|
||||
a = det.step([_pinch_hand(*_PINCH), None], 0.1) # fire
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.2) # release (ratio > off)
|
||||
c = det.step([_pinch_hand(*_PINCH), None], 0.3) # pinch again -> fire
|
||||
assert len(a) == 1
|
||||
assert len(c) == 1
|
||||
|
||||
|
||||
def test_pinch_refractory_blocks():
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=200)
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.0)
|
||||
a = det.step([_pinch_hand(*_PINCH), None], 0.05) # fire t=0.05
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.08) # release
|
||||
b = det.step([_pinch_hand(*_PINCH), None], 0.10) # within 200 ms -> blocked
|
||||
assert len(a) == 1
|
||||
assert b == []
|
||||
|
||||
|
||||
def test_pinch_engage_release_emits_both_edges():
|
||||
"""Engage emits state=1; subsequent open emits state=0 (same finger)."""
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0)
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.0)
|
||||
engage = det.step([_pinch_hand(*_PINCH), None], 0.1) # engage edge
|
||||
release = det.step([_pinch_hand(*_OPEN), None], 0.2) # release edge
|
||||
assert len(engage) == 1 and engage[0].state == 1
|
||||
assert len(release) == 1 and release[0].state == 0
|
||||
assert release[0].hand == 0 and release[0].finger == 1
|
||||
|
||||
|
||||
def test_pinch_hand_disappear_emits_release():
|
||||
"""When an engaged hand disappears, a release edge is synthesised."""
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0)
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.0)
|
||||
engage = det.step([_pinch_hand(*_PINCH), None], 0.1) # engage
|
||||
release = det.step([None, None], 0.2) # hand gone
|
||||
assert len(engage) == 1 and engage[0].state == 1
|
||||
assert len(release) == 1 and release[0].state == 0
|
||||
|
||||
|
||||
def _pinch_hand_multi(thumb_xy, tips):
|
||||
"""21-kp hand (size 0.3); `tips` maps finger 0..3 (index/middle/ring/pinky)
|
||||
-> (x,y) for that fingertip. Unlisted fingertips are parked far from thumb."""
|
||||
lm = [[0.5, 0.5, 0.0] for _ in range(21)]
|
||||
lm[0] = [0.5, 0.8, 0.0] # WRIST
|
||||
lm[9] = [0.5, 0.5, 0.0] # MIDDLE_MCP -> size 0.3
|
||||
lm[4] = [thumb_xy[0], thumb_xy[1], 0.0] # THUMB_TIP
|
||||
far = [(0.9, 0.5), (0.9, 0.6), (0.9, 0.7), (0.9, 0.8)]
|
||||
for i, idx in enumerate((8, 12, 16, 20)): # PINCH_TIPS
|
||||
x, y = tips.get(i, far[i])
|
||||
lm[idx] = [x, y, 0.0]
|
||||
return lm
|
||||
|
||||
|
||||
def test_pinch_closest_finger_wins():
|
||||
# index ratio ~0.067, middle ratio ~0.33 -> margin 0.27 >= 0.20 -> index only.
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0)
|
||||
det.step([_pinch_hand_multi((0.5, 0.5), {}), None], 0.0) # open
|
||||
ev = det.step([_pinch_hand_multi((0.5, 0.5),
|
||||
{0: (0.52, 0.5), 1: (0.60, 0.5)}), None], 0.1)
|
||||
assert len(ev) == 1
|
||||
assert ev[0].finger == 1 and ev[0].state == 1 # index only
|
||||
|
||||
|
||||
def test_pinch_ambiguous_adjacent_fires_nothing():
|
||||
# index ratio ~0.067, middle ratio ~0.10 -> margin 0.033 < 0.20 -> no winner.
|
||||
# (both are below ratio_on, so the OLD per-finger logic would fire both.)
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0)
|
||||
det.step([_pinch_hand_multi((0.5, 0.5), {}), None], 0.0) # open
|
||||
ev = det.step([_pinch_hand_multi((0.5, 0.5),
|
||||
{0: (0.52, 0.5), 1: (0.53, 0.5)}), None], 0.1)
|
||||
assert ev == []
|
||||
|
||||
|
||||
def _relaxed_hand():
|
||||
"""Relaxed/fist hand: thumb touches the index tip, but middle/ring/
|
||||
pinky are curled near the wrist (NOT extended). The runner-up tip is
|
||||
far from the thumb so the closest-wins margin passes -- only the
|
||||
extension gate can reject this hand."""
|
||||
lm = [[0.5, 0.5, 0.0] for _ in range(21)]
|
||||
lm[0] = [0.5, 0.8, 0.0] # WRIST
|
||||
lm[9] = [0.5, 0.5, 0.0] # MIDDLE_MCP -> hand size 0.3
|
||||
lm[4] = [0.45, 0.55, 0.0] # THUMB_TIP
|
||||
lm[8] = [0.47, 0.55, 0.0] # INDEX_TIP: ratio 0.067 (pinch-close)
|
||||
lm[12] = [0.60, 0.62, 0.0] # MIDDLE_TIP curled (ext ~0.69)
|
||||
lm[16] = [0.62, 0.65, 0.0] # RING_TIP curled (ext ~0.64)
|
||||
lm[20] = [0.64, 0.68, 0.0] # LITTLE_TIP curled (ext ~0.62)
|
||||
return lm
|
||||
|
||||
|
||||
def test_relaxed_hand_rejected_by_extension_gate():
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0,
|
||||
ext_min=2)
|
||||
out = []
|
||||
for i in range(10):
|
||||
out += det.step([_relaxed_hand(), None], i * 0.033)
|
||||
assert out == []
|
||||
|
||||
|
||||
def test_open_hand_pinch_passes_extension_gate():
|
||||
# _pinch_hand parks middle/ring/little far from the wrist
|
||||
# (ext 1.67 / 1.49 / 1.37, all >= 1.35) -> 3 extended >= ext_min 2.
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0,
|
||||
ext_min=2)
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.0)
|
||||
ev = det.step([_pinch_hand(*_PINCH), None], 0.1)
|
||||
assert len(ev) == 1 and ev[0].state == 1 and ev[0].finger == 1
|
||||
|
||||
|
||||
def test_extension_gate_defeat_reproduces_old_behavior():
|
||||
# ext_min=0 disables the gate: the relaxed hand fires like today.
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0,
|
||||
ext_min=0)
|
||||
ev = det.step([_relaxed_hand(), None], 0.1)
|
||||
assert len(ev) == 1 and ev[0].state == 1
|
||||
|
||||
|
||||
def test_debounce_delays_engage_to_nth_frame():
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0,
|
||||
debounce_frames=3)
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.00)
|
||||
a = det.step([_pinch_hand(*_PINCH), None], 0.10) # qualifying 1
|
||||
b = det.step([_pinch_hand(*_PINCH), None], 0.13) # qualifying 2
|
||||
c = det.step([_pinch_hand(*_PINCH), None], 0.16) # qualifying 3 -> engage
|
||||
assert a == [] and b == []
|
||||
assert len(c) == 1 and c[0].state == 1 and c[0].finger == 1
|
||||
|
||||
|
||||
def test_debounce_jitter_resets_counter():
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0,
|
||||
debounce_frames=3)
|
||||
det.step([_pinch_hand(*_PINCH), None], 0.00) # qualifying 1
|
||||
det.step([_pinch_hand(*_PINCH), None], 0.03) # qualifying 2
|
||||
det.step([_pinch_hand(*_OPEN), None], 0.06) # jitter -> counter resets
|
||||
a = det.step([_pinch_hand(*_PINCH), None], 0.09) # qualifying 1 again
|
||||
b = det.step([_pinch_hand(*_PINCH), None], 0.12) # qualifying 2
|
||||
assert a == [] and b == []
|
||||
|
||||
|
||||
def test_release_immediate_after_debounced_engage():
|
||||
det = PinchDetector(ratio_on=0.45, ratio_off=0.65, refractory_ms=0,
|
||||
debounce_frames=3)
|
||||
det.step([_pinch_hand(*_PINCH), None], 0.00)
|
||||
det.step([_pinch_hand(*_PINCH), None], 0.03)
|
||||
eng = det.step([_pinch_hand(*_PINCH), None], 0.06) # 3rd frame -> engage
|
||||
rel = det.step([_pinch_hand(*_OPEN), None], 0.09) # very next frame
|
||||
assert len(eng) == 1 and eng[0].state == 1
|
||||
assert len(rel) == 1 and rel[0].state == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FingerStrikeDetector.reset_slot — phantom-strike prevention
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _strike_hand(tip_y: float, base_y: float = 0.4, cx: float = 0.3) -> list:
|
||||
"""Hand where index finger tip_y controls the relative position."""
|
||||
return _hand({1: tip_y}, base_y=base_y, cx=cx)
|
||||
|
||||
|
||||
def test_reset_slot_prevents_phantom_strike_on_reappear():
|
||||
"""After reset_slot, the first real-hand frame primes without firing.
|
||||
|
||||
Scenario: prime detector, hold the last hand (frozen coords),
|
||||
then reset_slot before the real hand returns with a large delta.
|
||||
Expected: no strike on reappear (first frame after reset is a prime).
|
||||
"""
|
||||
det = FingerStrikeDetector(vel_thresh=0.02, refractory_ms=0)
|
||||
|
||||
# Prime: index tip at 0.40
|
||||
det.step([_strike_hand(0.40), None], t_now=0.00)
|
||||
|
||||
# "Held" frames: stabilizer would replay the same hand — same coords,
|
||||
# vel=0, no strike, but prev_rel stays at 0.40.
|
||||
det.step([_strike_hand(0.40), None], t_now=0.03)
|
||||
det.step([_strike_hand(0.40), None], t_now=0.06)
|
||||
|
||||
# Simulate held -> real transition: reset before consuming new position.
|
||||
det.reset_slot(0)
|
||||
|
||||
# Real hand reappears with a large downward delta (0.40 -> 0.70 = 0.30 >> thresh).
|
||||
events = det.step([_strike_hand(0.70), None], t_now=0.09)
|
||||
assert events == [], (
|
||||
"after reset_slot, first frame should prime without firing "
|
||||
f"(got {events})"
|
||||
)
|
||||
|
||||
|
||||
def test_without_reset_big_delta_fires_strike():
|
||||
"""Control: WITHOUT reset_slot, the same scenario fires a phantom strike."""
|
||||
det = FingerStrikeDetector(vel_thresh=0.02, refractory_ms=0)
|
||||
|
||||
det.step([_strike_hand(0.40), None], t_now=0.00) # prime
|
||||
det.step([_strike_hand(0.40), None], t_now=0.03) # held
|
||||
det.step([_strike_hand(0.40), None], t_now=0.06) # held
|
||||
|
||||
# No reset — prev_rel is still 0.40 → delta 0.30 > thresh → fires
|
||||
events = det.step([_strike_hand(0.70), None], t_now=0.09)
|
||||
assert len(events) > 0, "without reset_slot, phantom strike must fire (control)"
|
||||
|
||||
|
||||
def test_reset_slot_clears_all_five_fingers():
|
||||
"""reset_slot clears every finger state for the specified slot."""
|
||||
det = FingerStrikeDetector(vel_thresh=0.02, refractory_ms=0)
|
||||
# Prime all 5 fingers by stepping a neutral hand
|
||||
neutral = _hand({f: 0.40 for f in range(5)}, base_y=0.40)
|
||||
det.step([neutral, None], t_now=0.00)
|
||||
# All prev_rel should be non-None now
|
||||
for f in range(5):
|
||||
assert det._state[0][f].prev_rel is not None
|
||||
|
||||
det.reset_slot(0)
|
||||
|
||||
for f in range(5):
|
||||
assert det._state[0][f].prev_rel is None, f"finger {f} prev_rel not cleared"
|
||||
assert det._state[0][f].armed is True, f"finger {f} not re-armed"
|
||||
|
||||
|
||||
def test_reset_slot_does_not_affect_other_slot():
|
||||
"""Resetting slot 0 leaves slot 1 state intact."""
|
||||
det = FingerStrikeDetector(vel_thresh=0.02, refractory_ms=0)
|
||||
h = _hand({1: 0.40})
|
||||
det.step([h, h], t_now=0.00) # prime both slots
|
||||
# slot 1 finger 1 should have a non-None prev_rel
|
||||
assert det._state[1][1].prev_rel is not None
|
||||
|
||||
det.reset_slot(0)
|
||||
|
||||
# slot 1 unchanged
|
||||
assert det._state[1][1].prev_rel is not None, "slot 1 must be unaffected"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PinchDetector.engaged_slots
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_engaged_slots_both_false_initially():
|
||||
"""Fresh PinchDetector: no pinch engaged in either slot."""
|
||||
det = PinchDetector()
|
||||
assert det.engaged_slots() == (False, False)
|
||||
|
||||
|
||||
def test_engaged_slots_true_for_engaged_slot():
|
||||
"""engaged_slots() reflects per-slot engaged state correctly."""
|
||||
det = PinchDetector()
|
||||
det._state[0][0].engaged = True # slot 0, index finger
|
||||
s0, s1 = det.engaged_slots()
|
||||
assert s0 is True, "slot 0 has an engaged finger"
|
||||
assert s1 is False, "slot 1 has no engaged finger"
|
||||
|
||||
|
||||
def test_engaged_slots_false_after_release():
|
||||
"""Clearing engaged flag → engaged_slots returns False for that slot."""
|
||||
det = PinchDetector()
|
||||
det._state[1][2].engaged = True # slot 1, ring finger
|
||||
assert det.engaged_slots()[1] is True
|
||||
det._state[1][2].engaged = False
|
||||
assert det.engaged_slots() == (False, False)
|
||||
@@ -0,0 +1,772 @@
|
||||
"""Unit tests for hand_display geometry helpers.
|
||||
|
||||
All tests use tiny synthetic hands — no Metal, no pyobjc, no hardware required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
|
||||
from data_only_viz.hand_display import (
|
||||
HandPersistenceGate,
|
||||
arkit_2d_fresh,
|
||||
gauge_segments,
|
||||
gesture_quality,
|
||||
hand_facing,
|
||||
hand_plausible,
|
||||
hand_size,
|
||||
segment_ok,
|
||||
panel_rect,
|
||||
panel_segments,
|
||||
panel_frame,
|
||||
GAUGE_GAP,
|
||||
GAUGE_TICK_HALF,
|
||||
PANEL_SIDE,
|
||||
PANEL_MARGIN,
|
||||
PANEL_INNER,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal keypoint stub
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class Kp:
|
||||
x: float = 0.0
|
||||
y: float = 0.0
|
||||
z: float = 0.0
|
||||
c: float = 1.0
|
||||
|
||||
|
||||
def _make_hand(
|
||||
wrist_x: float = 0.5,
|
||||
wrist_y: float = 0.5,
|
||||
size: float = 0.1,
|
||||
c: float = 1.0,
|
||||
n: int = 21,
|
||||
) -> list[Kp]:
|
||||
"""Build a synthetic 21-landmark hand.
|
||||
|
||||
Landmark 0 = wrist, landmark 9 = middle-MCP at distance `size` along x.
|
||||
All other landmarks are scattered uniformly between wrist and MCP.
|
||||
"""
|
||||
kps: list[Kp] = []
|
||||
for i in range(n):
|
||||
# Spread landmarks roughly from wrist (0) to slightly past MCP (9)
|
||||
t = i / max(n - 1, 1)
|
||||
kps.append(Kp(x=wrist_x + t * size, y=wrist_y, z=0.0, c=c))
|
||||
# Ensure wrist and MCP are exactly placed so hand_size is deterministic
|
||||
kps[0] = Kp(x=wrist_x, y=wrist_y, z=0.0, c=c)
|
||||
kps[9] = Kp(x=wrist_x + size, y=wrist_y, z=0.0, c=c)
|
||||
return kps
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# hand_size
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_hand_size_simple():
|
||||
kp = _make_hand(wrist_x=0.3, wrist_y=0.4, size=0.15)
|
||||
assert math.isclose(hand_size(kp), 0.15, rel_tol=1e-6)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# hand_plausible — acceptance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_normal_mediapipe_hand_passes():
|
||||
"""CRITICAL: c=1.0 MediaPipe hand at normal size must pass gate unchanged."""
|
||||
kp = _make_hand(size=0.12, c=1.0)
|
||||
assert hand_plausible(kp) is True
|
||||
|
||||
|
||||
def test_plausible_boundary_size_min_exact():
|
||||
"""Size exactly at size_min (0.02) should be accepted."""
|
||||
kp = _make_hand(size=0.02, c=1.0)
|
||||
assert hand_plausible(kp) is True
|
||||
|
||||
|
||||
def test_plausible_boundary_size_max_exact():
|
||||
"""Size exactly at size_max (0.5) should be accepted."""
|
||||
kp = _make_hand(size=0.5, c=1.0)
|
||||
assert hand_plausible(kp) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# hand_plausible — rejections
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_fewer_than_21_landmarks_rejected():
|
||||
kp = _make_hand(size=0.1, c=1.0, n=20)
|
||||
assert hand_plausible(kp) is False
|
||||
|
||||
|
||||
def test_tiny_hand_rejected():
|
||||
"""size=0.005 < size_min=0.02 → reject."""
|
||||
kp = _make_hand(size=0.005, c=1.0)
|
||||
assert hand_plausible(kp) is False
|
||||
|
||||
|
||||
def test_huge_hand_rejected():
|
||||
"""size=0.8 > size_max=0.5 → reject."""
|
||||
kp = _make_hand(size=0.8, c=1.0)
|
||||
assert hand_plausible(kp) is False
|
||||
|
||||
|
||||
def test_low_conf_hand_rejected():
|
||||
"""mean(c)=0.1 < conf_min=0.3 → reject."""
|
||||
kp = _make_hand(size=0.1, c=0.1)
|
||||
assert hand_plausible(kp) is False
|
||||
|
||||
|
||||
def test_borderline_conf_just_below_rejected():
|
||||
"""mean(c) just under conf_min (0.29) → reject."""
|
||||
kp = _make_hand(size=0.1, c=0.29)
|
||||
assert hand_plausible(kp) is False
|
||||
|
||||
|
||||
def test_borderline_conf_at_threshold_accepted():
|
||||
"""mean(c) = conf_min (0.3) → accept."""
|
||||
kp = _make_hand(size=0.1, c=0.3)
|
||||
assert hand_plausible(kp) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# segment_ok
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_normal_bone_passes():
|
||||
"""A normal short bone at c=1.0 must pass."""
|
||||
size = 0.1
|
||||
A = Kp(x=0.5, y=0.5, c=1.0)
|
||||
B = Kp(x=0.55, y=0.5, c=1.0) # bone_len = 0.05 < 1.2 * 0.1 = 0.12
|
||||
assert segment_ok(A, B, size) is True
|
||||
|
||||
|
||||
def test_spike_segment_rejected():
|
||||
"""Bone length 2× hand size exceeds max_bone_ratio=1.2 → reject."""
|
||||
size = 0.1
|
||||
A = Kp(x=0.5, y=0.5, c=1.0)
|
||||
B = Kp(x=0.5 + 2 * size, y=0.5, c=1.0) # bone_len = 0.2 > 0.12
|
||||
assert segment_ok(A, B, size) is False
|
||||
|
||||
|
||||
def test_low_conf_endpoint_a_rejected():
|
||||
"""Endpoint A has c=0.1 < conf_min=0.3 → reject."""
|
||||
size = 0.1
|
||||
A = Kp(x=0.5, y=0.5, c=0.1)
|
||||
B = Kp(x=0.55, y=0.5, c=1.0)
|
||||
assert segment_ok(A, B, size) is False
|
||||
|
||||
|
||||
def test_low_conf_endpoint_b_rejected():
|
||||
"""Endpoint B has c=0.2 < conf_min=0.3 → reject."""
|
||||
size = 0.1
|
||||
A = Kp(x=0.5, y=0.5, c=1.0)
|
||||
B = Kp(x=0.55, y=0.5, c=0.2)
|
||||
assert segment_ok(A, B, size) is False
|
||||
|
||||
|
||||
def test_bone_exactly_at_ratio_limit_accepted():
|
||||
"""Bone length exactly max_bone_ratio * size should be accepted (<=)."""
|
||||
size = 0.1
|
||||
bone_len = 1.2 * size # exactly at limit
|
||||
A = Kp(x=0.5, y=0.5, c=1.0)
|
||||
B = Kp(x=0.5 + bone_len, y=0.5, c=1.0)
|
||||
# bone_len == 1.2 * size → not strictly greater, should pass
|
||||
assert segment_ok(A, B, size) is True
|
||||
|
||||
|
||||
def test_mediapipe_hand_all_bones_pass():
|
||||
"""CRITICAL invariant: for c=1.0 normal-size MediaPipe hand, ALL bones pass
|
||||
segment_ok (none treated as spikes given hand_size=0.12)."""
|
||||
kp = _make_hand(size=0.12, c=1.0)
|
||||
size = hand_size(kp)
|
||||
assert hand_plausible(kp) is True
|
||||
# All adjacent pairs in [0..20] should pass segment_ok
|
||||
# (spacing between consecutive kps = size/20 << 1.2 * size)
|
||||
failures = []
|
||||
for i in range(len(kp) - 1):
|
||||
if not segment_ok(kp[i], kp[i + 1], size):
|
||||
failures.append(i)
|
||||
assert failures == [], f"Segments failed segment_ok: {failures}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2D hand helper: spans a proper bbox with both x and y extent
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_hand_2d(
|
||||
cx: float = 0.5,
|
||||
cy: float = 0.5,
|
||||
half_w: float = 0.05,
|
||||
half_h: float = 0.05,
|
||||
c: float = 1.0,
|
||||
) -> list[Kp]:
|
||||
"""21 landmarks filling bbox [cx±half_w, cy±half_h].
|
||||
|
||||
kp[0] = wrist at (cx-half_w, cy+half_h) — bottom-left of bbox
|
||||
kp[9] = MCP at (cx+half_w, cy-half_h) — top-right of bbox
|
||||
All other kp are spread uniformly across the bbox via a diagonal.
|
||||
hand_size = dist(kp[0], kp[9]) = 2 * sqrt(half_w² + half_h²).
|
||||
"""
|
||||
n = 21
|
||||
kps: list[Kp] = []
|
||||
for i in range(n):
|
||||
t = i / (n - 1)
|
||||
kps.append(Kp(
|
||||
x=cx - half_w + t * 2 * half_w,
|
||||
y=cy - half_h + t * 2 * half_h,
|
||||
z=0.0,
|
||||
c=c,
|
||||
))
|
||||
# Override wrist and MCP so bbox spans full [cx-half_w..cx+half_w, cy-half_h..cy+half_h]
|
||||
kps[0] = Kp(x=cx - half_w, y=cy + half_h, z=0.0, c=c) # bottom-left
|
||||
kps[9] = Kp(x=cx + half_w, y=cy - half_h, z=0.0, c=c) # top-right
|
||||
return kps
|
||||
|
||||
|
||||
# Minimal bone list for panel tests: wrist(0) → middle-MCP(9)
|
||||
_SIMPLE_BONES: list[tuple[int, int]] = [(0, 9)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# panel_rect
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_panel_rect_left_x0_anchor():
|
||||
"""Left panel x0 == PANEL_MARGIN."""
|
||||
x0, _y0, _x1, _y1 = panel_rect("left", aspect=1.0)
|
||||
assert math.isclose(x0, PANEL_MARGIN, rel_tol=1e-6)
|
||||
|
||||
|
||||
def test_panel_rect_right_x1_anchor():
|
||||
"""Right panel x1 == 1 - PANEL_MARGIN."""
|
||||
_x0, _y0, x1, _y1 = panel_rect("right", aspect=1.0)
|
||||
assert math.isclose(x1, 1.0 - PANEL_MARGIN, rel_tol=1e-6)
|
||||
|
||||
|
||||
def test_panel_rect_vertically_centered():
|
||||
"""Panel vertical center is at 0.5."""
|
||||
_x0, y0, _x1, y1 = panel_rect("left", aspect=1.0)
|
||||
assert math.isclose((y0 + y1) / 2.0, 0.5, rel_tol=1e-6)
|
||||
|
||||
|
||||
def test_panel_rect_square_in_pixels_landscape():
|
||||
"""With 16:9 aspect ratio, normalized width × aspect == normalized height."""
|
||||
aspect = 16.0 / 9.0
|
||||
x0, y0, x1, y1 = panel_rect("left", aspect=aspect)
|
||||
norm_w = x1 - x0
|
||||
norm_h = y1 - y0
|
||||
# pixel_w = norm_w * view_w = norm_w * aspect * view_h == norm_h * view_h = pixel_h
|
||||
assert math.isclose(norm_w * aspect, norm_h, rel_tol=1e-6)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# panel_segments
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_panel_segments_implausible_returns_empty():
|
||||
"""hand_plausible fails → panel_segments returns []."""
|
||||
kp = _make_hand(size=0.005, c=1.0) # size < size_min=0.02 → implausible
|
||||
result = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_panel_segments_inside_inner_rect():
|
||||
"""All output segment endpoints lie inside the inner panel rect."""
|
||||
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=0.04, half_h=0.08, c=1.0)
|
||||
x0, y0, x1, y1 = panel_rect("left", aspect=1.0)
|
||||
pw, ph = x1 - x0, y1 - y0
|
||||
ix0 = x0 + PANEL_INNER * pw
|
||||
ix1 = x1 - PANEL_INNER * pw
|
||||
iy0 = y0 + PANEL_INNER * ph
|
||||
iy1 = y1 - PANEL_INNER * ph
|
||||
segs = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
|
||||
assert segs, "expected at least one segment"
|
||||
eps = 1e-9
|
||||
for ax, ay, bx, by in segs:
|
||||
assert ix0 - eps <= ax <= ix1 + eps, f"ax={ax} outside [{ix0}, {ix1}]"
|
||||
assert ix0 - eps <= bx <= ix1 + eps, f"bx={bx} outside [{ix0}, {ix1}]"
|
||||
assert iy0 - eps <= ay <= iy1 + eps, f"ay={ay} outside [{iy0}, {iy1}]"
|
||||
assert iy0 - eps <= by <= iy1 + eps, f"by={by} outside [{iy0}, {iy1}]"
|
||||
|
||||
|
||||
def test_panel_segments_aspect_preserved():
|
||||
"""A hand 2× taller than wide in pixel space maps to 2× taller than wide in panel.
|
||||
|
||||
With aspect=1.0 (square screen), pixel ratio == normalized ratio.
|
||||
The hand has half_h = 2*half_w, so pixel bbox is 2× taller than wide.
|
||||
After uniform pixel-space scaling the output segment's |Δy|/|Δx| must also be 2.
|
||||
"""
|
||||
half_w, half_h = 0.04, 0.08 # kh=0.16, kw=0.08 → ratio 2
|
||||
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=half_w, half_h=half_h, c=1.0)
|
||||
segs = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
|
||||
assert len(segs) == 1, f"expected 1 segment, got {len(segs)}"
|
||||
ax, ay, bx, by = segs[0]
|
||||
dx = abs(bx - ax)
|
||||
dy = abs(by - ay)
|
||||
assert dx > 1e-9 and dy > 1e-9, "segment must have both x and y extent"
|
||||
ratio = dy / dx # pixel ratio (= normalized ratio at aspect=1)
|
||||
assert math.isclose(ratio, 2.0, rel_tol=1e-3), f"expected pixel ratio 2.0, got {ratio}"
|
||||
|
||||
|
||||
def test_panel_segments_mirror_flips_x():
|
||||
"""mirror=True flips X within the panel relative to mirror=False."""
|
||||
kp = _make_hand_2d(cx=0.5, cy=0.5, half_w=0.04, half_h=0.04, c=1.0)
|
||||
# kp[0] at bottom-left (kx_min), kp[9] at top-right (kx_max)
|
||||
segs_normal = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=False)
|
||||
segs_mirror = panel_segments(kp, "left", _SIMPLE_BONES, aspect=1.0, mirror=True)
|
||||
assert len(segs_normal) == 1
|
||||
assert len(segs_mirror) == 1
|
||||
ax0, _ay0, bx0, _by0 = segs_normal[0]
|
||||
ax1, _ay1, bx1, _by1 = segs_mirror[0]
|
||||
# kp[0].x < kp[9].x → without mirror: ax < bx; with mirror: ax > bx
|
||||
assert ax0 < bx0, "without mirror: wrist (kx_min) should project left of MCP (kx_max)"
|
||||
assert ax1 > bx1, "with mirror: X is flipped so wrist projects right of MCP"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# panel_frame
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_panel_frame_returns_four_segments():
|
||||
"""panel_frame returns exactly 4 segments."""
|
||||
segs = panel_frame("left", aspect=1.0)
|
||||
assert len(segs) == 4
|
||||
|
||||
|
||||
def test_panel_frame_traces_rect():
|
||||
"""Every endpoint of each frame segment is a corner of the panel rect."""
|
||||
aspect = 1.0
|
||||
x0, y0, x1, y1 = panel_rect("left", aspect)
|
||||
corners = [(x0, y0), (x0, y1), (x1, y0), (x1, y1)]
|
||||
segs = panel_frame("left", aspect)
|
||||
for ax, ay, bx, by in segs:
|
||||
for ex, ey in [(ax, ay), (bx, by)]:
|
||||
on_corner = any(
|
||||
math.isclose(ex, cx, abs_tol=1e-9) and math.isclose(ey, cy, abs_tol=1e-9)
|
||||
for cx, cy in corners
|
||||
)
|
||||
assert on_corner, f"Endpoint ({ex:.4f}, {ey:.4f}) not a corner of panel rect"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# arkit_2d_fresh
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_arkit_2d_fresh_empty_dict_is_false() -> None:
|
||||
"""No pids → never fresh."""
|
||||
assert arkit_2d_fresh({}, now=1000.0) is False
|
||||
|
||||
|
||||
def test_arkit_2d_fresh_single_fresh_pid() -> None:
|
||||
"""A timestamp 0.5 s old with max_age=1.0 → fresh."""
|
||||
ts_by_pid = {0: 999.5}
|
||||
assert arkit_2d_fresh(ts_by_pid, now=1000.0) is True
|
||||
|
||||
|
||||
def test_arkit_2d_fresh_single_stale_pid() -> None:
|
||||
"""A timestamp 2 s old with max_age=1.0 → stale."""
|
||||
ts_by_pid = {0: 998.0}
|
||||
assert arkit_2d_fresh(ts_by_pid, now=1000.0) is False
|
||||
|
||||
|
||||
def test_arkit_2d_fresh_exactly_at_max_age_is_stale() -> None:
|
||||
"""age == max_age is NOT fresh (strict less-than)."""
|
||||
ts_by_pid = {0: 999.0}
|
||||
assert arkit_2d_fresh(ts_by_pid, now=1000.0, max_age=1.0) is False
|
||||
|
||||
|
||||
def test_arkit_2d_fresh_mix_stale_and_fresh() -> None:
|
||||
"""At least one fresh pid makes the whole dict fresh."""
|
||||
ts_by_pid = {0: 990.0, 1: 999.8} # pid 0 stale, pid 1 fresh
|
||||
assert arkit_2d_fresh(ts_by_pid, now=1000.0) is True
|
||||
|
||||
|
||||
def test_arkit_2d_fresh_all_pids_stale() -> None:
|
||||
"""All pids stale → False."""
|
||||
ts_by_pid = {0: 990.0, 1: 980.0}
|
||||
assert arkit_2d_fresh(ts_by_pid, now=1000.0) is False
|
||||
|
||||
|
||||
def test_arkit_2d_fresh_custom_max_age() -> None:
|
||||
"""max_age=0.5 makes a 0.4 s old timestamp fresh."""
|
||||
ts_by_pid = {5: 999.6}
|
||||
assert arkit_2d_fresh(ts_by_pid, now=1000.0, max_age=0.5) is True
|
||||
|
||||
|
||||
def test_arkit_2d_fresh_custom_max_age_stale() -> None:
|
||||
"""max_age=0.5 makes a 0.6 s old timestamp stale."""
|
||||
ts_by_pid = {5: 999.4}
|
||||
assert arkit_2d_fresh(ts_by_pid, now=1000.0, max_age=0.5) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# hand_plausible — wrist out-of-frame rejection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_plausible_wrist_out_of_frame_x_neg_rejected():
|
||||
"""Wrist x < -0.1 → ghost anchor, reject."""
|
||||
kp = _make_hand(wrist_x=-0.2, wrist_y=0.5, size=0.12, c=1.0)
|
||||
assert hand_plausible(kp) is False
|
||||
|
||||
|
||||
def test_plausible_wrist_out_of_frame_x_pos_rejected():
|
||||
"""Wrist x > 1.1 → ghost anchor, reject."""
|
||||
kp = _make_hand(wrist_x=1.2, wrist_y=0.5, size=0.12, c=1.0)
|
||||
assert hand_plausible(kp) is False
|
||||
|
||||
|
||||
def test_plausible_wrist_out_of_frame_y_rejected():
|
||||
"""Wrist y < -0.1 → ghost anchor, reject."""
|
||||
kp = _make_hand(wrist_x=0.5, wrist_y=-0.15, size=0.12, c=1.0)
|
||||
assert hand_plausible(kp) is False
|
||||
|
||||
|
||||
def test_plausible_wrist_at_exact_boundary_accepted():
|
||||
"""Wrist at x=-0.1 is at the boundary — still accepted (inclusive)."""
|
||||
kp = _make_hand(wrist_x=-0.1, wrist_y=0.5, size=0.12, c=1.0)
|
||||
assert hand_plausible(kp) is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HandPersistenceGate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_gate_ghost_two_frames_never_draws():
|
||||
"""A hand appearing only 2 times is never drawn (min_frames=3)."""
|
||||
gate = HandPersistenceGate(min_frames=3)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
assert gate.step([hand]) == [False] # frame 1: count=1
|
||||
assert gate.step([hand]) == [False] # frame 2: count=2
|
||||
|
||||
|
||||
def test_gate_steady_hand_appears_at_third_frame():
|
||||
"""A steady hand becomes drawable from the 3rd consecutive frame."""
|
||||
gate = HandPersistenceGate(min_frames=3)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
assert gate.step([hand]) == [False] # frame 1
|
||||
assert gate.step([hand]) == [False] # frame 2
|
||||
assert gate.step([hand]) == [True] # frame 3: count=3 >= min_frames
|
||||
assert gate.step([hand]) == [True] # frame 4: count=4, stays drawable
|
||||
|
||||
|
||||
def test_gate_teleporting_hand_never_draws():
|
||||
"""A hand that jumps >radius every frame never accumulates a track."""
|
||||
gate = HandPersistenceGate(min_frames=3, radius=0.15)
|
||||
for px in (0.1, 0.5, 0.9): # each jump is 0.4 > radius=0.15
|
||||
hand = _make_hand(wrist_x=px, wrist_y=0.5)
|
||||
assert gate.step([hand]) == [False]
|
||||
|
||||
|
||||
def test_gate_two_steady_hands_both_drawable():
|
||||
"""Two hands that remain steady both become drawable at frame 3."""
|
||||
gate = HandPersistenceGate(min_frames=3)
|
||||
lh = _make_hand(wrist_x=0.3, wrist_y=0.5)
|
||||
rh = _make_hand(wrist_x=0.7, wrist_y=0.5)
|
||||
for _ in range(2):
|
||||
gate.step([lh, rh])
|
||||
result = gate.step([lh, rh])
|
||||
assert result == [True, True]
|
||||
|
||||
|
||||
def test_gate_min_frames_one_immediate():
|
||||
"""min_frames=1 makes any hand drawable on first appearance."""
|
||||
gate = HandPersistenceGate(min_frames=1)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
assert gate.step([hand]) == [True]
|
||||
|
||||
|
||||
def test_gate_track_resets_after_absence():
|
||||
"""A track that disappears for one frame must re-accumulate from scratch."""
|
||||
gate = HandPersistenceGate(min_frames=3)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
gate.step([hand])
|
||||
gate.step([hand]) # count=2, not yet drawable
|
||||
gate.step([]) # hand absent → track dropped
|
||||
# Reappears: back to count=1
|
||||
assert gate.step([hand]) == [False]
|
||||
|
||||
|
||||
def test_gate_empty_input_returns_empty():
|
||||
"""step([]) always returns []."""
|
||||
gate = HandPersistenceGate()
|
||||
assert gate.step([]) == []
|
||||
|
||||
|
||||
def test_gate_greedy_no_double_assign():
|
||||
"""Two hands close together: each matches its own track, not the same one."""
|
||||
gate = HandPersistenceGate(min_frames=2, radius=0.15)
|
||||
h1 = _make_hand(wrist_x=0.3, wrist_y=0.5)
|
||||
h2 = _make_hand(wrist_x=0.4, wrist_y=0.5) # 0.1 apart, both within radius
|
||||
gate.step([h1, h2])
|
||||
result = gate.step([h1, h2])
|
||||
# Both should have count=2 >= min_frames=2
|
||||
assert result == [True, True]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HandPersistenceGate — grace frames
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_gate_grace_zero_is_strict():
|
||||
"""grace=0 restores original strict behavior (1 miss drops the track)."""
|
||||
gate = HandPersistenceGate(min_frames=3, grace=0)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
for _ in range(3):
|
||||
gate.step([hand]) # establish count=3
|
||||
gate.step([]) # miss=1 > grace=0 → dropped
|
||||
assert gate.step([hand]) == [False] # fresh count=1
|
||||
|
||||
|
||||
def test_gate_grace_preserves_track_across_one_miss():
|
||||
"""With grace=2, a 1-frame hole preserves the established count."""
|
||||
gate = HandPersistenceGate(min_frames=3, grace=2)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
for _ in range(3):
|
||||
gate.step([hand]) # establish count=3
|
||||
gate.step([]) # miss=1 ≤ grace → track survives
|
||||
result = gate.step([hand]) # matches surviving track: count=4
|
||||
assert result == [True]
|
||||
|
||||
|
||||
def test_gate_grace_gap_frame_has_empty_result():
|
||||
"""During a grace frame (no hands input), result list is empty."""
|
||||
gate = HandPersistenceGate(min_frames=3, grace=2)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
for _ in range(3):
|
||||
gate.step([hand])
|
||||
assert gate.step([]) == [] # no hands → result is []
|
||||
|
||||
|
||||
def test_gate_grace_drops_after_grace_plus_one_misses():
|
||||
"""After grace+1 consecutive misses the track is dropped."""
|
||||
gate = HandPersistenceGate(min_frames=3, grace=2)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
for _ in range(3):
|
||||
gate.step([hand]) # establish
|
||||
for _ in range(3): # 3 misses: miss=1,2,3 → last one > grace=2 → dropped
|
||||
gate.step([])
|
||||
assert gate.step([hand]) == [False] # fresh count=1
|
||||
|
||||
|
||||
def test_gate_grace_ghost_stays_suppressed():
|
||||
"""A ghost appearing 1 frame then disappearing never becomes drawable
|
||||
(count=1 < min_frames=3 even after being carried by grace)."""
|
||||
gate = HandPersistenceGate(min_frames=3, grace=2)
|
||||
hand = _make_hand(wrist_x=0.5, wrist_y=0.5)
|
||||
gate.step([hand]) # count=1
|
||||
gate.step([]) # miss=1 ≤ grace → track [0.5, 0.5, 1, 1] survives
|
||||
# Reappears: count increments to 2 (still not established)
|
||||
assert gate.step([hand]) == [False]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# hand_facing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_facing_kp(wrist_x=0.5, wrist_y=0.7, size=0.15, spread=0.14) -> list[Kp]:
|
||||
"""21-kp hand with index-MCP(5) and pinky-MCP(17) spread wide in x."""
|
||||
kp = _make_hand(wrist_x=wrist_x, wrist_y=wrist_y, size=size)
|
||||
kp[5] = Kp(x=wrist_x - spread / 2, y=wrist_y - size / 2)
|
||||
kp[17] = Kp(x=wrist_x + spread / 2, y=wrist_y - size / 2)
|
||||
return kp
|
||||
|
||||
|
||||
def _make_sideon_kp(wrist_x=0.5, wrist_y=0.7, size=0.15) -> list[Kp]:
|
||||
"""21-kp hand with index-MCP(5) and pinky-MCP(17) almost co-located."""
|
||||
kp = _make_hand(wrist_x=wrist_x, wrist_y=wrist_y, size=size)
|
||||
kp[5] = Kp(x=wrist_x, y=wrist_y - size / 2)
|
||||
kp[17] = Kp(x=wrist_x + 0.005, y=wrist_y - size / 2)
|
||||
return kp
|
||||
|
||||
|
||||
def test_hand_facing_camera_above_threshold():
|
||||
"""A palm facing the camera has a spread ratio >= 0.7."""
|
||||
kp = _make_facing_kp(size=0.15, spread=0.14)
|
||||
# facing = 0.14 / 0.15 ~ 0.93
|
||||
assert hand_facing(kp) >= 0.7
|
||||
|
||||
|
||||
def test_hand_facing_side_on_below_threshold():
|
||||
"""A side-on palm has a spread ratio < 0.4."""
|
||||
kp = _make_sideon_kp(size=0.15)
|
||||
# facing = 0.005 / 0.15 ~ 0.033
|
||||
assert hand_facing(kp) < 0.4
|
||||
|
||||
|
||||
def test_hand_facing_zero_size_returns_zero():
|
||||
"""Guard: when hand_size is ~0 (degenerate hand), returns 0.0."""
|
||||
kp = _make_hand(wrist_x=0.5, wrist_y=0.5, size=0.0)
|
||||
# Force wrist and MCP to the same position so hand_size = 0
|
||||
kp[0] = Kp(x=0.5, y=0.5)
|
||||
kp[9] = Kp(x=0.5, y=0.5)
|
||||
assert hand_facing(kp) == 0.0
|
||||
|
||||
|
||||
def test_hand_facing_list_format():
|
||||
"""hand_facing also accepts list-style kp ([x, y, z]) as used in hand_slots tests."""
|
||||
# Build a 21-element list-of-lists
|
||||
lm = [[0.5, 0.5, 0.0] for _ in range(21)]
|
||||
lm[0] = [0.5, 0.7, 0.0] # wrist
|
||||
lm[9] = [0.5, 0.55, 0.0] # middle-MCP, size = 0.15
|
||||
# Wide spread: index-MCP and pinky-MCP 0.14 apart in x
|
||||
lm[5] = [0.43, 0.62, 0.0]
|
||||
lm[17] = [0.57, 0.62, 0.0]
|
||||
f = hand_facing(lm)
|
||||
# facing = 0.14 / 0.15 ~ 0.93
|
||||
assert f >= 0.7
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# gesture_quality
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_quality_hand(size: float = 0.10, facing: float = 0.7) -> list[Kp]:
|
||||
"""21-kp hand with precise hand_size and hand_facing values.
|
||||
|
||||
hand_size = wrist(kp[0]) to middle-MCP(kp[9]) distance = size
|
||||
hand_facing = dist(kp[5], kp[17]) / hand_size = facing
|
||||
"""
|
||||
kp = _make_hand(wrist_x=0.5, wrist_y=0.5, size=size, c=1.0)
|
||||
spread = facing * size
|
||||
mid_y = 0.5 - size / 2
|
||||
kp[5] = Kp(x=0.5 - spread / 2, y=mid_y)
|
||||
kp[17] = Kp(x=0.5 + spread / 2, y=mid_y)
|
||||
return kp
|
||||
|
||||
|
||||
def test_quality_absent_hand_returns_zero():
|
||||
"""hand=None → quality 0.0 regardless of other params."""
|
||||
assert gesture_quality(None, face_min=0.5, near_on=0.10) == 0.0
|
||||
|
||||
|
||||
def test_quality_engaged_forces_one():
|
||||
"""engaged=True → quality 1.0 regardless of hand geometry."""
|
||||
hand = _make_quality_hand(size=0.05, facing=0.1) # far and side-on
|
||||
assert gesture_quality(hand, face_min=0.5, near_on=0.10, engaged=True) == 1.0
|
||||
|
||||
|
||||
def test_quality_far_but_fully_facing():
|
||||
"""Hand at near-threshold (size=0.5*near_on) but fully facing → 0.65.
|
||||
|
||||
near_norm = 0.0 (at lower bound), facing_norm = 1.0 → 0.30 + 0.35 = 0.65.
|
||||
"""
|
||||
face_min, near_on = 0.5, 0.10
|
||||
# size = 0.5 * near_on = 0.05 → near_norm = 0
|
||||
# facing = face_min = 0.5 → facing_norm = (0.5-0.25)/(0.5-0.25) = 1.0
|
||||
hand = _make_quality_hand(size=0.05, facing=face_min)
|
||||
q = gesture_quality(hand, face_min=face_min, near_on=near_on)
|
||||
assert math.isclose(q, 0.65, rel_tol=1e-6), f"expected 0.65, got {q}"
|
||||
|
||||
|
||||
def test_quality_near_but_side_on():
|
||||
"""Hand fully near (size=near_on) but side-on (facing=0.25) → 0.65.
|
||||
|
||||
facing_norm = 0.0 (at lower bound), near_norm = 1.0 → 0.30 + 0.35 = 0.65.
|
||||
"""
|
||||
face_min, near_on = 0.5, 0.10
|
||||
# size = near_on = 0.10 → near_norm = (0.10-0.05)/0.05 = 1.0
|
||||
# facing = 0.25 → facing_norm = (0.25-0.25)/0.25 = 0.0
|
||||
hand = _make_quality_hand(size=near_on, facing=0.25)
|
||||
q = gesture_quality(hand, face_min=face_min, near_on=near_on)
|
||||
assert math.isclose(q, 0.65, rel_tol=1e-6), f"expected 0.65, got {q}"
|
||||
|
||||
|
||||
def test_quality_full_caps_at_one():
|
||||
"""Hand well above near_on and facing threshold → quality = 1.0."""
|
||||
face_min, near_on = 0.5, 0.10
|
||||
hand = _make_quality_hand(size=0.20, facing=0.9) # near_norm=1, facing_norm=1
|
||||
q = gesture_quality(hand, face_min=face_min, near_on=near_on)
|
||||
assert math.isclose(q, 1.0, rel_tol=1e-6), f"expected 1.0, got {q}"
|
||||
|
||||
|
||||
def test_quality_monotonic_in_size():
|
||||
"""For fixed full-facing, quality increases strictly with hand size."""
|
||||
face_min, near_on = 0.5, 0.10
|
||||
facing = face_min # facing_norm = 1.0 at exactly face_min
|
||||
sizes = [0.5 * near_on, 0.75 * near_on, near_on]
|
||||
qs = [gesture_quality(_make_quality_hand(size=s, facing=facing),
|
||||
face_min=face_min, near_on=near_on)
|
||||
for s in sizes]
|
||||
assert qs[0] < qs[1] < qs[2], f"not monotone: {qs}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# gauge_segments
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_gauge_segments_no_data_returns_two_dim_rails():
|
||||
"""cx/cy=None → 2 rail segments (dim, no markers)."""
|
||||
segs = gauge_segments(None, None, "left", aspect=1.0)
|
||||
# Exactly 2 rails: one horizontal (X gauge), one vertical (Y gauge)
|
||||
assert len(segs) == 2, f"expected 2 rail segs, got {len(segs)}"
|
||||
confs = [s[4] for s in segs]
|
||||
assert all(c == 0.25 for c in confs), f"dim conf expected 0.25, got {confs}"
|
||||
pids = [s[5] for s in segs]
|
||||
assert all(p == 7 for p in pids), f"rail pid should be 7, got {pids}"
|
||||
|
||||
|
||||
def test_gauge_segments_with_data_returns_rails_and_markers():
|
||||
"""cx/cy provided → 2 rails + 2 X-marker ticks + 2 Y-marker ticks = 6 total."""
|
||||
segs = gauge_segments(0.5, 0.5, "left", aspect=1.0)
|
||||
assert len(segs) == 6, f"expected 6 segments (2 rail + 4 marker), got {len(segs)}"
|
||||
rail_segs = [s for s in segs if s[5] == 7]
|
||||
marker_segs = [s for s in segs if s[5] != 7]
|
||||
assert len(rail_segs) == 2 and len(marker_segs) == 4
|
||||
|
||||
|
||||
def test_gauge_x_marker_centered_on_rail():
|
||||
"""cx=0.5 → primary X marker tick is at x = (x0+x1)/2 of the panel."""
|
||||
x0, _y0, x1, _y1 = panel_rect("left", aspect=1.0)
|
||||
center_x = (x0 + x1) / 2.0
|
||||
segs = gauge_segments(0.5, 0.5, "left", aspect=1.0, content_pid=5)
|
||||
# X marker ticks are vertical segments (same x on both endpoints), pid != 7
|
||||
x_marker_segs = [s for s in segs
|
||||
if s[5] != 7 and math.isclose(s[0], s[2], abs_tol=1e-9)]
|
||||
assert x_marker_segs, "no vertical X-marker ticks found"
|
||||
# The primary tick (minimum x) should land at center_x; the bold-offset
|
||||
# tick is at center_x + GAUGE_BOLD (a tiny offset for visual thickness).
|
||||
primary_x = min(s[0] for s in x_marker_segs)
|
||||
assert math.isclose(primary_x, center_x, rel_tol=1e-6), \
|
||||
f"primary X marker at {primary_x}, expected {center_x}"
|
||||
|
||||
|
||||
def test_gauge_mirror_flips_x_marker():
|
||||
"""mirror=True → X marker at 1-cx (mirrored position), cx unchanged for Y."""
|
||||
x0, _y0, x1, _y1 = panel_rect("left", aspect=1.0)
|
||||
cx = 0.3
|
||||
# Without mirror: marker at x0 + 0.3*(x1-x0)
|
||||
segs_norm = gauge_segments(cx, 0.5, "left", 1.0, mirror=False, content_pid=5)
|
||||
# With mirror: marker at x0 + 0.7*(x1-x0)
|
||||
segs_mirr = gauge_segments(cx, 0.5, "left", 1.0, mirror=True, content_pid=5)
|
||||
# Extract first X-marker tick (vertical segment, pid=5)
|
||||
def _x_tick_x(segs):
|
||||
for ax, ay, bx, by, c, p in segs:
|
||||
if p == 5 and math.isclose(ax, bx, rel_tol=1e-9):
|
||||
return ax
|
||||
return None
|
||||
xn = _x_tick_x(segs_norm)
|
||||
xm = _x_tick_x(segs_mirr)
|
||||
assert xn is not None and xm is not None
|
||||
expected_n = x0 + cx * (x1 - x0)
|
||||
expected_m = x0 + (1.0 - cx) * (x1 - x0)
|
||||
assert math.isclose(xn, expected_n, rel_tol=1e-6), f"normal: {xn} != {expected_n}"
|
||||
assert math.isclose(xm, expected_m, rel_tol=1e-6), f"mirror: {xm} != {expected_m}"
|
||||
|
||||
|
||||
def test_gauge_y_rail_below_panel_x_rail():
|
||||
"""X gauge rail y-coordinate is below the panel bottom edge."""
|
||||
_x0, _y0, _x1, y1 = panel_rect("left", aspect=1.0)
|
||||
segs = gauge_segments(0.5, 0.5, "left", 1.0)
|
||||
# Horizontal rail: ay == by (same y, spanning x)
|
||||
h_rail = [s for s in segs if math.isclose(s[1], s[3], rel_tol=1e-9)
|
||||
and s[5] == 7]
|
||||
assert h_rail, "no horizontal rail segment"
|
||||
rail_y = h_rail[0][1]
|
||||
assert rail_y > y1, f"X gauge rail y={rail_y} should be below panel y1={y1}"
|
||||
assert math.isclose(rail_y, y1 + GAUGE_GAP, rel_tol=1e-6)
|
||||
@@ -0,0 +1,243 @@
|
||||
import math
|
||||
import pytest
|
||||
from data_only_viz.hand_features import HandFeatureExtractor, NEUTRAL_HAND
|
||||
|
||||
|
||||
class LM:
|
||||
"""Minimal landmark stub (PoseKp-shaped)."""
|
||||
def __init__(self, x, y, z=0.0, c=1.0):
|
||||
self.x, self.y, self.z, self.c = x, y, z, c
|
||||
|
||||
|
||||
def _hand(cx, cy, span):
|
||||
"""21 landmarks centered at (cx,cy); thumb/pinky separated by `span`.
|
||||
|
||||
Index 0 wrist, 4 thumb_tip, 9 middle_mcp, 20 pinky_tip.
|
||||
hand_size (wrist->middle_mcp) fixed at 0.10 so openness ~ span/size.
|
||||
"""
|
||||
pts = [LM(cx, cy) for _ in range(21)]
|
||||
pts[0] = LM(cx, cy + 0.05) # wrist below center
|
||||
pts[9] = LM(cx, cy - 0.05) # middle_mcp above -> size 0.10
|
||||
pts[4] = LM(cx - span / 2, cy) # thumb_tip
|
||||
pts[20] = LM(cx + span / 2, cy) # pinky_tip
|
||||
return pts
|
||||
|
||||
|
||||
def test_open_hand_high_openness():
|
||||
ext = HandFeatureExtractor()
|
||||
out = ext.step([_hand(0.5, 0.5, span=0.20)]) # span/size = 2.0
|
||||
hand = out["L"] or out["R"]
|
||||
assert hand["openness"] > 0.8
|
||||
|
||||
|
||||
def test_fist_low_openness():
|
||||
ext = HandFeatureExtractor()
|
||||
out = ext.step([_hand(0.5, 0.5, span=0.03)]) # span/size = 0.3
|
||||
hand = out["L"] or out["R"]
|
||||
assert hand["openness"] < 0.2
|
||||
|
||||
|
||||
def test_static_hand_zero_speed():
|
||||
ext = HandFeatureExtractor()
|
||||
h = _hand(0.5, 0.5, span=0.1)
|
||||
ext.step([h])
|
||||
out = ext.step([h])
|
||||
hand = out["L"] or out["R"]
|
||||
assert hand["speed"] == pytest.approx(0.0, abs=1e-6)
|
||||
|
||||
|
||||
def test_moving_hand_positive_speed():
|
||||
ext = HandFeatureExtractor()
|
||||
ext.step([_hand(0.2, 0.5, span=0.1)])
|
||||
out = ext.step([_hand(0.6, 0.5, span=0.1)])
|
||||
hand = out["L"] or out["R"]
|
||||
assert hand["speed"] > 0.0
|
||||
|
||||
|
||||
def test_left_right_by_screen_x():
|
||||
ext = HandFeatureExtractor()
|
||||
out = ext.step([_hand(0.8, 0.5, span=0.1), _hand(0.2, 0.5, span=0.1)])
|
||||
assert out["L"]["cx"] < out["R"]["cx"] # L is leftmost on screen
|
||||
assert out["dist"] > 0.0
|
||||
|
||||
|
||||
def test_single_hand_zero_dist():
|
||||
ext = HandFeatureExtractor()
|
||||
out = ext.step([_hand(0.5, 0.5, span=0.1)])
|
||||
assert out["dist"] == 0.0
|
||||
|
||||
|
||||
def test_no_hands_returns_none():
|
||||
ext = HandFeatureExtractor()
|
||||
out = ext.step([])
|
||||
assert out["L"] is None and out["R"] is None and out["dist"] == 0.0
|
||||
|
||||
|
||||
def test_nan_landmark_falls_back_no_exception():
|
||||
ext = HandFeatureExtractor()
|
||||
h = _hand(0.5, 0.5, span=0.1)
|
||||
h[0] = LM(float("nan"), float("nan"))
|
||||
out = ext.step([h]) # must not raise
|
||||
hand = out["L"] or out["R"]
|
||||
assert math.isfinite(hand["cx"]) and math.isfinite(hand["openness"])
|
||||
|
||||
|
||||
def test_numpy_row_landmarks_finite_features():
|
||||
"""Landmarks as numpy rows (indexable [x, y, ...]) must yield finite cx/cy/openness."""
|
||||
import numpy as np
|
||||
arr = np.zeros((21, 3), dtype=np.float32)
|
||||
# replicate _hand(0.5, 0.5, span=0.20) geometry via index access
|
||||
arr[:] = [0.5, 0.5, 0.0]
|
||||
arr[0] = [0.5, 0.55, 0.0] # wrist
|
||||
arr[9] = [0.5, 0.45, 0.0] # middle_mcp → size 0.10
|
||||
arr[4] = [0.4, 0.5, 0.0] # thumb_tip
|
||||
arr[20] = [0.6, 0.5, 0.0] # pinky_tip → span 0.20
|
||||
ext = HandFeatureExtractor()
|
||||
out = ext.step([arr]) # must not raise
|
||||
hand = out["L"] or out["R"]
|
||||
assert hand is not None
|
||||
assert math.isfinite(hand["cx"])
|
||||
assert math.isfinite(hand["cy"])
|
||||
assert math.isfinite(hand["openness"])
|
||||
assert hand["openness"] > 0.8 # span/size = 2.0, same as test_open_hand
|
||||
|
||||
|
||||
def _fist_hand(tip_ext):
|
||||
"""All 4 finger MCPs at center; each fingertip placed tip_ext*size away
|
||||
(so the per-finger tip->MCP ratio == tip_ext). size (wrist->mcp9) = 0.10."""
|
||||
cx, cy = 0.5, 0.5
|
||||
pts = [LM(cx, cy) for _ in range(21)]
|
||||
pts[0] = LM(cx, cy + 0.10) # wrist -> size 0.10
|
||||
for mcp in (5, 9, 13, 17):
|
||||
pts[mcp] = LM(cx, cy)
|
||||
for tip in (8, 12, 16, 20):
|
||||
pts[tip] = LM(cx + tip_ext * 0.10, cy)
|
||||
pts[4] = LM(cx - 0.05, cy) # thumb
|
||||
return pts
|
||||
|
||||
|
||||
def test_fist_all_fingers_curled():
|
||||
out = HandFeatureExtractor(fist_enable=True).step([_fist_hand(0.20)]) # tips at knuckles
|
||||
assert (out["L"] or out["R"])["fist"] > 0.9
|
||||
|
||||
|
||||
def test_open_hand_is_not_a_fist():
|
||||
out = HandFeatureExtractor(fist_enable=True).step([_fist_hand(0.80)]) # fingers extended
|
||||
assert (out["L"] or out["R"])["fist"] < 0.1
|
||||
|
||||
|
||||
def test_one_extended_finger_breaks_the_fist():
|
||||
h = _fist_hand(0.20) # all curled...
|
||||
h[8] = LM(0.5 + 0.80 * 0.10, 0.5) # ...except the index
|
||||
out = HandFeatureExtractor(fist_enable=True).step([h])
|
||||
assert (out["L"] or out["R"])["fist"] < 0.1 # min over fingers
|
||||
|
||||
|
||||
def test_swap_routes_inverted_chirality_to_user_slots():
|
||||
"""C1 pin: with the inverted-source swap ON (HAND_SWAP_LR default),
|
||||
a physical LEFT hand arriving labeled chir=1 must land in feature
|
||||
slot L — consistent with the pinch/strike gesture path."""
|
||||
ext = HandFeatureExtractor()
|
||||
left_hand = _hand(0.30, 0.5, span=0.20) # user's left (source says 1)
|
||||
right_hand = _hand(0.70, 0.5, span=0.06) # user's right (source says 0)
|
||||
out = ext.step([left_hand, right_hand], [1, 0], swap=True)
|
||||
assert out["L"] is not None and out["R"] is not None
|
||||
assert out["L"]["cx"] == pytest.approx(0.30, abs=0.01) # left hand in L
|
||||
assert out["R"]["cx"] == pytest.approx(0.70, abs=0.01)
|
||||
|
||||
|
||||
def test_no_swap_keeps_source_chirality():
|
||||
ext = HandFeatureExtractor()
|
||||
a = _hand(0.30, 0.5, span=0.20)
|
||||
b = _hand(0.70, 0.5, span=0.06)
|
||||
out = ext.step([a, b], [1, 0], swap=False)
|
||||
assert out["L"]["cx"] == pytest.approx(0.70, abs=0.01) # chir 0 -> L
|
||||
assert out["R"]["cx"] == pytest.approx(0.30, abs=0.01)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fist_enabled gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_fist_enabled_false_zeroes_fist_preserves_openness():
|
||||
"""fist_enabled=(False, False) forces fist=0 but leaves openness intact."""
|
||||
ext = HandFeatureExtractor()
|
||||
h = _fist_hand(0.20) # all curled -> fist > 0.9 normally
|
||||
out = ext.step([h], fist_enabled=(False, False))
|
||||
hand = out["L"] or out["R"]
|
||||
assert hand["fist"] == 0.0, "fist must be zeroed when disabled"
|
||||
assert hand["openness"] < 0.2, "openness unaffected"
|
||||
|
||||
|
||||
def test_fist_enabled_true_computes_fist():
|
||||
"""fist_enabled=(True, True) leaves fist computation unchanged."""
|
||||
ext = HandFeatureExtractor(fist_enable=True)
|
||||
h = _fist_hand(0.20)
|
||||
out = ext.step([h], fist_enabled=(True, True))
|
||||
hand = out["L"] or out["R"]
|
||||
assert hand["fist"] > 0.9
|
||||
|
||||
|
||||
def test_mirror_flips_cx_fallback_slot_assignment():
|
||||
"""mirror=True inverts the cx fallback so screen-right hand goes to slot L."""
|
||||
ext = HandFeatureExtractor()
|
||||
h_low = _hand(0.2, 0.5, span=0.10) # cx=0.2 -> slot L normally (leftmost)
|
||||
h_high = _hand(0.8, 0.5, span=0.10) # cx=0.8 -> slot R normally
|
||||
# No chirality -> cx fallback
|
||||
plain = ext.step([h_low, h_high], chirality=None, mirror=False)
|
||||
mirrored = ext.step([h_low, h_high], chirality=None, mirror=True)
|
||||
assert plain["L"]["cx"] == pytest.approx(0.2, abs=0.02), "non-mirrored: L gets leftmost"
|
||||
assert mirrored["L"]["cx"] == pytest.approx(0.8, abs=0.02), "mirrored: L gets rightmost"
|
||||
|
||||
|
||||
def test_mirror_default_false_unchanged():
|
||||
"""mirror defaults to False; existing tests that pass no mirror kwarg are unaffected."""
|
||||
ext = HandFeatureExtractor()
|
||||
h_a = _hand(0.2, 0.5, span=0.10)
|
||||
h_b = _hand(0.8, 0.5, span=0.10)
|
||||
no_kwarg = ext.step([h_a, h_b])
|
||||
explicit = ext.step([h_a, h_b], mirror=False)
|
||||
assert no_kwarg["L"]["cx"] == pytest.approx(explicit["L"]["cx"], abs=1e-6)
|
||||
|
||||
|
||||
def test_fist_enabled_per_slot_independence():
|
||||
"""fist_enabled masks only the specified slot."""
|
||||
ext = HandFeatureExtractor()
|
||||
# Build two 21-kp hands with distinct wrist positions for L/R routing
|
||||
# Left hand: cx~0.2, all fingers curled (fist)
|
||||
h_fist = _fist_hand(0.20)
|
||||
h_fist[0] = LM(0.2, 0.6)
|
||||
h_fist[9] = LM(0.2, 0.5)
|
||||
for mcp in (5, 13, 17):
|
||||
h_fist[mcp] = LM(0.2, 0.5)
|
||||
for tip in (8, 12, 16, 20):
|
||||
h_fist[tip] = LM(0.2 + 0.20 * 0.10, 0.5)
|
||||
# Right hand: cx~0.8, fingers extended (open)
|
||||
h_open = _fist_hand(0.80)
|
||||
h_open[0] = LM(0.8, 0.6)
|
||||
h_open[9] = LM(0.8, 0.5)
|
||||
for mcp in (5, 13, 17):
|
||||
h_open[mcp] = LM(0.8, 0.5)
|
||||
for tip in (8, 12, 16, 20):
|
||||
h_open[tip] = LM(0.8 + 0.80 * 0.10, 0.5)
|
||||
out = ext.step([h_fist, h_open], fist_enabled=(False, True))
|
||||
# L slot (cx~0.2) should have fist=0 even though it's curled
|
||||
assert out["L"]["fist"] == 0.0
|
||||
# R slot (cx~0.8) should compute normally (open -> fist < 0.1)
|
||||
assert out["R"]["fist"] < 0.1
|
||||
|
||||
|
||||
def test_fist_kill_switch_default_off(monkeypatch):
|
||||
"""FIST_ENABLE unset -> fist forced 0.0 even on a perfect fist
|
||||
(user-disabled live 2026-07-02); continuous features unaffected."""
|
||||
monkeypatch.delenv("FIST_ENABLE", raising=False)
|
||||
out = HandFeatureExtractor().step([_fist_hand(0.20)])
|
||||
hand = out["L"] or out["R"]
|
||||
assert hand["fist"] == 0.0
|
||||
assert hand["openness"] < 0.2
|
||||
|
||||
|
||||
def test_fist_kill_switch_env_reenable(monkeypatch):
|
||||
monkeypatch.setenv("FIST_ENABLE", "1")
|
||||
out = HandFeatureExtractor().step([_fist_hand(0.20)])
|
||||
assert (out["L"] or out["R"])["fist"] > 0.9
|
||||
@@ -0,0 +1,424 @@
|
||||
"""Tests for hand_slots.route_hands and GestureSlotStabilizer."""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from data_only_viz.hand_slots import route_hands, GestureSlotStabilizer
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _hand(cx: float = 0.5, size: float = 0.15) -> list:
|
||||
"""21-landmark hand (list of [x, y, z]). Wrist at cx, middle-MCP 'size' above."""
|
||||
lm = [[cx, 0.5, 0.0] for _ in range(21)]
|
||||
lm[0] = [cx, 0.6, 0.0] # wrist
|
||||
lm[9] = [cx, 0.6 - size, 0.0] # middle-MCP -> hand_size == size
|
||||
return lm
|
||||
|
||||
|
||||
def _small_hand(cx: float = 0.5) -> list:
|
||||
"""Hand with size 0.02 (below default near_min 0.10)."""
|
||||
return _hand(cx=cx, size=0.02)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Basic chirality routing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_chirality_routes_left_to_slot0():
|
||||
h_l = _hand(cx=0.8) # camera-right = user's left (mirror)
|
||||
h_r = _hand(cx=0.2)
|
||||
result = route_hands([h_l, h_r], [0, 1])
|
||||
assert result[0] is h_l
|
||||
assert result[1] is h_r
|
||||
|
||||
|
||||
def test_chirality_routes_right_to_slot1():
|
||||
h_r = _hand(cx=0.2)
|
||||
result = route_hands([h_r], [1])
|
||||
assert result[0] is None
|
||||
assert result[1] is h_r
|
||||
|
||||
|
||||
def test_chirality_first_match_wins_per_slot():
|
||||
"""Two hands with chirality==0: first wins, second dropped."""
|
||||
h1 = _hand(cx=0.3)
|
||||
h2 = _hand(cx=0.7)
|
||||
result = route_hands([h1, h2], [0, 0])
|
||||
assert result[0] is h1
|
||||
assert result[1] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chirality misaligned → cx fallback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_chirality_misaligned_uses_cx_fallback():
|
||||
h_l = _hand(cx=0.2)
|
||||
h_r = _hand(cx=0.8)
|
||||
# chirality length 1 != hands length 2 → misaligned
|
||||
result = route_hands([h_l, h_r], [0])
|
||||
# cx fallback: ascending cx order → cx=0.2 → slot 0, cx=0.8 → slot 1
|
||||
assert result[0] is h_l
|
||||
assert result[1] is h_r
|
||||
|
||||
|
||||
def test_chirality_none_uses_cx_fallback():
|
||||
h_l = _hand(cx=0.2)
|
||||
h_r = _hand(cx=0.8)
|
||||
result = route_hands([h_l, h_r], None)
|
||||
assert result[0] is h_l
|
||||
assert result[1] is h_r
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# mirror flips fallback order
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_mirror_flips_cx_fallback_order():
|
||||
h_a = _hand(cx=0.2) # low cx = screen-left normally
|
||||
h_b = _hand(cx=0.8)
|
||||
# Without mirror: slot0=cx0.2, slot1=cx0.8
|
||||
plain = route_hands([h_a, h_b])
|
||||
assert plain[0] is h_a
|
||||
# With mirror: flip cx → 1-0.2=0.8, 1-0.8=0.2 → sort ascending → cx0.8 goes slot0
|
||||
mirrored = route_hands([h_a, h_b], mirror=True)
|
||||
assert mirrored[0] is h_b
|
||||
assert mirrored[1] is h_a
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# swap: inverts the CHIRALITY interpretation only (source chirality flipped);
|
||||
# the cx fallback is screen-relative and must never be affected.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_swap_inverts_chirality_mapping():
|
||||
h_l = _hand(cx=0.8) # Vision says left (chir=0) but source is inverted
|
||||
h_r = _hand(cx=0.2) # Vision says right (chir=1)
|
||||
result = route_hands([h_l, h_r], [0, 1], swap=True)
|
||||
assert result[0] is h_r # chir 1 -> left slot under swap
|
||||
assert result[1] is h_l # chir 0 -> right slot under swap
|
||||
|
||||
|
||||
def test_swap_does_not_alter_cx_fallback():
|
||||
"""swap only concerns chirality; the cx fallback stays screen-relative."""
|
||||
h_a = _hand(cx=0.2)
|
||||
h_b = _hand(cx=0.8)
|
||||
result = route_hands([h_a, h_b], None, swap=True)
|
||||
assert result[0] is h_a # unchanged: leftmost keeps slot 0
|
||||
assert result[1] is h_b
|
||||
|
||||
|
||||
def test_swap_single_hand_moves_slot():
|
||||
"""A lone chir==0 hand lands in slot 1 when swap is on."""
|
||||
h = _hand(cx=0.5)
|
||||
result = route_hands([h], [0], swap=True)
|
||||
assert result[0] is None
|
||||
assert result[1] is h
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# near_min gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_near_min_drops_small_hand():
|
||||
big = _hand(cx=0.3, size=0.15)
|
||||
small = _hand(cx=0.7, size=0.02)
|
||||
result = route_hands([big, small], near_min=0.10)
|
||||
assert result[0] is big
|
||||
assert result[1] is None # too small
|
||||
|
||||
|
||||
def test_near_min_preserves_big_slot():
|
||||
big = _hand(cx=0.5, size=0.20)
|
||||
result = route_hands([big], near_min=0.10)
|
||||
assert result[0] is big
|
||||
|
||||
|
||||
def test_near_min_drops_both_when_all_small():
|
||||
s1 = _small_hand(cx=0.3)
|
||||
s2 = _small_hand(cx=0.7)
|
||||
result = route_hands([s1, s2], near_min=0.10)
|
||||
assert result == [None, None]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Invalid hand filtering (alignment preserved)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_invalid_hand_skipped_chirality_alignment_preserved():
|
||||
"""An invalid hand at index 0 must not shift chirality for index 1."""
|
||||
bad = None # invalid
|
||||
h_r = _hand(cx=0.5)
|
||||
# chirality: bad=0(ignored), h_r=1(right)
|
||||
result = route_hands([bad, h_r], [0, 1])
|
||||
assert result[0] is None
|
||||
assert result[1] is h_r
|
||||
|
||||
|
||||
def test_short_hand_rejected():
|
||||
short = [[0.5, 0.5, 0.0]] * 10 # <21 landmarks
|
||||
h_ok = _hand(cx=0.5)
|
||||
result = route_hands([short, h_ok], [0, 1])
|
||||
assert result[0] is None # short dropped
|
||||
assert result[1] is h_ok
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Edge cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_empty_hands_returns_none_pair():
|
||||
assert route_hands([]) == [None, None]
|
||||
|
||||
|
||||
def test_single_hand_no_chirality_slot0():
|
||||
h = _hand(cx=0.5)
|
||||
result = route_hands([h])
|
||||
assert result[0] is h
|
||||
assert result[1] is None
|
||||
|
||||
|
||||
def test_two_hands_chirality_all_left():
|
||||
"""Both hands claim left; first gets slot 0, second is dropped."""
|
||||
h1 = _hand(cx=0.2)
|
||||
h2 = _hand(cx=0.8)
|
||||
result = route_hands([h1, h2], [0, 0])
|
||||
assert result[0] is h1
|
||||
assert result[1] is None
|
||||
|
||||
|
||||
def test_result_always_length_2():
|
||||
for hands, chir in [
|
||||
([], None),
|
||||
([_hand()], None),
|
||||
([_hand(), _hand(cx=0.8)], [0, 1]),
|
||||
([_hand(), _hand(cx=0.8), _hand(cx=0.3)], [0, 1, 0]),
|
||||
]:
|
||||
r = route_hands(hands, chir)
|
||||
assert len(r) == 2, f"expected 2 slots, got {len(r)} for {chir}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GestureSlotStabilizer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _near_hand(cx: float = 0.5, size: float = 0.15) -> list:
|
||||
"""Near hand (size >= near_on=0.10) that also passes the face gate.
|
||||
|
||||
kp[5]/kp[17] spread by 0.12 → facing = 0.12/0.15 = 0.8 >= face_min=0.5.
|
||||
All tests that verify near-hysteresis / hold / resumed behavior should use
|
||||
this helper so they continue to pass after the facing gate was added.
|
||||
"""
|
||||
lm = _hand(cx=cx, size=size)
|
||||
lm[5] = [cx - 0.06, 0.5, 0.0] # index-MCP
|
||||
lm[17] = [cx + 0.06, 0.5, 0.0] # pinky-MCP → spread=0.12, facing≈0.8
|
||||
return lm
|
||||
|
||||
|
||||
def _sideon_near_hand(cx: float = 0.5, size: float = 0.15) -> list:
|
||||
"""Near hand (size >= near_on) whose kp[5]==kp[17] → facing=0.0 (fails face gate).
|
||||
|
||||
Use this for tests that specifically verify the face gate rejects a side-on
|
||||
palm even when the hand is close enough to pass the near gate.
|
||||
"""
|
||||
return _hand(cx=cx, size=size)
|
||||
|
||||
|
||||
def _far_hand(cx: float = 0.5) -> list:
|
||||
"""Hand with hand_size == 0.02 (below near_on=0.10)."""
|
||||
return _hand(cx=cx, size=0.02)
|
||||
|
||||
|
||||
def test_stab_hold_last_one_frame_hole():
|
||||
"""A 1-frame Vision hole keeps the last hand in the slot."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h = _near_hand()
|
||||
stab.step([h, None]) # activate slot 0
|
||||
stab.step([h, None]) # slot 0 established
|
||||
out = stab.step([None, None]) # miss=1 <= hold -> slot 0 holds last hand
|
||||
assert out[0] is h, "slot 0 should hold last hand on 1-frame miss"
|
||||
assert out[1] is None
|
||||
|
||||
|
||||
def test_stab_hold_last_two_frame_hole():
|
||||
"""A 2-frame Vision hole keeps the last hand both frames."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h = _near_hand()
|
||||
stab.step([h, None])
|
||||
out1 = stab.step([None, None]) # miss=1
|
||||
out2 = stab.step([None, None]) # miss=2 <= hold=2
|
||||
assert out1[0] is h
|
||||
assert out2[0] is h
|
||||
|
||||
|
||||
def test_stab_slot_goes_none_after_hold_frames():
|
||||
"""After hold_frames+1 consecutive misses the slot yields None."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h = _near_hand()
|
||||
stab.step([h, None])
|
||||
stab.step([None, None]) # miss=1
|
||||
stab.step([None, None]) # miss=2 (still holding)
|
||||
out = stab.step([None, None]) # miss=3 > hold=2 -> None for real
|
||||
assert out[0] is None
|
||||
|
||||
|
||||
def test_stab_hysteresis_stays_active_above_near_off():
|
||||
"""Once active, hand stays active while size >= near_off."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h_on = _near_hand(size=0.11) # size >= near_on -> activates
|
||||
h_mid = _near_hand(size=0.09) # near_off <= size < near_on -> stays active
|
||||
out1 = stab.step([h_on, None])
|
||||
assert out1[0] is h_on
|
||||
out2 = stab.step([h_mid, None])
|
||||
assert out2[0] is h_mid, "should stay active at size 0.09 (>= near_off 0.08)"
|
||||
|
||||
|
||||
def test_stab_hysteresis_deactivates_below_near_off():
|
||||
"""Hand deactivates when size drops below near_off."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h_on = _near_hand(size=0.11)
|
||||
h_off = _near_hand(size=0.07) # size < near_off -> deactivate
|
||||
stab.step([h_on, None])
|
||||
out = stab.step([h_off, None])
|
||||
assert out[0] is None, "slot should deactivate below near_off"
|
||||
|
||||
|
||||
def test_stab_hysteresis_needs_near_on_to_reactivate():
|
||||
"""After deactivation, size must reach near_on again to re-activate."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h_on = _near_hand(size=0.11)
|
||||
h_off = _near_hand(size=0.07) # deactivates
|
||||
h_mid = _near_hand(size=0.09) # between near_off and near_on -> stays inactive
|
||||
h_on2 = _near_hand(size=0.11) # back to near_on -> reactivates
|
||||
stab.step([h_on, None])
|
||||
stab.step([h_off, None]) # deactivated
|
||||
out_mid = stab.step([h_mid, None])
|
||||
assert out_mid[0] is None, "should stay inactive at 0.09 after deactivation"
|
||||
out_on2 = stab.step([h_on2, None])
|
||||
assert out_on2[0] is h_on2, "should re-activate at near_on"
|
||||
|
||||
|
||||
def test_stab_fresh_far_hand_returns_none():
|
||||
"""A far hand on a fresh stabilizer yields None (not near)."""
|
||||
stab = GestureSlotStabilizer(near_on=0.10, near_off=0.08)
|
||||
h = _far_hand()
|
||||
out = stab.step([h, None])
|
||||
assert out[0] is None
|
||||
|
||||
|
||||
def test_stab_active_flags_reflect_current_output():
|
||||
"""active_flags() matches the last step() output per slot."""
|
||||
stab = GestureSlotStabilizer(near_on=0.10, near_off=0.08)
|
||||
h = _near_hand()
|
||||
stab.step([h, None])
|
||||
flags = stab.active_flags()
|
||||
assert flags == (True, False)
|
||||
stab.step([None, None]) # miss=1 -> hold
|
||||
flags2 = stab.active_flags()
|
||||
assert flags2 == (True, False) # still holding
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GestureSlotStabilizer.resumed_flags
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_resumed_flags_true_exactly_one_step_on_reappear():
|
||||
"""resumed_flags True on the one step that transitions held -> real."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h = _near_hand()
|
||||
h2 = _near_hand(cx=0.4)
|
||||
stab.step([h, None]) # activate slot 0
|
||||
stab.step([None, None]) # miss=1 -> held replay
|
||||
stab.step([h2, None]) # real hand returns -> resumed
|
||||
assert stab.resumed_flags()[0] is True, "should be True on first real frame after hold"
|
||||
# next step, still real -> False
|
||||
stab.step([h2, None])
|
||||
assert stab.resumed_flags()[0] is False, "should be False once hand stays real"
|
||||
|
||||
|
||||
def test_resumed_flags_false_steady():
|
||||
"""Continuously present hand -> resumed_flags never True."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h = _near_hand()
|
||||
stab.step([h, None])
|
||||
stab.step([h, None])
|
||||
assert stab.resumed_flags()[0] is False
|
||||
|
||||
|
||||
def test_resumed_flags_false_after_expired_hold():
|
||||
"""Hand returning after hold window expired does not trigger resumed (slot went None)."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h = _near_hand()
|
||||
h2 = _near_hand(cx=0.4)
|
||||
stab.step([h, None]) # activate
|
||||
stab.step([None, None]) # miss=1, held
|
||||
stab.step([None, None]) # miss=2, held
|
||||
stab.step([None, None]) # miss=3 > hold -> near_active cleared, out=None
|
||||
stab.step([h2, None]) # reappear after expired hold
|
||||
# _held_flag was False (hold expired), so resumed must be False
|
||||
assert stab.resumed_flags()[0] is False, "expired hold: held_flag reset, no resume"
|
||||
|
||||
|
||||
def test_resumed_flags_independent_per_slot():
|
||||
"""Each slot tracks held/resumed independently."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h0 = _near_hand(cx=0.3)
|
||||
h1 = _near_hand(cx=0.7)
|
||||
h0b = _near_hand(cx=0.3)
|
||||
stab.step([h0, h1]) # activate both
|
||||
stab.step([None, h1]) # slot 0 held, slot 1 real
|
||||
stab.step([h0b, h1]) # slot 0 resumed, slot 1 unchanged
|
||||
r = stab.resumed_flags()
|
||||
assert r[0] is True, "slot 0 just transitioned held -> real"
|
||||
assert r[1] is False, "slot 1 was never held"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GestureSlotStabilizer — face gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_stab_face_gate_sideon_never_activates():
|
||||
"""A side-on hand (facing=0.0) never activates the slot even when near."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h = _sideon_near_hand() # size=0.15 >= near_on, but facing=0.0 < face_min=0.5
|
||||
out1 = stab.step([h, None])
|
||||
out2 = stab.step([h, None])
|
||||
assert out1[0] is None, "side-on: slot must be None (face gate rejects)"
|
||||
assert out2[0] is None, "side-on: slot still None after two frames"
|
||||
assert stab.active_flags() == (False, False)
|
||||
|
||||
|
||||
def test_stab_face_gate_facing_activates():
|
||||
"""A near, camera-facing hand activates the slot (passes both gates)."""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
h = _near_hand() # size=0.15 >= near_on, facing=0.8 >= face_min=0.5
|
||||
stab.step([h, None])
|
||||
out = stab.step([h, None])
|
||||
assert out[0] is h, "near+facing hand must be in slot 0"
|
||||
assert stab.active_flags() == (True, False)
|
||||
|
||||
|
||||
def test_stab_face_gate_hysteresis_stays_active_between_thresholds():
|
||||
"""Once active, slot stays active when facing drops between face_off and face_min.
|
||||
|
||||
face_min=0.5, face_off=0.4. A hand that was facing (0.8) then turns
|
||||
slightly side-on to facing=0.43 must NOT deactivate (hysteresis protects
|
||||
against flapping). Only a drop below face_off (0.4) would deactivate.
|
||||
"""
|
||||
stab = GestureSlotStabilizer(hold_frames=2, near_on=0.10, near_off=0.08)
|
||||
# Activate: near+facing hand
|
||||
h_facing = _near_hand() # facing=0.8
|
||||
stab.step([h_facing, None])
|
||||
assert stab.step([h_facing, None])[0] is h_facing
|
||||
|
||||
# Build a hand with facing=0.43 (between face_off=0.4 and face_min=0.5)
|
||||
# size=0.15, spread=0.43*0.15=0.0645
|
||||
h_mid = _hand()
|
||||
h_mid[5] = [0.5 - 0.032, 0.5, 0.0]
|
||||
h_mid[17] = [0.5 + 0.032, 0.5, 0.0] # dist≈0.064/0.15≈0.43
|
||||
|
||||
out = stab.step([h_mid, None])
|
||||
assert out[0] is h_mid, "hysteresis: slot stays active between face_off and face_min"
|
||||
assert stab.active_flags() == (True, False)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""The iPhone USB source must keep retrying the initial connect, not give up
|
||||
when the iPhone app isn't streaming yet. Auto-reconnect must cover a startup
|
||||
race (app not ready when data_only_viz starts), not only a mid-session drop."""
|
||||
import time
|
||||
|
||||
import data_only_viz.iphone_usb_source as m
|
||||
from data_only_viz.state import PoseKp, State
|
||||
|
||||
|
||||
def test_start_keeps_retrying_when_no_device(monkeypatch):
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_connect():
|
||||
calls["n"] += 1
|
||||
return None # device never available during the test
|
||||
|
||||
monkeypatch.setattr(m, "connect_device", fake_connect)
|
||||
monkeypatch.setattr(m, "iter_frames", lambda s: iter(()))
|
||||
src = m.IphoneUSBSource()
|
||||
try:
|
||||
assert src.start() is True # does NOT give up on initial connect failure
|
||||
time.sleep(0.25) # let the read thread attempt a few reconnects
|
||||
assert calls["n"] >= 2, (
|
||||
"start() should keep retrying; connect_device called %d time(s)" % calls["n"])
|
||||
finally:
|
||||
src.release()
|
||||
|
||||
|
||||
def test_hands_cleared_on_stream_drop(monkeypatch):
|
||||
"""After a USB stream drop, persons_hands* lists must be emptied so the
|
||||
renderer does not freeze on the last known hand positions."""
|
||||
|
||||
class FakeSock:
|
||||
def close(self): pass
|
||||
def shutdown(self, *a): pass
|
||||
|
||||
connect_calls = [0]
|
||||
|
||||
def fake_connect():
|
||||
connect_calls[0] += 1
|
||||
if connect_calls[0] == 1:
|
||||
return FakeSock() # initial connection succeeds
|
||||
return None # reconnect fails; release() will interrupt the wait
|
||||
|
||||
def fake_iter_frames(sock):
|
||||
raise OSError("stream dropped") # simulate immediate disconnect
|
||||
|
||||
state = State()
|
||||
dummy_hand = [PoseKp(x=0.5, y=0.5, z=0.0, c=1.0)] * 21
|
||||
with state.lock():
|
||||
state.persons_hands = [dummy_hand]
|
||||
state.persons_hands_iphone = [dummy_hand]
|
||||
state.persons_hands_chirality = [1]
|
||||
|
||||
monkeypatch.setattr(m, "connect_device", fake_connect)
|
||||
monkeypatch.setattr(m, "iter_frames", fake_iter_frames)
|
||||
|
||||
src = m.IphoneUSBSource(state=state, write_hands=True)
|
||||
src.start()
|
||||
src.release() # joins thread; ensures clearing happened before we check
|
||||
|
||||
with state.lock():
|
||||
assert state.persons_hands == [], "persons_hands not cleared on drop"
|
||||
assert state.persons_hands_iphone == [], "persons_hands_iphone not cleared on drop"
|
||||
assert state.persons_hands_chirality == [], "persons_hands_chirality not cleared on drop"
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Pure (no-device) unit tests for iphone_usb_source helpers.
|
||||
|
||||
Tests _to_annexb, _decode_hands, and apply_skeleton_joints without any
|
||||
hardware or network connection.
|
||||
"""
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from data_only_viz.iphone_usb_source import (
|
||||
_to_annexb, _decode_hands, _HAND_BYTES,
|
||||
apply_skeleton_joints, _ARKIT_JOINTS,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _avcc(nals: list[bytes]) -> bytes:
|
||||
"""Build an AVCC-style buffer with 4-byte big-endian length prefixes."""
|
||||
out = bytearray()
|
||||
for nal in nals:
|
||||
out += struct.pack(">I", len(nal)) + nal
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def _make_hands_payload(
|
||||
hands: list[list[tuple[float, float, float]]],
|
||||
chiralities: list[int] | None = None,
|
||||
) -> bytes:
|
||||
"""Build a synthetic HandsPayload matching the wire layout.
|
||||
|
||||
count:u8, then per hand: chirality:u8 (1=right) + 21 × (x,y,z) big-endian f32.
|
||||
chiralities[i] sets the chirality byte for hand i (default 1=right for all).
|
||||
"""
|
||||
out = bytearray()
|
||||
out += bytes([len(hands)])
|
||||
for i, hand_kps in enumerate(hands):
|
||||
chir = chiralities[i] if chiralities is not None else 1
|
||||
out += bytes([chir])
|
||||
for x, y, z in hand_kps:
|
||||
out += struct.pack(">fff", x, y, z)
|
||||
return bytes(out)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _to_annexb
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_to_annexb_two_nals():
|
||||
nal1 = b"\x67\x01\x02\x03" # SPS-like
|
||||
nal2 = b"\x41\xAB\xCD"
|
||||
avcc = _avcc([nal1, nal2])
|
||||
result = _to_annexb(avcc)
|
||||
# First unit: Annex-B start code + nal1
|
||||
assert result[:4] == b"\x00\x00\x00\x01"
|
||||
assert result[4:4 + len(nal1)] == nal1
|
||||
# Second unit: Annex-B start code + nal2
|
||||
rest = result[4 + len(nal1):]
|
||||
assert rest[:4] == b"\x00\x00\x00\x01"
|
||||
assert rest[4:4 + len(nal2)] == nal2
|
||||
assert len(result) == 4 + len(nal1) + 4 + len(nal2)
|
||||
|
||||
|
||||
def test_to_annexb_payload_bytes_preserved():
|
||||
nal = bytes(range(32))
|
||||
result = _to_annexb(_avcc([nal]))
|
||||
assert result[4:] == nal
|
||||
|
||||
|
||||
def test_to_annexb_empty():
|
||||
assert _to_annexb(b"") == b""
|
||||
|
||||
|
||||
def test_to_annexb_truncated_nal_is_dropped():
|
||||
# Length prefix claims 10 bytes but only 3 are present
|
||||
buf = struct.pack(">I", 10) + b"\x00" * 3
|
||||
result = _to_annexb(buf)
|
||||
assert result == b""
|
||||
|
||||
|
||||
def test_to_annexb_single_nal():
|
||||
nal = b"\x65\xB8"
|
||||
result = _to_annexb(_avcc([nal]))
|
||||
assert result == b"\x00\x00\x00\x01" + nal
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _decode_hands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_decode_hands_empty_payload():
|
||||
assert _decode_hands(b"") is None
|
||||
|
||||
|
||||
def test_decode_hands_count_zero():
|
||||
assert _decode_hands(bytes([0])) == ([], [])
|
||||
|
||||
|
||||
def test_decode_hands_two_hands_length_and_coords():
|
||||
hand_a = [(float(i) * 0.1, float(i) * 0.2, float(i) * 0.05) for i in range(21)]
|
||||
hand_b = [(1.0 - float(i) * 0.03, 0.5, float(i) * 0.01) for i in range(21)]
|
||||
payload = _make_hands_payload([hand_a, hand_b])
|
||||
result = _decode_hands(payload)
|
||||
assert isinstance(result, tuple)
|
||||
hands, chirality = result
|
||||
assert isinstance(hands, list)
|
||||
assert len(hands) == 2
|
||||
assert len(chirality) == 2
|
||||
for hand in hands:
|
||||
assert len(hand) == 21
|
||||
# x/y spot-checks for hand A
|
||||
assert abs(hands[0][0].x - hand_a[0][0]) < 1e-5
|
||||
assert abs(hands[0][5].y - hand_a[5][1]) < 1e-5
|
||||
# x/y spot-checks for hand B
|
||||
assert abs(hands[1][3].x - hand_b[3][0]) < 1e-5
|
||||
assert abs(hands[1][20].y - hand_b[20][1]) < 1e-5
|
||||
|
||||
|
||||
def test_decode_hands_one_hand():
|
||||
kps = [(0.1 * i, 0.2 * i, 0.0) for i in range(21)]
|
||||
result = _decode_hands(_make_hands_payload([kps]))
|
||||
assert isinstance(result, tuple)
|
||||
hands, chirality = result
|
||||
assert isinstance(hands, list)
|
||||
assert len(hands) == 1
|
||||
assert len(hands[0]) == 21
|
||||
assert len(chirality) == 1
|
||||
|
||||
|
||||
def test_decode_hands_truncated_does_not_raise():
|
||||
hand_kps = [(0.1, 0.2, 0.3)] * 21
|
||||
full = _make_hands_payload([hand_kps, hand_kps])
|
||||
# Truncate to half — second hand will be incomplete
|
||||
truncated = full[:len(full) // 2]
|
||||
# Must not raise; result is either None or a (hands, chirality) tuple
|
||||
result = _decode_hands(truncated)
|
||||
assert result is None or isinstance(result, tuple)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task A: confidence + chirality (new tests added for TDD)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_decode_hands_count_zero_returns_tuple():
|
||||
"""count=0 payload must return ([], []), not a bare []."""
|
||||
result = _decode_hands(bytes([0]))
|
||||
assert result == ([], [])
|
||||
|
||||
|
||||
def test_decode_hands_chirality_and_confidence_roundtrip():
|
||||
"""Two hands with chirality [0,1] and in-range z values round-trip correctly.
|
||||
|
||||
.c must equal clamp(z, 0, 1); .z must equal raw wire z.
|
||||
"""
|
||||
hand_a = [(0.1 * i, 0.2 * i, 0.7) for i in range(21)] # z=0.7 in range
|
||||
hand_b = [(0.05 * i, 0.1 * i, 0.3) for i in range(21)] # z=0.3 in range
|
||||
payload = _make_hands_payload([hand_a, hand_b], chiralities=[0, 1])
|
||||
result = _decode_hands(payload)
|
||||
assert result is not None
|
||||
hands, chirality = result
|
||||
assert chirality == [0, 1]
|
||||
assert len(hands) == 2
|
||||
# hand A: z=0.7 → .z=0.7 raw, .c=0.7 clamped (same, in range)
|
||||
assert abs(hands[0][0].z - 0.7) < 1e-5
|
||||
assert abs(hands[0][0].c - 0.7) < 1e-5
|
||||
# hand B: z=0.3 → .z=0.3 raw, .c=0.3 clamped (same, in range)
|
||||
assert abs(hands[1][0].z - 0.3) < 1e-5
|
||||
assert abs(hands[1][0].c - 0.3) < 1e-5
|
||||
|
||||
|
||||
def test_decode_hands_confidence_clamped_out_of_range():
|
||||
"""z outside [0,1] clamps in .c but stays raw in .z."""
|
||||
hand_low = [(0.5, 0.5, -0.2)] * 21 # z below 0
|
||||
hand_high = [(0.5, 0.5, 1.7)] * 21 # z above 1
|
||||
payload = _make_hands_payload([hand_low, hand_high])
|
||||
result = _decode_hands(payload)
|
||||
assert result is not None
|
||||
hands, _ = result
|
||||
# hand_low: raw z preserved, c clamped to 0
|
||||
assert abs(hands[0][0].z - (-0.2)) < 1e-5
|
||||
assert hands[0][0].c == 0.0
|
||||
# hand_high: raw z preserved, c clamped to 1
|
||||
assert abs(hands[1][0].z - 1.7) < 1e-5
|
||||
assert hands[1][0].c == 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# apply_skeleton_joints — B4 tearing fix (chantier 2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_all_valid_joints(value: float = 0.1) -> list[tuple[float, float, float, bool]]:
|
||||
"""Return _ARKIT_JOINTS tuples all marked valid with a constant value."""
|
||||
return [(value, value, value, True)] * _ARKIT_JOINTS
|
||||
|
||||
|
||||
def _make_joints_with_mask(
|
||||
values: "list[tuple[float, float, float]]",
|
||||
valid_mask: "list[bool]",
|
||||
) -> list[tuple[float, float, float, bool]]:
|
||||
"""Zip (x,y,z) values with a validity mask into the joints wire format."""
|
||||
return [(x, y, z, v) for (x, y, z), v in zip(values, valid_mask)]
|
||||
|
||||
|
||||
def test_apply_skeleton_joints_fresh_pid_returns_correct_shape():
|
||||
"""prev=None → zeros array of shape (_ARKIT_JOINTS, 3), dtype float32."""
|
||||
joints = [(0.0, 0.0, 0.0, False)] * _ARKIT_JOINTS
|
||||
result = apply_skeleton_joints(None, joints)
|
||||
assert result.shape == (_ARKIT_JOINTS, 3)
|
||||
assert result.dtype == np.float32
|
||||
assert (result == 0).all()
|
||||
|
||||
|
||||
def test_apply_skeleton_joints_returns_new_array_each_call():
|
||||
"""Each call returns a different array object — never the same reference.
|
||||
|
||||
This is the core identity guarantee for the B4 tearing fix: readers that
|
||||
captured a reference to arr1 continue to see a consistent arr1 even while
|
||||
the writer has moved on to arr2.
|
||||
"""
|
||||
joints1 = _make_all_valid_joints(0.1)
|
||||
arr1 = apply_skeleton_joints(None, joints1)
|
||||
|
||||
joints2 = _make_all_valid_joints(0.9)
|
||||
arr2 = apply_skeleton_joints(arr1, joints2)
|
||||
|
||||
assert arr2 is not arr1
|
||||
|
||||
|
||||
def test_apply_skeleton_joints_preserves_values_not_valid_this_frame():
|
||||
"""Joints with valid=False in update 2 keep their values from update 1.
|
||||
|
||||
Update 1 sets all joints to their index value. Update 2 marks only
|
||||
joint 0 as valid with a new value. Joints 1..N-1 must be unchanged.
|
||||
"""
|
||||
# Update 1: every joint valid, joint i has x=float(i)
|
||||
joints1 = [(float(i), 0.0, 0.0, True) for i in range(_ARKIT_JOINTS)]
|
||||
arr1 = apply_skeleton_joints(None, joints1)
|
||||
|
||||
# Update 2: only joint 0 valid (new value), rest invalid
|
||||
vals2 = [(99.0, 99.0, 99.0)] * _ARKIT_JOINTS
|
||||
mask2 = [i == 0 for i in range(_ARKIT_JOINTS)]
|
||||
joints2 = _make_joints_with_mask(vals2, mask2)
|
||||
arr2 = apply_skeleton_joints(arr1, joints2)
|
||||
|
||||
# Joint 0: overwritten by update 2
|
||||
assert abs(arr2[0, 0] - 99.0) < 1e-5
|
||||
|
||||
# Joints 1..N-1: preserved from update 1
|
||||
for i in range(1, _ARKIT_JOINTS):
|
||||
assert abs(arr2[i, 0] - float(i)) < 1e-5, f"joint {i} not preserved"
|
||||
|
||||
|
||||
def test_apply_skeleton_joints_valid_joints_applied():
|
||||
"""Valid joints from update 2 overwrite the previous values correctly."""
|
||||
joints1 = _make_all_valid_joints(0.0)
|
||||
arr1 = apply_skeleton_joints(None, joints1)
|
||||
|
||||
joints2 = [(1.0, 2.0, 3.0, True)] * _ARKIT_JOINTS
|
||||
arr2 = apply_skeleton_joints(arr1, joints2)
|
||||
|
||||
assert abs(arr2[5, 0] - 1.0) < 1e-5
|
||||
assert abs(arr2[5, 1] - 2.0) < 1e-5
|
||||
assert abs(arr2[5, 2] - 3.0) < 1e-5
|
||||
|
||||
|
||||
def test_apply_skeleton_joints_wrong_shape_prev_discarded():
|
||||
"""prev_arr with wrong shape is discarded and output starts from zeros."""
|
||||
wrong_shape = np.ones((10, 3), dtype=np.float32)
|
||||
joints = [(0.5, 0.5, 0.5, True)] * _ARKIT_JOINTS
|
||||
result = apply_skeleton_joints(wrong_shape, joints)
|
||||
assert result.shape == (_ARKIT_JOINTS, 3)
|
||||
assert abs(result[0, 0] - 0.5) < 1e-5
|
||||
|
||||
|
||||
def test_apply_skeleton_joints_does_not_mutate_prev():
|
||||
"""apply_skeleton_joints must never modify the array passed as prev_arr."""
|
||||
joints1 = _make_all_valid_joints(1.0)
|
||||
arr1 = apply_skeleton_joints(None, joints1)
|
||||
arr1_copy = arr1.copy()
|
||||
|
||||
joints2 = _make_all_valid_joints(2.0)
|
||||
apply_skeleton_joints(arr1, joints2)
|
||||
|
||||
# arr1 must be byte-for-byte identical to what it was before the call
|
||||
np.testing.assert_array_equal(arr1, arr1_copy)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VIDEO_ROTATE normalized-point counterpart
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_rotate_norm_xy_matches_rot90():
|
||||
from data_only_viz.iphone_usb_source import rotate_norm_xy
|
||||
# np.rot90 k=1 (ccw): original (x, y) lands at (y, 1-x)
|
||||
assert rotate_norm_xy(0.2, 0.7, "ccw") == pytest.approx((0.7, 0.8))
|
||||
# cw (k=3): (x, y) -> (1-y, x)
|
||||
assert rotate_norm_xy(0.2, 0.7, "cw") == pytest.approx((0.3, 0.2))
|
||||
assert rotate_norm_xy(0.2, 0.7, "180") == pytest.approx((0.8, 0.3))
|
||||
assert rotate_norm_xy(0.2, 0.7, "none") == (0.2, 0.7)
|
||||
assert rotate_norm_xy(0.2, 0.7, "garbage") == (0.2, 0.7)
|
||||
|
||||
|
||||
def test_effective_point_rotate_mirror_conjugation():
|
||||
"""Video = flip THEN rotate; points = rotate THEN flip (renderer).
|
||||
Prove the conjugated mode lands points where the video pixel lands:
|
||||
flip_x(T(x,y)) must equal rot(flip_x(x,y)) for T = conjugate(rot)."""
|
||||
from data_only_viz.iphone_usb_source import (
|
||||
effective_point_rotate, rotate_norm_xy,
|
||||
)
|
||||
flip = lambda x, y: (1.0 - x, y)
|
||||
for video_mode in ("ccw", "cw", "180", "none"):
|
||||
pt_mode = effective_point_rotate(video_mode, mirror=True)
|
||||
for (x, y) in [(0.2, 0.7), (0.9, 0.1), (0.5, 0.5)]:
|
||||
video_dest = rotate_norm_xy(*flip(x, y), video_mode)
|
||||
point_dest = flip(*rotate_norm_xy(x, y, pt_mode))
|
||||
assert video_dest == pytest.approx(point_dest), (
|
||||
video_mode, x, y)
|
||||
# No mirror: modes pass through untouched
|
||||
assert effective_point_rotate("ccw", mirror=False) == "ccw"
|
||||
assert effective_point_rotate("none", mirror=True) == "none"
|
||||
@@ -80,8 +80,12 @@ def test_infer_latency_under_target():
|
||||
times.sort()
|
||||
median_ms = times[n // 2]
|
||||
print(f"median latency: {median_ms:.1f} ms (n={n})")
|
||||
# Target 50ms = 20fps. M5 bench shows ~29ms. Generous margin.
|
||||
assert median_ms < 80.0, f"median {median_ms:.1f}ms > 80ms target"
|
||||
# Full Multi-HMR CoreML on M5: ~120-140 ms standalone (7-8 fps),
|
||||
# see scripts/bench_multihmr_coreml.py and multihmr_coreml.py
|
||||
# docstring. The earlier 80 ms target was a backbone-only probe
|
||||
# estimate that does not hold for the full model. 250 ms gives
|
||||
# headroom for thermal/contention without masking a regression.
|
||||
assert median_ms < 250.0, f"median {median_ms:.1f}ms > 250ms target"
|
||||
|
||||
|
||||
def test_filter_threshold():
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Tests for the Multi-HMR remote TCP backend (client-side).
|
||||
|
||||
A loopback server is spun up in a background thread; it returns
|
||||
deterministic stub outputs so we can exercise the byte-for-byte
|
||||
protocol without depending on a live mlpackage.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from data_only_viz.multihmr_remote import (
|
||||
IMG_SIZE,
|
||||
MAGIC_REQ,
|
||||
MAGIC_RSP,
|
||||
N_PERSONS_FIXED,
|
||||
N_VERTS,
|
||||
REQ_PAYLOAD_LEN,
|
||||
RSP_PAYLOAD_LEN,
|
||||
MultiHMRRemoteBackend,
|
||||
decode_response,
|
||||
encode_request,
|
||||
)
|
||||
from data_only_viz.scripts.multihmr_server import (
|
||||
decode_request,
|
||||
encode_response,
|
||||
)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Pure protocol roundtrip tests (no socket, no model).
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def _make_K() -> np.ndarray:
|
||||
return np.array([[672.0, 0.0, 336.0],
|
||||
[0.0, 672.0, 336.0],
|
||||
[0.0, 0.0, 1.0]], dtype=np.float32)
|
||||
|
||||
|
||||
def test_request_encode_decode_roundtrip():
|
||||
rng = np.random.default_rng(0)
|
||||
img = rng.integers(0, 256, (IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)
|
||||
K = _make_K()
|
||||
req = encode_request(img, K)
|
||||
# length prefix + payload
|
||||
assert len(req) == 4 + REQ_PAYLOAD_LEN
|
||||
payload_len = struct.unpack("<I", req[:4])[0]
|
||||
assert payload_len == REQ_PAYLOAD_LEN
|
||||
assert req[4:8] == MAGIC_REQ
|
||||
img_back, K_back, _ = decode_request(req[4:])
|
||||
assert img_back.shape == (IMG_SIZE, IMG_SIZE, 3)
|
||||
np.testing.assert_array_equal(img_back, img)
|
||||
np.testing.assert_array_equal(K_back, K)
|
||||
|
||||
|
||||
def test_response_encode_decode_roundtrip():
|
||||
rng = np.random.default_rng(1)
|
||||
v3d = rng.standard_normal(
|
||||
(N_PERSONS_FIXED, N_VERTS, 3)).astype(np.float32)
|
||||
transl = rng.standard_normal(
|
||||
(N_PERSONS_FIXED, 1, 3)).astype(np.float32)
|
||||
scores = rng.random(N_PERSONS_FIXED).astype(np.float32)
|
||||
betas = rng.standard_normal(
|
||||
(N_PERSONS_FIXED, 10)).astype(np.float32)
|
||||
expr = rng.standard_normal(
|
||||
(N_PERSONS_FIXED, 10)).astype(np.float32)
|
||||
resp = encode_response(v3d, transl, scores, betas, expr, status=0)
|
||||
assert len(resp) == 4 + RSP_PAYLOAD_LEN
|
||||
assert resp[4:8] == MAGIC_RSP
|
||||
v3d2, transl2, scores2, betas2, expr2, status = decode_response(resp[4:])
|
||||
assert status == 0
|
||||
np.testing.assert_array_equal(v3d2, v3d)
|
||||
np.testing.assert_array_equal(transl2, transl)
|
||||
np.testing.assert_array_equal(scores2, scores)
|
||||
np.testing.assert_array_equal(betas2, betas)
|
||||
np.testing.assert_array_equal(expr2, expr)
|
||||
|
||||
|
||||
def test_request_rejects_wrong_dtype():
|
||||
img = np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.float32)
|
||||
with pytest.raises(ValueError):
|
||||
encode_request(img, _make_K())
|
||||
|
||||
|
||||
def test_request_rejects_wrong_shape():
|
||||
img = np.zeros((100, 100, 3), dtype=np.uint8)
|
||||
with pytest.raises(ValueError):
|
||||
encode_request(img, _make_K())
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Loopback mock server: full client.infer() end-to-end.
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
class _StubServer:
|
||||
"""Minimal TCP server that replies with deterministic stub outputs."""
|
||||
|
||||
def __init__(self, host: str = "127.0.0.1", port: int = 0) -> None:
|
||||
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
self.sock.bind((host, port))
|
||||
self.sock.listen(1)
|
||||
self.sock.settimeout(2.0)
|
||||
self.host, self.port = self.sock.getsockname()
|
||||
self._stop = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self.requests_seen: list[tuple[np.ndarray, np.ndarray]] = []
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
try:
|
||||
self.sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=2.0)
|
||||
|
||||
@staticmethod
|
||||
def _stub_outputs() -> bytes:
|
||||
# Distinct values per person so the test can assert order.
|
||||
v3d = np.zeros((N_PERSONS_FIXED, N_VERTS, 3), dtype=np.float32)
|
||||
transl = np.zeros((N_PERSONS_FIXED, 1, 3), dtype=np.float32)
|
||||
betas = np.zeros((N_PERSONS_FIXED, 10), dtype=np.float32)
|
||||
expr = np.zeros((N_PERSONS_FIXED, 10), dtype=np.float32)
|
||||
for k in range(N_PERSONS_FIXED):
|
||||
v3d[k] = float(k + 1)
|
||||
transl[k] = float(k + 1) * 0.1
|
||||
betas[k] = float(k + 1) * 0.01
|
||||
expr[k] = float(k + 1) * 0.02
|
||||
scores = np.array([0.9, 0.8, 0.2, 0.1], dtype=np.float32)
|
||||
return encode_response(v3d, transl, scores, betas, expr, status=0)
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
conn, _addr = self.sock.accept()
|
||||
except OSError:
|
||||
return
|
||||
conn.settimeout(2.0)
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
len_buf = conn.recv(4)
|
||||
except socket.timeout:
|
||||
continue
|
||||
if not len_buf or len(len_buf) < 4:
|
||||
return
|
||||
payload_len = struct.unpack("<I", len_buf)[0]
|
||||
buf = bytearray()
|
||||
while len(buf) < payload_len:
|
||||
chunk = conn.recv(payload_len - len(buf))
|
||||
if not chunk:
|
||||
return
|
||||
buf.extend(chunk)
|
||||
img, K, _ = decode_request(bytes(buf))
|
||||
self.requests_seen.append((img.copy(), K.copy()))
|
||||
conn.sendall(self._stub_outputs())
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_server():
|
||||
srv = _StubServer()
|
||||
srv.start()
|
||||
try:
|
||||
yield srv
|
||||
finally:
|
||||
srv.stop()
|
||||
|
||||
|
||||
def test_remote_backend_infer_against_stub(stub_server: _StubServer, monkeypatch):
|
||||
monkeypatch.setenv("MULTIHMR_REMOTE_ASYNC", "0") # sync path for determinism
|
||||
backend = MultiHMRRemoteBackend(
|
||||
host=stub_server.host, port=stub_server.port,
|
||||
connect_timeout=2.0, io_timeout=2.0)
|
||||
rng = np.random.default_rng(7)
|
||||
img = rng.random((3, IMG_SIZE, IMG_SIZE), dtype=np.float32)
|
||||
K = _make_K()
|
||||
humans = backend.infer(img, K, det_thresh=0.3)
|
||||
backend.close()
|
||||
|
||||
# det_thresh 0.3 keeps scores 0.9 and 0.8 only.
|
||||
assert len(humans) == 2
|
||||
h0 = humans[0]
|
||||
v = h0["v3d"].detach().cpu().numpy()
|
||||
assert v.shape == (N_VERTS, 3)
|
||||
assert float(v[0, 0]) == pytest.approx(1.0)
|
||||
t = h0["transl_pelvis"].detach().cpu().numpy()
|
||||
assert t.shape == (1, 3)
|
||||
assert float(h0["scores"].item()) == pytest.approx(0.9, abs=1e-5)
|
||||
assert h0["shape"].detach().cpu().numpy().shape == (10,)
|
||||
assert h0["expression"].detach().cpu().numpy().shape == (10,)
|
||||
|
||||
h1 = humans[1]
|
||||
assert float(h1["scores"].item()) == pytest.approx(0.8, abs=1e-5)
|
||||
|
||||
# Server saw exactly one request, and the image round-tripped as
|
||||
# uint8 (we lose [0,1] precision but the shape and dtype are right).
|
||||
assert len(stub_server.requests_seen) == 1
|
||||
img_seen, K_seen = stub_server.requests_seen[0]
|
||||
assert img_seen.shape == (IMG_SIZE, IMG_SIZE, 3)
|
||||
assert img_seen.dtype == np.uint8
|
||||
np.testing.assert_array_equal(K_seen, K)
|
||||
|
||||
|
||||
def test_remote_backend_threshold_filters_all(stub_server: _StubServer, monkeypatch):
|
||||
monkeypatch.setenv("MULTIHMR_REMOTE_ASYNC", "0") # sync path for determinism
|
||||
backend = MultiHMRRemoteBackend(
|
||||
host=stub_server.host, port=stub_server.port)
|
||||
img = np.zeros((3, IMG_SIZE, IMG_SIZE), dtype=np.float32)
|
||||
humans = backend.infer(img, _make_K(), det_thresh=1.5)
|
||||
backend.close()
|
||||
assert humans == []
|
||||
|
||||
|
||||
def test_is_available_returns_true_for_live_stub(stub_server: _StubServer):
|
||||
assert MultiHMRRemoteBackend.is_available(
|
||||
host=stub_server.host, port=stub_server.port) is True
|
||||
|
||||
|
||||
def test_is_available_false_for_dead_port():
|
||||
# Bind+release a port to get a guaranteed-closed one.
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.bind(("127.0.0.1", 0))
|
||||
_, port = s.getsockname()
|
||||
s.close()
|
||||
# Tiny race window: port may be reused by something else, but very
|
||||
# unlikely in unit-test scope.
|
||||
time.sleep(0.05)
|
||||
assert MultiHMRRemoteBackend.is_available(
|
||||
host="127.0.0.1", port=port) is False
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Tests for PoseSoundBridge.send_finger and send_pinch."""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from data_only_viz.finger_strike import StrikeEvent
|
||||
from data_only_viz.pose_bridge import PoseSoundBridge
|
||||
|
||||
|
||||
def _bridge() -> PoseSoundBridge:
|
||||
b = PoseSoundBridge()
|
||||
b._client = MagicMock() # stub the UDP client
|
||||
return b
|
||||
|
||||
|
||||
def test_send_finger_emits_expected_osc():
|
||||
b = _bridge()
|
||||
ev = StrikeEvent(hand=1, finger=2, strike_speed=0.5, z=0.1,
|
||||
tipx=0.7, tipy=0.6)
|
||||
b.send_finger(ev)
|
||||
b._client.send_message.assert_called_once()
|
||||
addr, args = b._client.send_message.call_args[0]
|
||||
assert addr == "/pose/finger"
|
||||
assert args[0] == 0 # pid
|
||||
assert args[1] == 1 # hand
|
||||
assert args[2] == 2 # finger
|
||||
assert abs(args[3] - 0.5) < 1e-6
|
||||
assert len(args) == 7
|
||||
# Verify z, tipx, tipy values
|
||||
assert abs(args[4] - 0.1) < 1e-6 # z
|
||||
assert abs(args[5] - 0.7) < 1e-6 # tipx
|
||||
assert abs(args[6] - 0.6) < 1e-6 # tipy
|
||||
# types matter for OSC tags / the SC ~fpResolve reader
|
||||
assert isinstance(args[0], int) and not isinstance(args[0], bool)
|
||||
assert isinstance(args[1], int) and isinstance(args[2], int)
|
||||
assert all(isinstance(a, float) for a in args[3:7])
|
||||
|
||||
|
||||
def test_send_pinch_emits_state_as_4th_arg():
|
||||
"""send_pinch must put the edge state (1=engage, 0=release) as 4th arg."""
|
||||
b = _bridge()
|
||||
|
||||
class _FakePinchEvent:
|
||||
hand = 0
|
||||
finger = 1
|
||||
state = 1
|
||||
|
||||
b.send_pinch(_FakePinchEvent())
|
||||
b._client.send_message.assert_called_once()
|
||||
addr, args = b._client.send_message.call_args[0]
|
||||
assert addr == "/pose/pinch"
|
||||
assert args[0] == 0 # pid
|
||||
assert args[1] == 0 # hand
|
||||
assert args[2] == 1 # finger
|
||||
assert args[3] == 1 # state (engage edge)
|
||||
assert len(args) == 4
|
||||
|
||||
|
||||
def test_send_pinch_release_state_is_zero():
|
||||
"""Release edge: state=0 appears as 4th arg."""
|
||||
b = _bridge()
|
||||
|
||||
class _FakeRelease:
|
||||
hand = 1
|
||||
finger = 2
|
||||
state = 0
|
||||
|
||||
b.send_pinch(_FakeRelease())
|
||||
addr, args = b._client.send_message.call_args[0]
|
||||
assert addr == "/pose/pinch"
|
||||
assert args[3] == 0 # state = release
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user