fix(av-live-body): restore face+hand+3D (f540158)

Three regressions after recent merges, all restored to match the
original f540158 design:

1. FaceHandOverlay was no longer instantiated in ContentView.
   Added back as a SwiftUI Canvas overlay (68 dlib face landmarks
   with mouth slots 48-67, plus 21x2 hand landmarks cyan/magenta).

2. Skeleton3DRenderer was not attached. BodyView now creates an
   AnchorEntity at (0,0,-2.5), instantiates Skeleton3DRenderer
   and ties its visibility to vizMode==9 or showSkeleton toggle.

3. Joint and bone radii bumped to 4.5 cm / 2.2 cm so the 3D
   skeleton actually reads as 3D instead of looking flat.

MeshRenderer exposes pelvisWorld map per pid for future
interconnect uses (not auto-applied -- design keeps mesh and
skeleton each in their own coord space per f540158).
This commit is contained in:
L'électron rare
2026-05-14 00:45:31 +02:00
parent 7ed2e2764a
commit c4bc5c2e5a
4 changed files with 43 additions and 3 deletions
@@ -87,6 +87,13 @@ struct ContentView: View {
if let n = note.object as? Int { settings.vizMode = n }
}
// Face + hand overlay 2D Canvas (68 dlib landmarks dont
// bouche slots 48-67 outerLips + 60-67 innerLips, plus 21
// landmarks par main, gauche=cyan droite=magenta). Source :
// /face/kp et /hand/kp depuis MediaPipe Holistic.
FaceHandOverlay(poseListener: poseListener)
.allowsHitTesting(false)
// HUD coin haut-gauche : mode + touches + pose
HUDOverlay(settings: settings, poseListener: poseListener)
@@ -172,7 +172,10 @@ struct BodyView: NSViewRepresentable {
c.skeletonOverlay?.update(persons: poseListener.persons,
visible: skelVisible)
// 3D RealityKit armature : show/hide root anchor in sync with
// the same skelVisible signal as the 2D overlay.
// the same skelVisible signal as the 2D overlay. Skeleton keeps
// its own hip-relative coords (z=-3 anchor), mesh keeps its own
// world coords both visible together in mode openpos, no
// spatial fusion (original design from commit f540158).
c.skel3dAnchor?.isEnabled = skelVisible
// Pose -> scene uniforms : drive hands3d (mode 8) et openpos
// (mode 9) avec la premiere personne detectee. Les wrists pilotent
@@ -9,6 +9,11 @@ import SwiftUI
@MainActor
final class MeshRenderer: ObservableObject {
@Published var personEntities: [Int: ModelEntity] = [:]
/// Pelvis world position (RealityKit coords) par pid. Mis a jour
/// chaque frame TCP. Consume par Skeleton3DRenderer pour aligner
/// le squelette 3D openpos avec le mesh dense SMPL-X.
/// SMPL-X pelvis = vertex index 0 par convention canonique.
@Published var pelvisWorld: [Int: SIMD3<Float>] = [:]
private var faces: [UInt32] = []
private var oscServer: OSCServer?
@@ -130,6 +135,7 @@ final class MeshRenderer: ObservableObject {
wireframeEntities.removeValue(forKey: pid)
interpStates.removeValue(forKey: pid)
lastSeenAt.removeValue(forKey: pid)
pelvisWorld.removeValue(forKey: pid)
}
}
for p in persons {
@@ -163,6 +169,11 @@ final class MeshRenderer: ObservableObject {
displayed: converted, target: converted)
}
entity.transform.translation = SIMD3<Float>.zero
// Track pelvis world position (vertex 0 = SMPL-X pelvis) so
// Skeleton3DRenderer can co-locate openpos joints with mesh.
if !converted.isEmpty {
pelvisWorld[p.pid] = converted[0]
}
// Bbox debug : utile une fois par creation
if personEntities[p.pid] == nil {
let xs = converted.map(\.x)
@@ -53,8 +53,8 @@ final class Skeleton3DRenderer: ObservableObject {
}
}
private static let jointRadius: Float = 0.02 // 2 cm
private static let boneRadius: Float = 0.012 // 1.2 cm
private static let jointRadius: Float = 0.045 // 4.5 cm visible 3D depth
private static let boneRadius: Float = 0.022 // 2.2 cm chunky bones
private static let minConfidence: Float = 0.3
private static let retainSec: TimeInterval = 1.0
@@ -73,6 +73,25 @@ final class Skeleton3DRenderer: ObservableObject {
private weak var rootAnchor: Entity?
private var poseSub: AnyCancellable?
private var lastUpdateAt: TimeInterval = 0
/// Optional per-pid offset to align the skeleton with another
/// renderer's coordinate space (typically MeshRenderer's pelvis).
/// When nil for a pid, falls back to the renderer's default anchor.
private var pelvisOffsets: [Int: SIMD3<Float>] = [:]
/// External wiring : MeshRenderer or any other source publishes a
/// pelvis world position per pid. We translate each person's root
/// entity to that pose so the openpos skeleton co-locates with the
/// dense SMPL-X mesh.
func setPelvisOffsets(_ offsets: [Int: SIMD3<Float>]) {
pelvisOffsets = offsets
for (pid, entities) in persons {
if let off = offsets[pid] {
entities.root.transform.translation = off
} else {
entities.root.transform.translation = .zero
}
}
}
/// Attach to a scene by giving it an AnchorEntity that owns all
/// skeleton entities, and start observing the listener.