feat(av-live): wireframe skel + face/hand filter

Skeleton3DRenderer now renders a wireframe: joint radius 1 mm (quasi-invisible), bone radius 3 mm (line-like). Replaces the chunky bead armature with a clean filaire silhouette covering body 33 joints + face 68 dlib + hands 21x2, all 3D. FaceHandOverlay 2D Canvas removed from ContentView -- face and hand landmarks now live in the same 3D RealityKit armature as the body skeleton (Skeleton3DRenderer.applyFace / applyHands, anchored on nose joint 0 + wrist joints 15/16). pose_filter.py extended with FaceFilterChain (alpha-beta + 30 ms lookahead) and HandFilterChain. multi.py wires them after the 2D smoothers, plus ghost rejection (POSE_GHOST_MIN_VISIBLE), bbox NMS (POSE_NMS_IOU), and pid hysteresis. 10 new tests, all green. CoreML perf audit (bench_multihmr_coreml.py): predict() = 99% of wall-time on FP32. ANE catastrophic for DINOv2 (1300 ms), INT8 weight quant = no live gain (GPU compute-bound). 6.4-6.8 fps live is the hardware ceiling on this model. quantize_multihmr_int8.py left in scripts/ for future trials.
2026-05-14 01:06:27 +02:00
parent c4bc5c2e5a
commit 91f4a46ceb
10 changed files with 1129 additions and 23 deletions
@@ -87,12 +87,9 @@ struct ContentView: View {
                    if let n = note.object as? Int { settings.vizMode = n }
                }

-            // Face + hand overlay 2D Canvas (68 dlib landmarks dont
-            // bouche slots 48-67 outerLips + 60-67 innerLips, plus 21
-            // landmarks par main, gauche=cyan droite=magenta). Source :
-            // /face/kp et /hand/kp depuis MediaPipe Holistic.
-            FaceHandOverlay(poseListener: poseListener)
-                .allowsHitTesting(false)
+            // Face + hand overlay 2D Canvas retire : les landmarks sont
+            // maintenant integres au squelette 3D RealityKit (cf.
+            // Skeleton3DRenderer.applyFace/applyHands).

            // HUD coin haut-gauche : mode + touches + pose
            HUDOverlay(settings: settings, poseListener: poseListener)
@@ -53,8 +53,14 @@ final class Skeleton3DRenderer: ObservableObject {
        }
    }

-    private static let jointRadius: Float = 0.045  // 4.5 cm — visible 3D depth
-    private static let boneRadius:  Float = 0.022  // 2.2 cm — chunky bones
+    // Wireframe pur : joints micro (1 mm, quasi invisibles), bones
+    // 2 mm = lignes fines. radius=0 fait planter MeshResource.generate.
+    private static let jointRadius: Float = 0.001   // 1 mm — quasi nul
+    private static let boneRadius:  Float = 0.003   // 3 mm — line-like
+    private static let faceJointRadius: Float = 0.001
+    private static let handJointRadius: Float = 0.001
+    private static let handScale3D: Float = 0.18      // typical hand size (m)
+    private static let faceForwardOffset: Float = 0.05 // push face in front of nose
    private static let minConfidence: Float = 0.3
    private static let retainSec: TimeInterval = 1.0

@@ -66,12 +72,19 @@ final class Skeleton3DRenderer: ObservableObject {
        var root: Entity
        var joints: [ModelEntity]     // 33 spheres
        var bones: [ModelEntity]      // 32 bone entities, same order as POSE_CONNECTIONS
+        var faceJoints: [ModelEntity]      // 68 dlib face landmarks
+        var leftHandJoints: [ModelEntity]  // 21 cyan
+        var rightHandJoints: [ModelEntity] // 21 magenta
    }

    private var persons: [Int: PersonEntities] = [:]
    private var lastSeenAt: [Int: TimeInterval] = [:]
+    private var lastFace: [Int: PoseOSCListener.FaceFrame] = [:]
+    private var lastHands: [Int: PoseOSCListener.HandFrame] = [:]
    private weak var rootAnchor: Entity?
    private var poseSub: AnyCancellable?
+    private var faceSub: AnyCancellable?
+    private var handSub: AnyCancellable?
    private var lastUpdateAt: TimeInterval = 0
    /// Optional per-pid offset to align the skeleton with another
    /// renderer's coordinate space (typically MeshRenderer's pelvis).
@@ -102,14 +115,30 @@ final class Skeleton3DRenderer: ObservableObject {
            .sink { [weak self] frames in
                Task { @MainActor in self?.update(frames: frames) }
            }
+        faceSub = listener.$faces
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] frames in
+                Task { @MainActor in self?.lastFace = frames }
+            }
+        handSub = listener.$hands
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] frames in
+                Task { @MainActor in self?.lastHands = frames }
+            }
    }

    func detach() {
        poseSub?.cancel()
        poseSub = nil
+        faceSub?.cancel()
+        faceSub = nil
+        handSub?.cancel()
+        handSub = nil
        for (_, p) in persons { p.root.removeFromParent() }
        persons.removeAll()
        lastSeenAt.removeAll()
+        lastFace.removeAll()
+        lastHands.removeAll()
    }

    // MARK: - Update
@@ -134,11 +163,12 @@ final class Skeleton3DRenderer: ObservableObject {
        for (pid, frame) in frames {
            let entities = persons[pid] ?? makePerson(pid: pid, parent: anchor)
            persons[pid] = entities
-            apply(frame: frame, to: entities)
+            apply(frame: frame, pid: pid, to: entities)
        }
    }

    private func apply(frame: PoseOSCListener.Pose3DFrame,
+                       pid: Int,
                       to entities: PersonEntities) {
        // Convert all 33 keypoints to RealityKit space once.
        var rk = [SIMD3<Float>](repeating: .zero, count: 33)
@@ -199,6 +229,108 @@ final class Skeleton3DRenderer: ObservableObject {
            bone.transform.scale = SIMD3<Float>(1, len, 1)
            bone.isEnabled = true
        }
+
+        // --- Face landmarks (68) anchored on nose joint rk[0] ---
+        applyFace(pid: pid, rk: rk, valid: valid, to: entities)
+
+        // --- Hand landmarks (21 left + 21 right) anchored on wrists ---
+        applyHands(rk: rk, valid: valid, to: entities)
+    }
+
+    private func applyFace(pid: Int,
+                           rk: [SIMD3<Float>],
+                           valid: [Bool],
+                           to entities: PersonEntities) {
+        guard let face = lastFace[pid] else {
+            for j in entities.faceJoints { j.isEnabled = false }
+            return
+        }
+        // Head width in 3D : distance between ears (rk[7] left ear,
+        // rk[8] right ear). Fall back to a sane default if missing.
+        let headWidth3D: Float
+        if valid[7] && valid[8] {
+            headWidth3D = max(0.10, simd_length(rk[7] - rk[8]))
+        } else {
+            headWidth3D = 0.18
+        }
+        // Compute 2D bbox width of face points + centroid.
+        var minX: Float =  .infinity, maxX: Float = -.infinity
+        var sumX: Float = 0, sumY: Float = 0
+        var n: Float = 0
+        for i in 0..<68 where face.hasPoint[i] {
+            let p = face.points[i]
+            if p.x < minX { minX = p.x }
+            if p.x > maxX { maxX = p.x }
+            sumX += p.x
+            sumY += p.y
+            n += 1
+        }
+        let nose = valid[0] ? rk[0] : SIMD3<Float>(0, 0, 0)
+        guard n > 4, maxX > minX else {
+            for j in entities.faceJoints { j.isEnabled = false }
+            return
+        }
+        let face2DWidth = max(maxX - minX, 1e-4)
+        let scale = headWidth3D / face2DWidth
+        let cx = sumX / n
+        let cy = sumY / n
+        for i in 0..<68 {
+            let j = entities.faceJoints[i]
+            if face.hasPoint[i] {
+                let dx = face.points[i].x - cx
+                let dy = face.points[i].y - cy
+                // Flip y : image y down -> RK y up. +z to push in front of nose.
+                j.transform.translation = nose + SIMD3<Float>(
+                    dx * scale, -dy * scale, Self.faceForwardOffset)
+                j.isEnabled = true
+            } else {
+                j.isEnabled = false
+            }
+        }
+    }
+
+    private func applyHands(rk: [SIMD3<Float>],
+                            valid: [Bool],
+                            to entities: PersonEntities) {
+        // Disable by default ; re-enable per side if a matching frame found.
+        for j in entities.leftHandJoints  { j.isEnabled = false }
+        for j in entities.rightHandJoints { j.isEnabled = false }
+
+        // Iterate cached hand frames (keyed by pid in OSC ; here we route
+        // purely by `side` since the python emits side=0/1 explicitly).
+        for (_, hand) in lastHands {
+            let isLeft = (hand.side == 0)
+            let wristIdx = isLeft ? 15 : 16
+            guard valid[wristIdx] else { continue }
+            let wrist = rk[wristIdx]
+            let target = isLeft
+                ? entities.leftHandJoints
+                : entities.rightHandJoints
+
+            // Centroid of valid hand points.
+            var sumX: Float = 0, sumY: Float = 0, n: Float = 0
+            for i in 0..<21 where hand.hasPoint[i] {
+                sumX += hand.points[i].x
+                sumY += hand.points[i].y
+                n += 1
+            }
+            guard n > 2 else { continue }
+            let cx = sumX / n
+            let cy = sumY / n
+            let scale = Self.handScale3D
+            for i in 0..<21 {
+                let j = target[i]
+                if hand.hasPoint[i] {
+                    let dx = hand.points[i].x - cx
+                    let dy = hand.points[i].y - cy
+                    j.transform.translation = wrist + SIMD3<Float>(
+                        dx * scale, -dy * scale, 0)
+                    j.isEnabled = true
+                } else {
+                    j.isEnabled = false
+                }
+            }
+        }
    }

    // MARK: - Construction
@@ -234,8 +366,49 @@ final class Skeleton3DRenderer: ObservableObject {
            root.addChild(e)
            bones.append(e)
        }
-        NSLog("Skeleton3DRenderer: spawned pid=%d (33 joints, %d bones)",
+        // Face sub-spheres (68 dlib landmarks, neutral light grey).
+        let faceMesh = MeshResource.generateSphere(
+            radius: Self.faceJointRadius)
+        let faceMat = SimpleMaterial(
+            color: NSColor(white: 0.85, alpha: 1.0),
+            roughness: 0.7, isMetallic: false)
+        var faceJoints: [ModelEntity] = []
+        faceJoints.reserveCapacity(68)
+        for _ in 0..<68 {
+            let e = ModelEntity(mesh: faceMesh, materials: [faceMat])
+            e.isEnabled = false
+            root.addChild(e)
+            faceJoints.append(e)
+        }
+
+        // Hand sub-spheres : left=cyan, right=magenta, 21 each.
+        let handMesh = MeshResource.generateSphere(
+            radius: Self.handJointRadius)
+        let leftMat = SimpleMaterial(
+            color: .systemCyan, roughness: 0.6, isMetallic: false)
+        let rightMat = SimpleMaterial(
+            color: .systemPink, roughness: 0.6, isMetallic: false)
+        var leftHand: [ModelEntity] = []
+        var rightHand: [ModelEntity] = []
+        leftHand.reserveCapacity(21)
+        rightHand.reserveCapacity(21)
+        for _ in 0..<21 {
+            let el = ModelEntity(mesh: handMesh, materials: [leftMat])
+            el.isEnabled = false
+            root.addChild(el)
+            leftHand.append(el)
+            let er = ModelEntity(mesh: handMesh, materials: [rightMat])
+            er.isEnabled = false
+            root.addChild(er)
+            rightHand.append(er)
+        }
+        NSLog("Skeleton3DRenderer: spawned pid=%d (33 joints, %d bones, 68 face, 21+21 hands)",
              pid, bones.count)
-        return PersonEntities(root: root, joints: joints, bones: bones)
+        return PersonEntities(root: root,
+                              joints: joints,
+                              bones: bones,
+                              faceJoints: faceJoints,
+                              leftHandJoints: leftHand,
+                              rightHandJoints: rightHand)
    }
 }