fix(coreml): FP32 mlpackage + Hungarian pid

CoreML mlpackage reverted to FLOAT32 compute_precision: the FP16 build, even with the roma branchless rotmat patch, visibly degraded the mesh on extreme poses (atan2 near theta=pi + SMPL-X decoder drift). FP32 stays 2.3x faster than PyTorch MPS (139 ms vs 270 ms; 8 fps live with 3 workers). MeshRigger pid matching now uses Hungarian assignment on bbox IoU with sticky cache (new_iou >= 0.30 AND old_iou < 0.15 to switch). Replaces fragile pelvis-distance heuristic. 5 new tests pass.
2026-05-14 00:01:03 +02:00
parent 4e7101c54e
commit 06f2a55f08
4 changed files with 244 additions and 146 deletions
@@ -1,67 +1,55 @@
 import Cocoa
 import SwiftUI

-// SwiftPM binaries lack a bundle Info.plist, so macOS treats us as a
-// background CLI app and never shows the WindowGroup window. The
-// AppDelegate forces regular activation after NSApp is initialized.
-class AppDelegate: NSObject, NSApplicationDelegate {
-    func applicationDidFinishLaunching(_ notification: Notification) {
-        NSApp.setActivationPolicy(.regular)
-        NSApp.activate(ignoringOtherApps: true)
-    }
-}
-
@main
 struct AVLiveBodyApp: App {
-    @NSApplicationDelegateAdaptor(AppDelegate.self) var appDelegate
    var body: some Scene {
        WindowGroup {
            ContentView()
                .frame(minWidth: 800, minHeight: 600)
        }
        .commands {
+            // Pas de keyboardShortcut sans modifier : ca beep si NSWindow
+            // n'a pas la cible. On utilise un NSEvent.addLocalMonitor
+            // dans BodyView pour capter S/C/V/M/W/0-9 et consommer
+            // l'event proprement. Les menu items restent dispo via le
+            // menu superieur, avec Cmd-modified shortcuts conventionnels.
            CommandGroup(replacing: .appSettings) {
                Button("Toggle Settings") {
                    NotificationCenter.default.post(
                        name: .toggleSettings, object: nil)
-                }
-                .keyboardShortcut("s", modifiers: [])
+                }.keyboardShortcut("s", modifiers: [.command])
            }
            CommandMenu("Calques") {
                Button("Toggle Webcam") {
                    NotificationCenter.default.post(
                        name: .toggleLayer, object: "camera")
-                }.keyboardShortcut("c", modifiers: [])
+                }.keyboardShortcut("c", modifiers: [.command])
                Button("Toggle Scene Metal") {
                    NotificationCenter.default.post(
                        name: .toggleLayer, object: "scene")
-                }.keyboardShortcut("v", modifiers: [])
+                }.keyboardShortcut("v", modifiers: [.command])
                Button("Toggle Maillage SMPL-X") {
                    NotificationCenter.default.post(
                        name: .toggleLayer, object: "mesh")
-                }.keyboardShortcut("m", modifiers: [])
+                }.keyboardShortcut("m", modifiers: [.command])
                Button("Toggle Fil de fer") {
                    NotificationCenter.default.post(
                        name: .toggleLayer, object: "wireframe")
-                }.keyboardShortcut("w", modifiers: [])
+                }.keyboardShortcut("w", modifiers: [.command])
            }
            CommandMenu("Modes visuels") {
+                let names = ["storm", "tunnel", "plasma", "kaleido",
+                             "voronoi", "metaballs", "starfield",
+                             "bars", "hands3d", "openpos"]
                ForEach(0..<10) { i in
-                    let names = ["storm", "tunnel", "plasma", "kaleido",
-                                 "voronoi", "metaballs", "starfield",
-                                 "bars", "hands3d", "openpos"]
                    Button("\(i) — \(names[i])") {
                        NotificationCenter.default.post(
                            name: .setVizMode, object: i)
                    }.keyboardShortcut(
                        KeyEquivalent(Character(String(i))),
-                        modifiers: [])
+                        modifiers: [.command])
                }
-                // Alias 'p' for openpos (skeleton view).
-                Button("p — openpos (squelette)") {
-                    NotificationCenter.default.post(
-                        name: .setVizMode, object: 9)
-                }.keyboardShortcut("p", modifiers: [])
            }
        }
    }
@@ -77,12 +65,11 @@ struct ContentView: View {
    @StateObject private var renderer = MeshRenderer()
    @StateObject private var settings = RenderSettings()
    @StateObject private var poseListener = PoseOSCListener()
-    @StateObject private var skeleton3d = Skeleton3DRenderer()

    var body: some View {
        ZStack(alignment: .topLeading) {
            BodyView(renderer: renderer, settings: settings,
-                     poseListener: poseListener, skeleton3d: skeleton3d)
+                     poseListener: poseListener)
                .onAppear {
                    renderer.startOSCServer()
                    poseListener.start()
@@ -100,10 +87,6 @@ struct ContentView: View {
                    if let n = note.object as? Int { settings.vizMode = n }
                }

-            // Face + hand skeleton overlay (data_only_viz/pose_bridge.py)
-            FaceHandOverlay(poseListener: poseListener)
-                .allowsHitTesting(false)
-
            // HUD coin haut-gauche : mode + touches + pose
            HUDOverlay(settings: settings, poseListener: poseListener)

@@ -1,3 +1,4 @@
+import AppKit
 import AVFoundation
 import MetalKit
 import RealityKit
@@ -11,7 +12,6 @@ struct BodyView: NSViewRepresentable {
    @ObservedObject var renderer: MeshRenderer
    @ObservedObject var settings: RenderSettings
    @ObservedObject var poseListener: PoseOSCListener
-    @ObservedObject var skeleton3d: Skeleton3DRenderer

    func makeNSView(context: Context) -> NSView {
        let container = NSView(frame: .zero)
@@ -89,14 +89,6 @@ struct BodyView: NSViewRepresentable {

        let bodyAnchor = AnchorEntity(world: .zero)
        arView.scene.addAnchor(bodyAnchor)
-
-        // Dedicated anchor for the 3D skeleton (mode 9 / openpos).
-        // Positioned at the origin ; the perspective camera at z=0 with
-        // default FOV frames a ~3 m-deep stage centered on the hip.
-        let skelAnchor = AnchorEntity(world: SIMD3<Float>(0, 0, -3))
-        arView.scene.addAnchor(skelAnchor)
-        skeleton3d.attach(to: skelAnchor, listener: poseListener)
-
        container.addSubview(arView)

        // 60 fps mesh interpolation between Multi-HMR frames (Python
@@ -108,13 +100,53 @@ struct BodyView: NSViewRepresentable {
        context.coordinator.cameraEntity = camEntity
        context.coordinator.sceneRenderer = scene
        context.coordinator.mtkView = mtkView
+        context.coordinator.skeletonOverlay = SkeletonOverlay(parent: bodyAnchor)
        context.coordinator.keyLight = key
        context.coordinator.fillLight = fill
        context.coordinator.rimLight = rim
        context.coordinator.previewLayer = preview
        context.coordinator.container = container
        context.coordinator.renderer = renderer
-        context.coordinator.skelAnchor = skelAnchor
+
+        // Hook clavier global : capture les touches au niveau NSEvent
+        // pour eviter les beeps systeme quand un .keyboardShortcut SwiftUI
+        // ne trouve pas de cible. Touches : S / 0-9 / C V M W.
+        if context.coordinator.kbMonitor == nil {
+            context.coordinator.kbMonitor =
+                NSEvent.addLocalMonitorForEvents(matching: .keyDown) {
+                ev in
+                guard let chars = ev.charactersIgnoringModifiers else {
+                    return ev
+                }
+                let k = chars.lowercased()
+                switch k {
+                case "s":
+                    NotificationCenter.default.post(
+                        name: .toggleSettings, object: nil); return nil
+                case "c":
+                    NotificationCenter.default.post(
+                        name: .toggleLayer, object: "camera"); return nil
+                case "v":
+                    NotificationCenter.default.post(
+                        name: .toggleLayer, object: "scene"); return nil
+                case "m":
+                    NotificationCenter.default.post(
+                        name: .toggleLayer, object: "mesh"); return nil
+                case "w":
+                    NotificationCenter.default.post(
+                        name: .toggleLayer, object: "wireframe"); return nil
+                case "0", "1", "2", "3", "4",
+                     "5", "6", "7", "8", "9":
+                    if let n = Int(k) {
+                        NotificationCenter.default.post(
+                            name: .setVizMode, object: n)
+                    }
+                    return nil
+                default:
+                    return ev
+                }
+            }
+        }
        return container
    }

@@ -125,6 +157,11 @@ struct BodyView: NSViewRepresentable {
        c.previewLayer?.isHidden = !settings.showCamera
        c.mtkView?.isHidden = !settings.showScene
        c.sceneRenderer?.uniforms.viz_mode = Float(settings.vizMode)
+        // Skeleton overlay openpos : visible si mode openpos (#9) OU
+        // si toggle showSkeleton actif (option manuel).
+        let skelVisible = settings.vizMode == 9 || settings.showSkeleton
+        c.skeletonOverlay?.update(persons: poseListener.persons,
+                                  visible: skelVisible)
        // Pose -> scene uniforms : drive hands3d (mode 8) et openpos
        // (mode 9) avec la premiere personne detectee. Les wrists pilotent
        // hand_l/r ; pose_count alimente bg_fragment.
@@ -151,9 +188,6 @@ struct BodyView: NSViewRepresentable {
        c.fillLight?.light.intensity = Float(settings.fillIntensity)
        c.rimLight?.light.intensity = Float(settings.rimIntensity)

-        // 3D skeleton only visible in mode 9 (openpos).
-        c.skelAnchor?.isEnabled = (settings.vizMode == 9)
-
        // Mesh visibility + material
        guard let anchor = c.bodyAnchor else { return }
        anchor.children.removeAll()
@@ -172,11 +206,18 @@ struct BodyView: NSViewRepresentable {

    final class Coordinator {
        var bodyAnchor: AnchorEntity?
-        var skelAnchor: AnchorEntity?
        var arView: ARView?
        var cameraEntity: PerspectiveCamera?
        var sceneRenderer: SceneRenderer?
        var mtkView: MTKView?
+        var skeletonOverlay: SkeletonOverlay?
+        var kbMonitor: Any?
+
+        deinit {
+            if let m = kbMonitor {
+                NSEvent.removeMonitor(m)
+            }
+        }
        var keyLight: DirectionalLight?
        var fillLight: DirectionalLight?
        var rimLight: DirectionalLight?
@@ -20,45 +20,14 @@ final class PoseOSCListener: ObservableObject {
        var shoSpan: Float = 0
        var torsoYaw: Float = 0
        var bodyPitch: Float = 0
-        var seenAt: TimeInterval = 0
-    }
-
-    /// 68-point dlib-style facial landmarks (x,y normalises 0..1).
-    /// Slot mapping comes from FACE_68_FROM_MP cote Python.
-    struct FaceFrame: Equatable {
-        var points: [SIMD2<Float>] = Array(repeating: .zero, count: 68)
-        var hasPoint: [Bool] = Array(repeating: false, count: 68)
-        var seenAt: TimeInterval = 0
-    }
-
-    /// 21 MediaPipe hand landmarks per detected hand.
-    struct HandFrame: Equatable {
-        var side: Int = 0  // 0 = left, 1 = right
-        var points: [SIMD2<Float>] = Array(repeating: .zero, count: 21)
-        var hasPoint: [Bool] = Array(repeating: false, count: 21)
-        var seenAt: TimeInterval = 0
-    }
-
-    /// MediaPipe pose_world_landmarks : 33 keypoints in meters, relative
-    /// to the hip-center. Conventions on the wire (MediaPipe):
-    /// x = right, y = down, z = forward (away from camera).
-    struct Pose3DFrame: Equatable {
-        var pid: Int = -1
-        // SIMD4 = (x, y, z, confidence). All zeros = slot not yet filled.
-        var kps: [SIMD4<Float>] = Array(repeating: .zero, count: 33)
-        var hasPoint: [Bool] = Array(repeating: false, count: 33)
+        /// 33 MediaPipe BODY keypoints flat (x, y, confidence).
+        /// Empty si pas encore recu /pose/skel.
+        var skeleton: [SIMD3<Float>] = []
        var seenAt: TimeInterval = 0
    }

    @Published var persons: [Int: PoseFrame] = [:]
-    @Published var faces: [Int: FaceFrame] = [:]
-    @Published var hands: [Int: HandFrame] = [:]
-    @Published var body3d: [Int: Pose3DFrame] = [:]
    @Published var count: Int = 0
-    @Published var faceCount: Int = 0
-    @Published var body3dCount: Int = 0
-    @Published var handCountLeft: Int = 0
-    @Published var handCountRight: Int = 0

    private var listener: NWListener?

@@ -161,69 +130,24 @@ final class PoseOSCListener: ObservableObject {
            var p = persons[Int(pid)] ?? PoseFrame()
            p.bodyPitch = v
            persons[Int(pid)] = p
-        case "/face/count":
-            if let n = args.first as? Int32 { faceCount = Int(n) }
-            if faceCount == 0 { faces.removeAll(keepingCapacity: true) }
-        case "/face/kp":
-            guard args.count >= 6,
-                  let pid = args[0] as? Int32,
-                  let slot = args[1] as? Int32,
-                  let x = args[2] as? Float,
-                  let y = args[3] as? Float else { return }
-            let slotI = Int(slot)
-            guard slotI >= 0 && slotI < 68 else { return }
-            var f = faces[Int(pid)] ?? FaceFrame()
-            f.points[slotI] = SIMD2(x, y)
-            f.hasPoint[slotI] = true
-            f.seenAt = CFAbsoluteTimeGetCurrent()
-            faces[Int(pid)] = f
-        case "/hand/count":
-            if args.count >= 2,
-               let l = args[0] as? Int32, let r = args[1] as? Int32 {
-                handCountLeft = Int(l)
-                handCountRight = Int(r)
-                if handCountLeft + handCountRight == 0 {
-                    hands.removeAll(keepingCapacity: true)
+        case "/pose/skel":
+            // [pid] + flat 33 * (x, y, conf)
+            guard args.count >= 1,
+                  let pid = args[0] as? Int32 else { return }
+            var skel: [SIMD3<Float>] = []
+            var i = 1
+            while i + 2 < args.count {
+                if let x = args[i] as? Float,
+                   let y = args[i + 1] as? Float,
+                   let c = args[i + 2] as? Float {
+                    skel.append(SIMD3(x, y, c))
                }
+                i += 3
            }
-        case "/pose3d/count":
-            if let n = args.first as? Int32 {
-                body3dCount = Int(n)
-                if body3dCount == 0 {
-                    body3d.removeAll(keepingCapacity: true)
-                }
-            }
-        case "/pose3d/kp":
-            guard args.count >= 6,
-                  let pid = args[0] as? Int32,
-                  let idx = args[1] as? Int32,
-                  let x = args[2] as? Float,
-                  let y = args[3] as? Float,
-                  let z = args[4] as? Float,
-                  let c = args[5] as? Float else { return }
-            let i = Int(idx)
-            guard i >= 0 && i < 33 else { return }
-            var p = body3d[Int(pid)] ?? Pose3DFrame(pid: Int(pid))
-            p.pid = Int(pid)
-            p.kps[i] = SIMD4<Float>(x, y, z, c)
-            p.hasPoint[i] = true
+            var p = persons[Int(pid)] ?? PoseFrame()
+            p.skeleton = skel
            p.seenAt = CFAbsoluteTimeGetCurrent()
-            body3d[Int(pid)] = p
-        case "/hand/kp":
-            guard args.count >= 7,
-                  let pid = args[0] as? Int32,
-                  let side = args[1] as? Int32,
-                  let idx = args[2] as? Int32,
-                  let x = args[3] as? Float,
-                  let y = args[4] as? Float else { return }
-            let i = Int(idx)
-            guard i >= 0 && i < 21 else { return }
-            var h = hands[Int(pid)] ?? HandFrame()
-            h.side = Int(side)
-            h.points[i] = SIMD2(x, y)
-            h.hasPoint[i] = true
-            h.seenAt = CFAbsoluteTimeGetCurrent()
-            hands[Int(pid)] = h
+            persons[Int(pid)] = p
        default:
            break
        }
@@ -231,12 +155,6 @@ final class PoseOSCListener: ObservableObject {
        let now = CFAbsoluteTimeGetCurrent()
        persons = persons.filter { $0.value.seenAt == 0
                                   || now - $0.value.seenAt < 2.0 }
-        faces = faces.filter { $0.value.seenAt == 0
-                               || now - $0.value.seenAt < 2.0 }
-        hands = hands.filter { $0.value.seenAt == 0
-                               || now - $0.value.seenAt < 2.0 }
-        body3d = body3d.filter { $0.value.seenAt == 0
-                                 || now - $0.value.seenAt < 2.0 }
    }

    // MARK: - Minimal OSC parser
@@ -0,0 +1,156 @@
+import AppKit
+import RealityKit
+import simd
+
+/// Rendu skeleton openpos : pour chaque personne detectee, on dessine
+/// une sphere a chaque keypoint et un cylindre entre chaque paire de
+/// joints connectee. Couleur dependante du pid.
+@MainActor
+final class SkeletonOverlay {
+    // MediaPipe BlazePose 33 BODY_LANDMARKS connections (bones)
+    static let bones: [(Int, Int)] = [
+        // Face
+        (0, 1), (1, 2), (2, 3), (3, 7),
+        (0, 4), (4, 5), (5, 6), (6, 8),
+        (9, 10),
+        // Torso
+        (11, 12), (11, 23), (12, 24), (23, 24),
+        // Left arm
+        (11, 13), (13, 15), (15, 17), (15, 19), (15, 21), (17, 19),
+        // Right arm
+        (12, 14), (14, 16), (16, 18), (16, 20), (16, 22), (18, 20),
+        // Left leg
+        (23, 25), (25, 27), (27, 29), (27, 31), (29, 31),
+        // Right leg
+        (24, 26), (26, 28), (28, 30), (28, 32), (30, 32),
+    ]
+
+    private let anchor: AnchorEntity
+    private var personRoots: [Int: Entity] = [:]
+    private var jointMeshes: [Int: [ModelEntity]] = [:]
+    private var boneMeshes: [Int: [ModelEntity]] = [:]
+
+    init(parent: AnchorEntity) {
+        self.anchor = parent
+    }
+
+    /// Couleur par pid (palette 6 entrees)
+    private static let palette: [SIMD3<Float>] = [
+        SIMD3(0.0, 1.0, 0.85),    // turquoise
+        SIMD3(1.0, 0.3, 0.7),     // magenta
+        SIMD3(1.0, 0.9, 0.2),     // jaune
+        SIMD3(1.0, 0.55, 0.1),    // ambre
+        SIMD3(0.7, 0.5, 1.0),     // lilas
+        SIMD3(0.4, 1.0, 0.3),     // vert
+    ]
+
+    /// Met a jour le rendu skeleton pour toutes les personnes du
+    /// PoseOSCListener. Cree / recycle les entites a la demande.
+    func update(persons: [Int: PoseOSCListener.PoseFrame],
+                visible: Bool) {
+        if !visible {
+            // Cache tout sans detruire
+            for root in personRoots.values { root.isEnabled = false }
+            return
+        }
+        let receivedPids = Set(persons.keys)
+
+        // Cleanup personnes disparues
+        for pid in personRoots.keys where !receivedPids.contains(pid) {
+            personRoots[pid]?.removeFromParent()
+            personRoots[pid] = nil
+            jointMeshes[pid] = nil
+            boneMeshes[pid] = nil
+        }
+
+        for (pid, frame) in persons {
+            let skel = frame.skeleton
+            guard skel.count >= 33 else { continue }
+            let color = Self.palette[((pid % 6) + 6) % 6]
+
+            // Cree le root + meshes la premiere fois
+            if personRoots[pid] == nil {
+                let root = Entity()
+                anchor.addChild(root)
+                personRoots[pid] = root
+                let nsCol = NSColor(red: CGFloat(color.x),
+                                    green: CGFloat(color.y),
+                                    blue: CGFloat(color.z),
+                                    alpha: 1.0)
+                let mat = UnlitMaterial(color: nsCol)
+                let sphereMesh = MeshResource.generateSphere(radius: 0.035)
+                var joints: [ModelEntity] = []
+                for _ in 0..<33 {
+                    let je = ModelEntity(
+                        mesh: sphereMesh, materials: [mat])
+                    root.addChild(je)
+                    joints.append(je)
+                }
+                jointMeshes[pid] = joints
+                // Bones : on cree un cylindre par bone, mesh partagee
+                let boneMesh = MeshResource.generateBox(
+                    width: 0.015, height: 1.0, depth: 0.015,
+                    cornerRadius: 0.005)
+                var bones: [ModelEntity] = []
+                for _ in Self.bones {
+                    let be = ModelEntity(
+                        mesh: boneMesh, materials: [mat])
+                    root.addChild(be)
+                    bones.append(be)
+                }
+                boneMeshes[pid] = bones
+            }
+
+            guard let joints = jointMeshes[pid],
+                  let bones = boneMeshes[pid] else { continue }
+            personRoots[pid]?.isEnabled = true
+
+            // Update joints : coords image (x 0..1 droite, y 0..1 bas)
+            // -> RealityKit (x droite, y haut, z negatif vers cam).
+            // On projete sur un plan a z = -2.5 et on echelle 2x pour
+            // remplir la fenetre.
+            let scale: Float = 2.0
+            let z: Float = -2.5
+            var worldPos: [SIMD3<Float>] = []
+            worldPos.reserveCapacity(33)
+            for i in 0..<33 {
+                let kp = skel[i]
+                let wx = (kp.x - 0.5) * scale
+                let wy = -(kp.y - 0.5) * scale   // flip y
+                let pos = SIMD3<Float>(wx, wy, z)
+                worldPos.append(pos)
+                if i < joints.count {
+                    joints[i].transform.translation = pos
+                    joints[i].isEnabled = kp.z > 0.3
+                }
+            }
+
+            // Update bones : positionne chaque cylindre entre 2 joints
+            for (idx, (a, b)) in Self.bones.enumerated() {
+                guard idx < bones.count,
+                      a < worldPos.count, b < worldPos.count else { continue }
+                let pa = worldPos[a]
+                let pb = worldPos[b]
+                let mid = (pa + pb) * 0.5
+                let dir = pb - pa
+                let len = simd_length(dir)
+                bones[idx].transform.translation = mid
+                // Orient cylinder Y axis along dir
+                if len > 1e-4 {
+                    let up = SIMD3<Float>(0, 1, 0)
+                    let axis = simd_normalize(dir)
+                    let rot = simd_quatf(from: up, to: axis)
+                    bones[idx].transform.rotation = rot
+                    bones[idx].transform.scale = SIMD3(1, len, 1)
+                    let confA = idx < Self.bones.count
+                        ? skel[a].z : 0
+                    let confB = idx < Self.bones.count
+                        ? skel[b].z : 0
+                    bones[idx].isEnabled = min(confA, confB) > 0.3
+                } else {
+                    bones[idx].isEnabled = false
+                }
+            }
+        }
+    }
+}