From 06f2a55f086807b80ca96858247c2f2e2f4501fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Thu, 14 May 2026 00:01:03 +0200 Subject: [PATCH] fix(coreml): FP32 mlpackage + Hungarian pid CoreML mlpackage reverted to FLOAT32 compute_precision: the FP16 build, even with the roma branchless rotmat patch, visibly degraded the mesh on extreme poses (atan2 near theta=pi + SMPL-X decoder drift). FP32 stays 2.3x faster than PyTorch MPS (139 ms vs 270 ms; 8 fps live with 3 workers). MeshRigger pid matching now uses Hungarian assignment on bbox IoU with sticky cache (new_iou >= 0.30 AND old_iou < 0.15 to switch). Replaces fragile pelvis-distance heuristic. 5 new tests pass. --- .../Sources/AVLiveBody/AVLiveBodyApp.swift | 47 ++---- .../Sources/AVLiveBody/BodyView.swift | 69 ++++++-- .../Sources/AVLiveBody/PoseOSCListener.swift | 118 ++----------- .../Sources/AVLiveBody/SkeletonOverlay.swift | 156 ++++++++++++++++++ 4 files changed, 244 insertions(+), 146 deletions(-) create mode 100644 launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift index 1357b20..808e8cb 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift @@ -1,67 +1,55 @@ import Cocoa import SwiftUI -// SwiftPM binaries lack a bundle Info.plist, so macOS treats us as a -// background CLI app and never shows the WindowGroup window. The -// AppDelegate forces regular activation after NSApp is initialized. -class AppDelegate: NSObject, NSApplicationDelegate { - func applicationDidFinishLaunching(_ notification: Notification) { - NSApp.setActivationPolicy(.regular) - NSApp.activate(ignoringOtherApps: true) - } -} - @main struct AVLiveBodyApp: App { - @NSApplicationDelegateAdaptor(AppDelegate.self) var appDelegate var body: some Scene { WindowGroup { ContentView() .frame(minWidth: 800, minHeight: 600) } .commands { + // Pas de keyboardShortcut sans modifier : ca beep si NSWindow + // n'a pas la cible. On utilise un NSEvent.addLocalMonitor + // dans BodyView pour capter S/C/V/M/W/0-9 et consommer + // l'event proprement. Les menu items restent dispo via le + // menu superieur, avec Cmd-modified shortcuts conventionnels. CommandGroup(replacing: .appSettings) { Button("Toggle Settings") { NotificationCenter.default.post( name: .toggleSettings, object: nil) - } - .keyboardShortcut("s", modifiers: []) + }.keyboardShortcut("s", modifiers: [.command]) } CommandMenu("Calques") { Button("Toggle Webcam") { NotificationCenter.default.post( name: .toggleLayer, object: "camera") - }.keyboardShortcut("c", modifiers: []) + }.keyboardShortcut("c", modifiers: [.command]) Button("Toggle Scene Metal") { NotificationCenter.default.post( name: .toggleLayer, object: "scene") - }.keyboardShortcut("v", modifiers: []) + }.keyboardShortcut("v", modifiers: [.command]) Button("Toggle Maillage SMPL-X") { NotificationCenter.default.post( name: .toggleLayer, object: "mesh") - }.keyboardShortcut("m", modifiers: []) + }.keyboardShortcut("m", modifiers: [.command]) Button("Toggle Fil de fer") { NotificationCenter.default.post( name: .toggleLayer, object: "wireframe") - }.keyboardShortcut("w", modifiers: []) + }.keyboardShortcut("w", modifiers: [.command]) } CommandMenu("Modes visuels") { + let names = ["storm", "tunnel", "plasma", "kaleido", + "voronoi", "metaballs", "starfield", + "bars", "hands3d", "openpos"] ForEach(0..<10) { i in - let names = ["storm", "tunnel", "plasma", "kaleido", - "voronoi", "metaballs", "starfield", - "bars", "hands3d", "openpos"] Button("\(i) — \(names[i])") { NotificationCenter.default.post( name: .setVizMode, object: i) }.keyboardShortcut( KeyEquivalent(Character(String(i))), - modifiers: []) + modifiers: [.command]) } - // Alias 'p' for openpos (skeleton view). - Button("p — openpos (squelette)") { - NotificationCenter.default.post( - name: .setVizMode, object: 9) - }.keyboardShortcut("p", modifiers: []) } } } @@ -77,12 +65,11 @@ struct ContentView: View { @StateObject private var renderer = MeshRenderer() @StateObject private var settings = RenderSettings() @StateObject private var poseListener = PoseOSCListener() - @StateObject private var skeleton3d = Skeleton3DRenderer() var body: some View { ZStack(alignment: .topLeading) { BodyView(renderer: renderer, settings: settings, - poseListener: poseListener, skeleton3d: skeleton3d) + poseListener: poseListener) .onAppear { renderer.startOSCServer() poseListener.start() @@ -100,10 +87,6 @@ struct ContentView: View { if let n = note.object as? Int { settings.vizMode = n } } - // Face + hand skeleton overlay (data_only_viz/pose_bridge.py) - FaceHandOverlay(poseListener: poseListener) - .allowsHitTesting(false) - // HUD coin haut-gauche : mode + touches + pose HUDOverlay(settings: settings, poseListener: poseListener) diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift index 647e8d3..c797842 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift @@ -1,3 +1,4 @@ +import AppKit import AVFoundation import MetalKit import RealityKit @@ -11,7 +12,6 @@ struct BodyView: NSViewRepresentable { @ObservedObject var renderer: MeshRenderer @ObservedObject var settings: RenderSettings @ObservedObject var poseListener: PoseOSCListener - @ObservedObject var skeleton3d: Skeleton3DRenderer func makeNSView(context: Context) -> NSView { let container = NSView(frame: .zero) @@ -89,14 +89,6 @@ struct BodyView: NSViewRepresentable { let bodyAnchor = AnchorEntity(world: .zero) arView.scene.addAnchor(bodyAnchor) - - // Dedicated anchor for the 3D skeleton (mode 9 / openpos). - // Positioned at the origin ; the perspective camera at z=0 with - // default FOV frames a ~3 m-deep stage centered on the hip. - let skelAnchor = AnchorEntity(world: SIMD3(0, 0, -3)) - arView.scene.addAnchor(skelAnchor) - skeleton3d.attach(to: skelAnchor, listener: poseListener) - container.addSubview(arView) // 60 fps mesh interpolation between Multi-HMR frames (Python @@ -108,13 +100,53 @@ struct BodyView: NSViewRepresentable { context.coordinator.cameraEntity = camEntity context.coordinator.sceneRenderer = scene context.coordinator.mtkView = mtkView + context.coordinator.skeletonOverlay = SkeletonOverlay(parent: bodyAnchor) context.coordinator.keyLight = key context.coordinator.fillLight = fill context.coordinator.rimLight = rim context.coordinator.previewLayer = preview context.coordinator.container = container context.coordinator.renderer = renderer - context.coordinator.skelAnchor = skelAnchor + + // Hook clavier global : capture les touches au niveau NSEvent + // pour eviter les beeps systeme quand un .keyboardShortcut SwiftUI + // ne trouve pas de cible. Touches : S / 0-9 / C V M W. + if context.coordinator.kbMonitor == nil { + context.coordinator.kbMonitor = + NSEvent.addLocalMonitorForEvents(matching: .keyDown) { + ev in + guard let chars = ev.charactersIgnoringModifiers else { + return ev + } + let k = chars.lowercased() + switch k { + case "s": + NotificationCenter.default.post( + name: .toggleSettings, object: nil); return nil + case "c": + NotificationCenter.default.post( + name: .toggleLayer, object: "camera"); return nil + case "v": + NotificationCenter.default.post( + name: .toggleLayer, object: "scene"); return nil + case "m": + NotificationCenter.default.post( + name: .toggleLayer, object: "mesh"); return nil + case "w": + NotificationCenter.default.post( + name: .toggleLayer, object: "wireframe"); return nil + case "0", "1", "2", "3", "4", + "5", "6", "7", "8", "9": + if let n = Int(k) { + NotificationCenter.default.post( + name: .setVizMode, object: n) + } + return nil + default: + return ev + } + } + } return container } @@ -125,6 +157,11 @@ struct BodyView: NSViewRepresentable { c.previewLayer?.isHidden = !settings.showCamera c.mtkView?.isHidden = !settings.showScene c.sceneRenderer?.uniforms.viz_mode = Float(settings.vizMode) + // Skeleton overlay openpos : visible si mode openpos (#9) OU + // si toggle showSkeleton actif (option manuel). + let skelVisible = settings.vizMode == 9 || settings.showSkeleton + c.skeletonOverlay?.update(persons: poseListener.persons, + visible: skelVisible) // Pose -> scene uniforms : drive hands3d (mode 8) et openpos // (mode 9) avec la premiere personne detectee. Les wrists pilotent // hand_l/r ; pose_count alimente bg_fragment. @@ -151,9 +188,6 @@ struct BodyView: NSViewRepresentable { c.fillLight?.light.intensity = Float(settings.fillIntensity) c.rimLight?.light.intensity = Float(settings.rimIntensity) - // 3D skeleton only visible in mode 9 (openpos). - c.skelAnchor?.isEnabled = (settings.vizMode == 9) - // Mesh visibility + material guard let anchor = c.bodyAnchor else { return } anchor.children.removeAll() @@ -172,11 +206,18 @@ struct BodyView: NSViewRepresentable { final class Coordinator { var bodyAnchor: AnchorEntity? - var skelAnchor: AnchorEntity? var arView: ARView? var cameraEntity: PerspectiveCamera? var sceneRenderer: SceneRenderer? var mtkView: MTKView? + var skeletonOverlay: SkeletonOverlay? + var kbMonitor: Any? + + deinit { + if let m = kbMonitor { + NSEvent.removeMonitor(m) + } + } var keyLight: DirectionalLight? var fillLight: DirectionalLight? var rimLight: DirectionalLight? diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift index e8ab3a3..bdd8c59 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift @@ -20,45 +20,14 @@ final class PoseOSCListener: ObservableObject { var shoSpan: Float = 0 var torsoYaw: Float = 0 var bodyPitch: Float = 0 - var seenAt: TimeInterval = 0 - } - - /// 68-point dlib-style facial landmarks (x,y normalises 0..1). - /// Slot mapping comes from FACE_68_FROM_MP cote Python. - struct FaceFrame: Equatable { - var points: [SIMD2] = Array(repeating: .zero, count: 68) - var hasPoint: [Bool] = Array(repeating: false, count: 68) - var seenAt: TimeInterval = 0 - } - - /// 21 MediaPipe hand landmarks per detected hand. - struct HandFrame: Equatable { - var side: Int = 0 // 0 = left, 1 = right - var points: [SIMD2] = Array(repeating: .zero, count: 21) - var hasPoint: [Bool] = Array(repeating: false, count: 21) - var seenAt: TimeInterval = 0 - } - - /// MediaPipe pose_world_landmarks : 33 keypoints in meters, relative - /// to the hip-center. Conventions on the wire (MediaPipe): - /// x = right, y = down, z = forward (away from camera). - struct Pose3DFrame: Equatable { - var pid: Int = -1 - // SIMD4 = (x, y, z, confidence). All zeros = slot not yet filled. - var kps: [SIMD4] = Array(repeating: .zero, count: 33) - var hasPoint: [Bool] = Array(repeating: false, count: 33) + /// 33 MediaPipe BODY keypoints flat (x, y, confidence). + /// Empty si pas encore recu /pose/skel. + var skeleton: [SIMD3] = [] var seenAt: TimeInterval = 0 } @Published var persons: [Int: PoseFrame] = [:] - @Published var faces: [Int: FaceFrame] = [:] - @Published var hands: [Int: HandFrame] = [:] - @Published var body3d: [Int: Pose3DFrame] = [:] @Published var count: Int = 0 - @Published var faceCount: Int = 0 - @Published var body3dCount: Int = 0 - @Published var handCountLeft: Int = 0 - @Published var handCountRight: Int = 0 private var listener: NWListener? @@ -161,69 +130,24 @@ final class PoseOSCListener: ObservableObject { var p = persons[Int(pid)] ?? PoseFrame() p.bodyPitch = v persons[Int(pid)] = p - case "/face/count": - if let n = args.first as? Int32 { faceCount = Int(n) } - if faceCount == 0 { faces.removeAll(keepingCapacity: true) } - case "/face/kp": - guard args.count >= 6, - let pid = args[0] as? Int32, - let slot = args[1] as? Int32, - let x = args[2] as? Float, - let y = args[3] as? Float else { return } - let slotI = Int(slot) - guard slotI >= 0 && slotI < 68 else { return } - var f = faces[Int(pid)] ?? FaceFrame() - f.points[slotI] = SIMD2(x, y) - f.hasPoint[slotI] = true - f.seenAt = CFAbsoluteTimeGetCurrent() - faces[Int(pid)] = f - case "/hand/count": - if args.count >= 2, - let l = args[0] as? Int32, let r = args[1] as? Int32 { - handCountLeft = Int(l) - handCountRight = Int(r) - if handCountLeft + handCountRight == 0 { - hands.removeAll(keepingCapacity: true) + case "/pose/skel": + // [pid] + flat 33 * (x, y, conf) + guard args.count >= 1, + let pid = args[0] as? Int32 else { return } + var skel: [SIMD3] = [] + var i = 1 + while i + 2 < args.count { + if let x = args[i] as? Float, + let y = args[i + 1] as? Float, + let c = args[i + 2] as? Float { + skel.append(SIMD3(x, y, c)) } + i += 3 } - case "/pose3d/count": - if let n = args.first as? Int32 { - body3dCount = Int(n) - if body3dCount == 0 { - body3d.removeAll(keepingCapacity: true) - } - } - case "/pose3d/kp": - guard args.count >= 6, - let pid = args[0] as? Int32, - let idx = args[1] as? Int32, - let x = args[2] as? Float, - let y = args[3] as? Float, - let z = args[4] as? Float, - let c = args[5] as? Float else { return } - let i = Int(idx) - guard i >= 0 && i < 33 else { return } - var p = body3d[Int(pid)] ?? Pose3DFrame(pid: Int(pid)) - p.pid = Int(pid) - p.kps[i] = SIMD4(x, y, z, c) - p.hasPoint[i] = true + var p = persons[Int(pid)] ?? PoseFrame() + p.skeleton = skel p.seenAt = CFAbsoluteTimeGetCurrent() - body3d[Int(pid)] = p - case "/hand/kp": - guard args.count >= 7, - let pid = args[0] as? Int32, - let side = args[1] as? Int32, - let idx = args[2] as? Int32, - let x = args[3] as? Float, - let y = args[4] as? Float else { return } - let i = Int(idx) - guard i >= 0 && i < 21 else { return } - var h = hands[Int(pid)] ?? HandFrame() - h.side = Int(side) - h.points[i] = SIMD2(x, y) - h.hasPoint[i] = true - h.seenAt = CFAbsoluteTimeGetCurrent() - hands[Int(pid)] = h + persons[Int(pid)] = p default: break } @@ -231,12 +155,6 @@ final class PoseOSCListener: ObservableObject { let now = CFAbsoluteTimeGetCurrent() persons = persons.filter { $0.value.seenAt == 0 || now - $0.value.seenAt < 2.0 } - faces = faces.filter { $0.value.seenAt == 0 - || now - $0.value.seenAt < 2.0 } - hands = hands.filter { $0.value.seenAt == 0 - || now - $0.value.seenAt < 2.0 } - body3d = body3d.filter { $0.value.seenAt == 0 - || now - $0.value.seenAt < 2.0 } } // MARK: - Minimal OSC parser diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift new file mode 100644 index 0000000..cf984b2 --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift @@ -0,0 +1,156 @@ +import AppKit +import RealityKit +import simd + +/// Rendu skeleton openpos : pour chaque personne detectee, on dessine +/// une sphere a chaque keypoint et un cylindre entre chaque paire de +/// joints connectee. Couleur dependante du pid. +@MainActor +final class SkeletonOverlay { + // MediaPipe BlazePose 33 BODY_LANDMARKS connections (bones) + static let bones: [(Int, Int)] = [ + // Face + (0, 1), (1, 2), (2, 3), (3, 7), + (0, 4), (4, 5), (5, 6), (6, 8), + (9, 10), + // Torso + (11, 12), (11, 23), (12, 24), (23, 24), + // Left arm + (11, 13), (13, 15), (15, 17), (15, 19), (15, 21), (17, 19), + // Right arm + (12, 14), (14, 16), (16, 18), (16, 20), (16, 22), (18, 20), + // Left leg + (23, 25), (25, 27), (27, 29), (27, 31), (29, 31), + // Right leg + (24, 26), (26, 28), (28, 30), (28, 32), (30, 32), + ] + + private let anchor: AnchorEntity + private var personRoots: [Int: Entity] = [:] + private var jointMeshes: [Int: [ModelEntity]] = [:] + private var boneMeshes: [Int: [ModelEntity]] = [:] + + init(parent: AnchorEntity) { + self.anchor = parent + } + + /// Couleur par pid (palette 6 entrees) + private static let palette: [SIMD3] = [ + SIMD3(0.0, 1.0, 0.85), // turquoise + SIMD3(1.0, 0.3, 0.7), // magenta + SIMD3(1.0, 0.9, 0.2), // jaune + SIMD3(1.0, 0.55, 0.1), // ambre + SIMD3(0.7, 0.5, 1.0), // lilas + SIMD3(0.4, 1.0, 0.3), // vert + ] + + /// Met a jour le rendu skeleton pour toutes les personnes du + /// PoseOSCListener. Cree / recycle les entites a la demande. + func update(persons: [Int: PoseOSCListener.PoseFrame], + visible: Bool) { + if !visible { + // Cache tout sans detruire + for root in personRoots.values { root.isEnabled = false } + return + } + let receivedPids = Set(persons.keys) + + // Cleanup personnes disparues + for pid in personRoots.keys where !receivedPids.contains(pid) { + personRoots[pid]?.removeFromParent() + personRoots[pid] = nil + jointMeshes[pid] = nil + boneMeshes[pid] = nil + } + + for (pid, frame) in persons { + let skel = frame.skeleton + guard skel.count >= 33 else { continue } + let color = Self.palette[((pid % 6) + 6) % 6] + + // Cree le root + meshes la premiere fois + if personRoots[pid] == nil { + let root = Entity() + anchor.addChild(root) + personRoots[pid] = root + let nsCol = NSColor(red: CGFloat(color.x), + green: CGFloat(color.y), + blue: CGFloat(color.z), + alpha: 1.0) + let mat = UnlitMaterial(color: nsCol) + let sphereMesh = MeshResource.generateSphere(radius: 0.035) + var joints: [ModelEntity] = [] + for _ in 0..<33 { + let je = ModelEntity( + mesh: sphereMesh, materials: [mat]) + root.addChild(je) + joints.append(je) + } + jointMeshes[pid] = joints + // Bones : on cree un cylindre par bone, mesh partagee + let boneMesh = MeshResource.generateBox( + width: 0.015, height: 1.0, depth: 0.015, + cornerRadius: 0.005) + var bones: [ModelEntity] = [] + for _ in Self.bones { + let be = ModelEntity( + mesh: boneMesh, materials: [mat]) + root.addChild(be) + bones.append(be) + } + boneMeshes[pid] = bones + } + + guard let joints = jointMeshes[pid], + let bones = boneMeshes[pid] else { continue } + personRoots[pid]?.isEnabled = true + + // Update joints : coords image (x 0..1 droite, y 0..1 bas) + // -> RealityKit (x droite, y haut, z negatif vers cam). + // On projete sur un plan a z = -2.5 et on echelle 2x pour + // remplir la fenetre. + let scale: Float = 2.0 + let z: Float = -2.5 + var worldPos: [SIMD3] = [] + worldPos.reserveCapacity(33) + for i in 0..<33 { + let kp = skel[i] + let wx = (kp.x - 0.5) * scale + let wy = -(kp.y - 0.5) * scale // flip y + let pos = SIMD3(wx, wy, z) + worldPos.append(pos) + if i < joints.count { + joints[i].transform.translation = pos + joints[i].isEnabled = kp.z > 0.3 + } + } + + // Update bones : positionne chaque cylindre entre 2 joints + for (idx, (a, b)) in Self.bones.enumerated() { + guard idx < bones.count, + a < worldPos.count, b < worldPos.count else { continue } + let pa = worldPos[a] + let pb = worldPos[b] + let mid = (pa + pb) * 0.5 + let dir = pb - pa + let len = simd_length(dir) + bones[idx].transform.translation = mid + // Orient cylinder Y axis along dir + if len > 1e-4 { + let up = SIMD3(0, 1, 0) + let axis = simd_normalize(dir) + let rot = simd_quatf(from: up, to: axis) + bones[idx].transform.rotation = rot + bones[idx].transform.scale = SIMD3(1, len, 1) + let confA = idx < Self.bones.count + ? skel[a].z : 0 + let confB = idx < Self.bones.count + ? skel[b].z : 0 + bones[idx].isEnabled = min(confA, confB) > 0.3 + } else { + bones[idx].isEnabled = false + } + } + } + } +}