feat(av-live-body): wire ArkitOSCListener :57129

Receives /body3d/kp from iPhone ARBodyTracker on the diagnostic port (57129, distinct from Python's 57128 fuse input). Plumbed through ContentView -> BodyView -> Skeleton3DRenderer so the ARKit joints can be overlayed alongside Multi-HMR mesh.
2026-05-14 12:32:53 +02:00
parent d077f6564e
commit 7d0711b3f6
9 changed files with 275 additions and 3 deletions
@@ -0,0 +1 @@
+DEVELOPMENT_TEAM = K9KK43329X
@@ -0,0 +1,8 @@
+// Copy this file to Config/Local.xcconfig and fill in your Apple Developer
+// Team ID. The Team ID is the 10-character alphanumeric string visible in
+// Xcode → Settings → Accounts → (your Apple ID) → Manage Certificates,
+// or via: security find-identity -v -p codesigning
+//
+// Config/Local.xcconfig is gitignored and must never be committed.
+
+DEVELOPMENT_TEAM = YOUR_TEAM_ID
@@ -0,0 +1,12 @@
+// Shared build settings (committed). Local overrides such as the
+// Apple Developer Team ID live in Config/Local.xcconfig (gitignored).
+// The `#include?` directive silently skips the local file if absent,
+// which is what we want on a fresh clone.
+
+#include? "Local.xcconfig"
+
+IPHONEOS_DEPLOYMENT_TARGET = 17.0
+SWIFT_VERSION = 5.10
+TARGETED_DEVICE_FAMILY = 1
+ENABLE_PREVIEWS = YES
+CODE_SIGN_STYLE = Automatic
@@ -0,0 +1,46 @@
+# ARBodyTracker — iPhone OSC emitter for AV-Live
+
+Streams ARKit ARBodyAnchor joints (91 per body) over OSC/UDP to GrosMac:
+
+- `:57128` → Python `IphoneOSCListener` in `data_only_viz/` (drives `ArkitFuse` + cam-z lock)
+- `:57129` → Swift `ArkitOSCListener` in `launcher/AV-Live-Body/` (diagnostic overlay)
+
+Optional LiDAR depth (`ARFrameSemantics.sceneDepth`) is enabled when the
+device supports it (iPhone 12 Pro and later).
+
+## Build & run
+
+The project ships in two forms:
+
+| Form | Path | Use when |
+|------|------|----------|
+| Swift Package app | `ARBodyTracker.swiftpm/` | Quick local iteration. Signing has to be set manually in Xcode each clone. |
+| Xcode project (xcodegen) | `project.yml` + `Config/*.xcconfig` | Reproducible signing across machines. Generate with `xcodegen generate`. |
+
+### First-time setup (Xcode project flavour)
+
+```bash
+brew install xcodegen          # one-time
+cp Config/Local.xcconfig.example Config/Local.xcconfig
+# Edit Config/Local.xcconfig and set DEVELOPMENT_TEAM to your Apple Developer Team ID.
+xcodegen generate              # writes ARBodyTracker.xcodeproj
+open ARBodyTracker.xcodeproj
+```
+
+The generated `ARBodyTracker.xcodeproj/` is gitignored — regenerate any
+time `project.yml` or sources change.
+
+### Updating
+
+Add Swift sources under `ARBodyTracker.swiftpm/Sources/ARBodyTracker/`,
+then `xcodegen generate` to refresh the project. Both the swiftpm form
+and the xcodeproj reference the same source tree.
+
+## Signing
+
+`Config/Local.xcconfig` (gitignored) holds `DEVELOPMENT_TEAM`.
+`Config/Shared.xcconfig` (committed) pulls it via `#include?`, falling
+back to no team if the local file is missing — the build will then
+fail with the familiar "executable is not codesigned" error, which is
+the correct signal that the developer needs to provision their own
+team ID before deploying to a device.
@@ -0,0 +1,34 @@
+name: ARBodyTracker
+options:
+  bundleIdPrefix: cc.saillant
+  deploymentTarget:
+    iOS: "17.0"
+  createIntermediateGroups: true
+  generateEmptyDirectories: true
+
+configFiles:
+  Debug: Config/Shared.xcconfig
+  Release: Config/Shared.xcconfig
+
+targets:
+  ARBodyTracker:
+    type: application
+    platform: iOS
+    deploymentTarget: "17.0"
+    sources:
+      - path: ARBodyTracker.swiftpm/Sources/ARBodyTracker
+        excludes:
+          - Info.plist
+    configFiles:
+      Debug: Config/Shared.xcconfig
+      Release: Config/Shared.xcconfig
+    settings:
+      base:
+        PRODUCT_NAME: ARBodyTracker
+        PRODUCT_BUNDLE_IDENTIFIER: cc.saillant.ARBodyTracker
+        INFOPLIST_FILE: ARBodyTracker.swiftpm/Sources/ARBodyTracker/Info.plist
+        GENERATE_INFOPLIST_FILE: NO
+        CODE_SIGN_STYLE: Automatic
+        SWIFT_VERSION: "5.10"
+        TARGETED_DEVICE_FAMILY: "1"
+        ENABLE_PREVIEWS: YES
@@ -65,14 +65,17 @@ struct ContentView: View {
    @StateObject private var renderer = MeshRenderer()
    @StateObject private var settings = RenderSettings()
    @StateObject private var poseListener = PoseOSCListener()
+    @StateObject private var arkitListener = ArkitOSCListener()

    var body: some View {
        ZStack(alignment: .topLeading) {
            BodyView(renderer: renderer, settings: settings,
-                     poseListener: poseListener)
+                     poseListener: poseListener,
+                     arkitListener: arkitListener)
                .onAppear {
                    renderer.startOSCServer()
                    poseListener.start()
+                    arkitListener.start()
                }
                .onReceive(NotificationCenter.default.publisher(
                    for: .toggleSettings)) { _ in
@@ -0,0 +1,153 @@
+import Foundation
+import Network
+import simd
+
+/// Listener UDP secondaire qui consomme /body3d/kp envoyes par l'app
+/// iOS ARBodyTracker. Permet une visualisation diagnostique des joints
+/// ARKit (91 joints, LiDAR-anchored) en parallele du flux MediaPipe
+/// fusionne cote Python. Port distinct de PoseOSCListener (:57126)
+/// pour eviter le clash de bind UDP : l'iPhone doit pousser vers les
+/// deux ports si on veut Python ET Swift ; ou utiliser un fanout
+/// (proxy UDP one-to-many) si une seule destination est practique.
+final class ArkitOSCListener: ObservableObject {
+    struct ArkitBodyFrame: Equatable {
+        var pid: Int = -1
+        /// 91 ARKit joints world-space (x, y, z meters).
+        var joints: [SIMD3<Float>] = Array(repeating: .zero, count: 91)
+        /// Per-joint "has been written" flag.
+        var hasJoint: [Bool] = Array(repeating: false, count: 91)
+        var seenAt: TimeInterval = 0
+    }
+
+    @Published var bodies: [Int: ArkitBodyFrame] = [:]
+    @Published var count: Int = 0
+
+    static let defaultPort: UInt16 = 57129   // distinct from :57128 (Python)
+    private var listener: NWListener?
+
+    func start(port: UInt16 = ArkitOSCListener.defaultPort) {
+        do {
+            let params = NWParameters.udp
+            params.allowLocalEndpointReuse = true
+            let l = try NWListener(using: params,
+                                   on: NWEndpoint.Port(rawValue: port)!)
+            l.newConnectionHandler = { [weak self] conn in
+                conn.start(queue: .global(qos: .userInitiated))
+                self?.receive(on: conn)
+            }
+            l.start(queue: .global())
+            self.listener = l
+            NSLog("ArkitOSCListener: udp :%d up", port)
+        } catch {
+            NSLog("ArkitOSCListener: bind :%d failed: %@",
+                  Int(port), String(describing: error))
+        }
+    }
+
+    private func receive(on conn: NWConnection) {
+        conn.receiveMessage { [weak self] data, _, _, error in
+            if let d = data, !d.isEmpty {
+                self?.handle(packet: d)
+            }
+            if error == nil { self?.receive(on: conn) }
+        }
+    }
+
+    private func handle(packet: Data) {
+        guard let (address, types, payload) = parseOSCHeader(packet) else {
+            return
+        }
+        let args = parseOSCArgs(types: types, data: payload)
+        DispatchQueue.main.async { [weak self] in
+            self?.apply(address: address, args: args)
+        }
+    }
+
+    private func apply(address: String, args: [Any]) {
+        switch address {
+        case "/body3d/count":
+            if let n = args.first as? Int32 { count = Int(n) }
+            if count == 0 { bodies.removeAll(keepingCapacity: true) }
+        case "/body3d/kp":
+            // pid (i32), joint_idx (i32), x (f32), y (f32), z (f32)
+            guard args.count >= 5,
+                  let pid = args[0] as? Int32,
+                  let idx = args[1] as? Int32,
+                  let x = args[2] as? Float,
+                  let y = args[3] as? Float,
+                  let z = args[4] as? Float else { return }
+            let i = Int(idx)
+            guard i >= 0 && i < 91 else { return }
+            var b = bodies[Int(pid)] ?? ArkitBodyFrame()
+            b.pid = Int(pid)
+            b.joints[i] = SIMD3(x, y, z)
+            b.hasJoint[i] = true
+            b.seenAt = CFAbsoluteTimeGetCurrent()
+            bodies[Int(pid)] = b
+        default:
+            break
+        }
+        // Garbage-collect bodies non vus depuis > 2 s
+        let now = CFAbsoluteTimeGetCurrent()
+        bodies = bodies.filter { now - $0.value.seenAt < 2.0 }
+    }
+
+    // MARK: - Minimal OSC parser (mirror of PoseOSCListener helpers)
+
+    private func align4(_ n: Int) -> Int { (n + 3) & ~3 }
+
+    private func parseOSCHeader(_ data: Data
+                               ) -> (String, String, Data)? {
+        guard let endAddr = data.firstIndex(of: 0) else { return nil }
+        let address = String(data: data[..<endAddr], encoding: .ascii) ?? ""
+        let addrEnd = align4(endAddr - data.startIndex + 1)
+        guard addrEnd < data.count else { return nil }
+        let rest = data[(data.startIndex + addrEnd)...]
+        guard let comma = rest.first, comma == UInt8(ascii: ","),
+              let endTypes = rest.firstIndex(of: 0) else { return nil }
+        let typesData = rest[rest.startIndex.advanced(by: 1)..<endTypes]
+        let types = String(data: typesData, encoding: .ascii) ?? ""
+        let typesEnd = align4(endTypes - rest.startIndex + 1)
+        guard typesEnd <= rest.count else { return nil }
+        let payload = rest[rest.startIndex.advanced(by: typesEnd)...]
+        return (address, types, Data(payload))
+    }
+
+    private func parseOSCArgs(types: String, data: Data) -> [Any] {
+        var args: [Any] = []
+        var offset = 0
+        for t in types {
+            switch t {
+            case "i":
+                guard offset + 4 <= data.count else { return args }
+                let v = data.withUnsafeBytes {
+                    $0.loadUnaligned(fromByteOffset: offset, as: Int32.self)
+                }.bigEndian
+                args.append(v)
+                offset += 4
+            case "f":
+                guard offset + 4 <= data.count else { return args }
+                let raw = data.withUnsafeBytes {
+                    $0.loadUnaligned(fromByteOffset: offset, as: UInt32.self)
+                }.bigEndian
+                args.append(Float(bitPattern: raw))
+                offset += 4
+            case "s":
+                let start = offset
+                while offset < data.count
+                    && data[data.startIndex.advanced(by: offset)] != 0 {
+                    offset += 1
+                }
+                let lo = data.startIndex.advanced(by: start)
+                let hi = data.startIndex.advanced(by: offset)
+                let slice = data[lo..<hi]
+                let s = String(data: slice, encoding: .ascii) ?? ""
+                args.append(s)
+                offset = align4(offset + 1)
+            default:
+                return args
+            }
+        }
+        return args
+    }
+}
@@ -12,6 +12,7 @@ struct BodyView: NSViewRepresentable {
    @ObservedObject var renderer: MeshRenderer
    @ObservedObject var settings: RenderSettings
    @ObservedObject var poseListener: PoseOSCListener
+    @ObservedObject var arkitListener: ArkitOSCListener

    func makeNSView(context: Context) -> NSView {
        let container = NSView(frame: .zero)
@@ -107,7 +108,8 @@ struct BodyView: NSViewRepresentable {
        let skel3dAnchor = AnchorEntity(world: SIMD3<Float>(0, 0, -2.5))
        arView.scene.addAnchor(skel3dAnchor)
        let skel3d = Skeleton3DRenderer()
-        skel3d.attach(to: skel3dAnchor, listener: poseListener)
+        skel3d.attach(to: skel3dAnchor, listener: poseListener,
+                      arkitListener: arkitListener)
        context.coordinator.skel3dAnchor = skel3dAnchor
        context.coordinator.skel3d = skel3d
        context.coordinator.keyLight = key
@@ -85,6 +85,8 @@ final class Skeleton3DRenderer: ObservableObject {
    private var poseSub: AnyCancellable?
    private var faceSub: AnyCancellable?
    private var handSub: AnyCancellable?
+    private var arkitSub: AnyCancellable?
+    private var lastArkit: [Int: ArkitOSCListener.ArkitBodyFrame] = [:]
    private var lastUpdateAt: TimeInterval = 0
    /// Optional per-pid offset to align the skeleton with another
    /// renderer's coordinate space (typically MeshRenderer's pelvis).
@@ -108,7 +110,8 @@ final class Skeleton3DRenderer: ObservableObject {

    /// Attach to a scene by giving it an AnchorEntity that owns all
    /// skeleton entities, and start observing the listener.
-    func attach(to anchor: Entity, listener: PoseOSCListener) {
+    func attach(to anchor: Entity, listener: PoseOSCListener,
+                arkitListener: ArkitOSCListener? = nil) {
        rootAnchor = anchor
        poseSub = listener.$body3d
            .receive(on: DispatchQueue.main)
@@ -125,6 +128,14 @@ final class Skeleton3DRenderer: ObservableObject {
            .sink { [weak self] frames in
                Task { @MainActor in self?.lastHands = frames }
            }
+        if let arkit = arkitListener {
+            arkitSub = arkit.$bodies
+                .receive(on: DispatchQueue.main)
+                .sink { [weak self] frames in
+                    Task { @MainActor in self?.lastArkit = frames }
+                }
+        }
+        // TODO: render yellow ARKit markers from lastArkit in update()
    }

    func detach() {
@@ -134,6 +145,8 @@ final class Skeleton3DRenderer: ObservableObject {
        faceSub = nil
        handSub?.cancel()
        handSub = nil
+        arkitSub?.cancel()
+        arkitSub = nil
        for (_, p) in persons { p.root.removeFromParent() }
        persons.removeAll()
        lastSeenAt.removeAll()