diff --git a/iphone-arbody/Config/Local.xcconfig b/iphone-arbody/Config/Local.xcconfig new file mode 100644 index 0000000..e3988a7 --- /dev/null +++ b/iphone-arbody/Config/Local.xcconfig @@ -0,0 +1 @@ +DEVELOPMENT_TEAM = K9KK43329X diff --git a/iphone-arbody/Config/Local.xcconfig.example b/iphone-arbody/Config/Local.xcconfig.example new file mode 100644 index 0000000..bb94c32 --- /dev/null +++ b/iphone-arbody/Config/Local.xcconfig.example @@ -0,0 +1,8 @@ +// Copy this file to Config/Local.xcconfig and fill in your Apple Developer +// Team ID. The Team ID is the 10-character alphanumeric string visible in +// Xcode → Settings → Accounts → (your Apple ID) → Manage Certificates, +// or via: security find-identity -v -p codesigning +// +// Config/Local.xcconfig is gitignored and must never be committed. + +DEVELOPMENT_TEAM = YOUR_TEAM_ID diff --git a/iphone-arbody/Config/Shared.xcconfig b/iphone-arbody/Config/Shared.xcconfig new file mode 100644 index 0000000..3f5e797 --- /dev/null +++ b/iphone-arbody/Config/Shared.xcconfig @@ -0,0 +1,12 @@ +// Shared build settings (committed). Local overrides such as the +// Apple Developer Team ID live in Config/Local.xcconfig (gitignored). +// The `#include?` directive silently skips the local file if absent, +// which is what we want on a fresh clone. + +#include? "Local.xcconfig" + +IPHONEOS_DEPLOYMENT_TARGET = 17.0 +SWIFT_VERSION = 5.10 +TARGETED_DEVICE_FAMILY = 1 +ENABLE_PREVIEWS = YES +CODE_SIGN_STYLE = Automatic diff --git a/iphone-arbody/README.md b/iphone-arbody/README.md new file mode 100644 index 0000000..42fd9e0 --- /dev/null +++ b/iphone-arbody/README.md @@ -0,0 +1,46 @@ +# ARBodyTracker — iPhone OSC emitter for AV-Live + +Streams ARKit ARBodyAnchor joints (91 per body) over OSC/UDP to GrosMac: + +- `:57128` → Python `IphoneOSCListener` in `data_only_viz/` (drives `ArkitFuse` + cam-z lock) +- `:57129` → Swift `ArkitOSCListener` in `launcher/AV-Live-Body/` (diagnostic overlay) + +Optional LiDAR depth (`ARFrameSemantics.sceneDepth`) is enabled when the +device supports it (iPhone 12 Pro and later). + +## Build & run + +The project ships in two forms: + +| Form | Path | Use when | +|------|------|----------| +| Swift Package app | `ARBodyTracker.swiftpm/` | Quick local iteration. Signing has to be set manually in Xcode each clone. | +| Xcode project (xcodegen) | `project.yml` + `Config/*.xcconfig` | Reproducible signing across machines. Generate with `xcodegen generate`. | + +### First-time setup (Xcode project flavour) + +```bash +brew install xcodegen # one-time +cp Config/Local.xcconfig.example Config/Local.xcconfig +# Edit Config/Local.xcconfig and set DEVELOPMENT_TEAM to your Apple Developer Team ID. +xcodegen generate # writes ARBodyTracker.xcodeproj +open ARBodyTracker.xcodeproj +``` + +The generated `ARBodyTracker.xcodeproj/` is gitignored — regenerate any +time `project.yml` or sources change. + +### Updating + +Add Swift sources under `ARBodyTracker.swiftpm/Sources/ARBodyTracker/`, +then `xcodegen generate` to refresh the project. Both the swiftpm form +and the xcodeproj reference the same source tree. + +## Signing + +`Config/Local.xcconfig` (gitignored) holds `DEVELOPMENT_TEAM`. +`Config/Shared.xcconfig` (committed) pulls it via `#include?`, falling +back to no team if the local file is missing — the build will then +fail with the familiar "executable is not codesigned" error, which is +the correct signal that the developer needs to provision their own +team ID before deploying to a device. diff --git a/iphone-arbody/project.yml b/iphone-arbody/project.yml new file mode 100644 index 0000000..1b4ce0d --- /dev/null +++ b/iphone-arbody/project.yml @@ -0,0 +1,34 @@ +name: ARBodyTracker +options: + bundleIdPrefix: cc.saillant + deploymentTarget: + iOS: "17.0" + createIntermediateGroups: true + generateEmptyDirectories: true + +configFiles: + Debug: Config/Shared.xcconfig + Release: Config/Shared.xcconfig + +targets: + ARBodyTracker: + type: application + platform: iOS + deploymentTarget: "17.0" + sources: + - path: ARBodyTracker.swiftpm/Sources/ARBodyTracker + excludes: + - Info.plist + configFiles: + Debug: Config/Shared.xcconfig + Release: Config/Shared.xcconfig + settings: + base: + PRODUCT_NAME: ARBodyTracker + PRODUCT_BUNDLE_IDENTIFIER: cc.saillant.ARBodyTracker + INFOPLIST_FILE: ARBodyTracker.swiftpm/Sources/ARBodyTracker/Info.plist + GENERATE_INFOPLIST_FILE: NO + CODE_SIGN_STYLE: Automatic + SWIFT_VERSION: "5.10" + TARGETED_DEVICE_FAMILY: "1" + ENABLE_PREVIEWS: YES diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift index 8f704c8..f7ca65b 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift @@ -65,14 +65,17 @@ struct ContentView: View { @StateObject private var renderer = MeshRenderer() @StateObject private var settings = RenderSettings() @StateObject private var poseListener = PoseOSCListener() + @StateObject private var arkitListener = ArkitOSCListener() var body: some View { ZStack(alignment: .topLeading) { BodyView(renderer: renderer, settings: settings, - poseListener: poseListener) + poseListener: poseListener, + arkitListener: arkitListener) .onAppear { renderer.startOSCServer() poseListener.start() + arkitListener.start() } .onReceive(NotificationCenter.default.publisher( for: .toggleSettings)) { _ in diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/ArkitOSCListener.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/ArkitOSCListener.swift new file mode 100644 index 0000000..29e2c18 --- /dev/null +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/ArkitOSCListener.swift @@ -0,0 +1,153 @@ +import Foundation +import Network +import simd + +/// Listener UDP secondaire qui consomme /body3d/kp envoyes par l'app +/// iOS ARBodyTracker. Permet une visualisation diagnostique des joints +/// ARKit (91 joints, LiDAR-anchored) en parallele du flux MediaPipe +/// fusionne cote Python. Port distinct de PoseOSCListener (:57126) +/// pour eviter le clash de bind UDP : l'iPhone doit pousser vers les +/// deux ports si on veut Python ET Swift ; ou utiliser un fanout +/// (proxy UDP one-to-many) si une seule destination est practique. +final class ArkitOSCListener: ObservableObject { + struct ArkitBodyFrame: Equatable { + var pid: Int = -1 + /// 91 ARKit joints world-space (x, y, z meters). + var joints: [SIMD3] = Array(repeating: .zero, count: 91) + /// Per-joint "has been written" flag. + var hasJoint: [Bool] = Array(repeating: false, count: 91) + var seenAt: TimeInterval = 0 + } + + @Published var bodies: [Int: ArkitBodyFrame] = [:] + @Published var count: Int = 0 + + static let defaultPort: UInt16 = 57129 // distinct from :57128 (Python) + private var listener: NWListener? + + func start(port: UInt16 = ArkitOSCListener.defaultPort) { + do { + let params = NWParameters.udp + params.allowLocalEndpointReuse = true + let l = try NWListener(using: params, + on: NWEndpoint.Port(rawValue: port)!) + l.newConnectionHandler = { [weak self] conn in + conn.start(queue: .global(qos: .userInitiated)) + self?.receive(on: conn) + } + l.start(queue: .global()) + self.listener = l + NSLog("ArkitOSCListener: udp :%d up", port) + } catch { + NSLog("ArkitOSCListener: bind :%d failed: %@", + Int(port), String(describing: error)) + } + } + + private func receive(on conn: NWConnection) { + conn.receiveMessage { [weak self] data, _, _, error in + if let d = data, !d.isEmpty { + self?.handle(packet: d) + } + if error == nil { self?.receive(on: conn) } + } + } + + private func handle(packet: Data) { + guard let (address, types, payload) = parseOSCHeader(packet) else { + return + } + let args = parseOSCArgs(types: types, data: payload) + DispatchQueue.main.async { [weak self] in + self?.apply(address: address, args: args) + } + } + + private func apply(address: String, args: [Any]) { + switch address { + case "/body3d/count": + if let n = args.first as? Int32 { count = Int(n) } + if count == 0 { bodies.removeAll(keepingCapacity: true) } + case "/body3d/kp": + // pid (i32), joint_idx (i32), x (f32), y (f32), z (f32) + guard args.count >= 5, + let pid = args[0] as? Int32, + let idx = args[1] as? Int32, + let x = args[2] as? Float, + let y = args[3] as? Float, + let z = args[4] as? Float else { return } + let i = Int(idx) + guard i >= 0 && i < 91 else { return } + var b = bodies[Int(pid)] ?? ArkitBodyFrame() + b.pid = Int(pid) + b.joints[i] = SIMD3(x, y, z) + b.hasJoint[i] = true + b.seenAt = CFAbsoluteTimeGetCurrent() + bodies[Int(pid)] = b + default: + break + } + // Garbage-collect bodies non vus depuis > 2 s + let now = CFAbsoluteTimeGetCurrent() + bodies = bodies.filter { now - $0.value.seenAt < 2.0 } + } + + // MARK: - Minimal OSC parser (mirror of PoseOSCListener helpers) + + private func align4(_ n: Int) -> Int { (n + 3) & ~3 } + + private func parseOSCHeader(_ data: Data + ) -> (String, String, Data)? { + guard let endAddr = data.firstIndex(of: 0) else { return nil } + let address = String(data: data[.. [Any] { + var args: [Any] = [] + var offset = 0 + for t in types { + switch t { + case "i": + guard offset + 4 <= data.count else { return args } + let v = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: Int32.self) + }.bigEndian + args.append(v) + offset += 4 + case "f": + guard offset + 4 <= data.count else { return args } + let raw = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: UInt32.self) + }.bigEndian + args.append(Float(bitPattern: raw)) + offset += 4 + case "s": + let start = offset + while offset < data.count + && data[data.startIndex.advanced(by: offset)] != 0 { + offset += 1 + } + let lo = data.startIndex.advanced(by: start) + let hi = data.startIndex.advanced(by: offset) + let slice = data[lo.. NSView { let container = NSView(frame: .zero) @@ -107,7 +108,8 @@ struct BodyView: NSViewRepresentable { let skel3dAnchor = AnchorEntity(world: SIMD3(0, 0, -2.5)) arView.scene.addAnchor(skel3dAnchor) let skel3d = Skeleton3DRenderer() - skel3d.attach(to: skel3dAnchor, listener: poseListener) + skel3d.attach(to: skel3dAnchor, listener: poseListener, + arkitListener: arkitListener) context.coordinator.skel3dAnchor = skel3dAnchor context.coordinator.skel3d = skel3d context.coordinator.keyLight = key diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift index 09932a2..7c64684 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift @@ -85,6 +85,8 @@ final class Skeleton3DRenderer: ObservableObject { private var poseSub: AnyCancellable? private var faceSub: AnyCancellable? private var handSub: AnyCancellable? + private var arkitSub: AnyCancellable? + private var lastArkit: [Int: ArkitOSCListener.ArkitBodyFrame] = [:] private var lastUpdateAt: TimeInterval = 0 /// Optional per-pid offset to align the skeleton with another /// renderer's coordinate space (typically MeshRenderer's pelvis). @@ -108,7 +110,8 @@ final class Skeleton3DRenderer: ObservableObject { /// Attach to a scene by giving it an AnchorEntity that owns all /// skeleton entities, and start observing the listener. - func attach(to anchor: Entity, listener: PoseOSCListener) { + func attach(to anchor: Entity, listener: PoseOSCListener, + arkitListener: ArkitOSCListener? = nil) { rootAnchor = anchor poseSub = listener.$body3d .receive(on: DispatchQueue.main) @@ -125,6 +128,14 @@ final class Skeleton3DRenderer: ObservableObject { .sink { [weak self] frames in Task { @MainActor in self?.lastHands = frames } } + if let arkit = arkitListener { + arkitSub = arkit.$bodies + .receive(on: DispatchQueue.main) + .sink { [weak self] frames in + Task { @MainActor in self?.lastArkit = frames } + } + } + // TODO: render yellow ARKit markers from lastArkit in update() } func detach() { @@ -134,6 +145,8 @@ final class Skeleton3DRenderer: ObservableObject { faceSub = nil handSub?.cancel() handSub = nil + arkitSub?.cancel() + arkitSub = nil for (_, p) in persons { p.root.removeFromParent() } persons.removeAll() lastSeenAt.removeAll()