feat(ios): iphone ARBodyTracker swiftpm app
iOS 17+ Swift Package app (.swiftpm) streaming ARKit body joints via OSC UDP to two destinations: :57128 -> data_only_viz/iphone_osc_listener.py :57129 -> launcher/AV-Live-Body ArkitOSCListener.swift Features: - ARBodyTrackingConfiguration + sceneDepth (LiDAR) when supported - 91 joints per body, /body3d/kp pid joint_idx x y z - 30 fps throttle - SwiftUI UI: Host/Port fields, Start/Stop, live joints-per-second - Inline OSC encoder (no external dep) Env mesh (TCP :5500) NOT yet implemented; requires a separate ARWorldTrackingConfiguration session. ICP fusion path runs on bench data only until phase 2.
This commit is contained in:
L'électron rare
Generated
+7
@@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Workspace
|
||||
version = "1.0">
|
||||
<FileRef
|
||||
location = "self:">
|
||||
</FileRef>
|
||||
</Workspace>
|
||||
@@ -0,0 +1,17 @@
|
||||
// swift-tools-version:5.10
|
||||
import PackageDescription
|
||||
|
||||
let package = Package(
|
||||
name: "ARBodyTracker",
|
||||
defaultLocalization: "en",
|
||||
platforms: [.iOS(.v17)],
|
||||
products: [
|
||||
.executable(name: "ARBodyTracker", targets: ["ARBodyTracker"]),
|
||||
],
|
||||
targets: [
|
||||
.executableTarget(
|
||||
name: "ARBodyTracker",
|
||||
path: "Sources/ARBodyTracker"
|
||||
),
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,199 @@
|
||||
import ARKit
|
||||
import Combine
|
||||
import Foundation
|
||||
import Network
|
||||
import RealityKit
|
||||
import SwiftUI
|
||||
|
||||
/// Drives the ARKit body-tracking session and broadcasts joints to
|
||||
/// GrosMac via OSC UDP. Two destinations are supported simultaneously :
|
||||
/// - Python `IphoneOSCListener` on :57128 (drives ArkitFuse + cam-z lock)
|
||||
/// - Swift `ArkitOSCListener` on :57129 (diagnostic overlay in AVLiveBody)
|
||||
///
|
||||
/// LiDAR (sceneDepth + scene reconstruction mesh) is enabled when the
|
||||
/// device supports it (iPhone Pro / Pro Max). RGB-only fallback on
|
||||
/// non-LiDAR devices.
|
||||
@MainActor
|
||||
final class ARBodySession: NSObject, ObservableObject, ARSessionDelegate {
|
||||
@Published var running: Bool = false
|
||||
@Published var status: String = "idle"
|
||||
@Published var framesSent: Int = 0
|
||||
@Published var jointsPerSec: Double = 0
|
||||
private var host: String = "192.168.0.159"
|
||||
private var pythonPort: UInt16 = 57128
|
||||
private var swiftPort: UInt16 = 57129
|
||||
private var sendEnvMesh: Bool = false
|
||||
private let session = ARSession()
|
||||
private var conns: [NWConnection] = []
|
||||
private var lastFrameTime: TimeInterval = 0
|
||||
private var jointsInSecond: Int = 0
|
||||
private var lastSecond: TimeInterval = 0
|
||||
|
||||
let arView = ARView(frame: .zero)
|
||||
|
||||
override init() {
|
||||
super.init()
|
||||
arView.session = session
|
||||
arView.session.delegate = self
|
||||
arView.environment.background = .color(.black)
|
||||
arView.debugOptions = []
|
||||
}
|
||||
|
||||
func configure(host: String, pythonPort: UInt16, swiftPort: UInt16,
|
||||
sendEnvMesh: Bool) {
|
||||
self.host = host
|
||||
self.pythonPort = pythonPort
|
||||
self.swiftPort = swiftPort
|
||||
self.sendEnvMesh = sendEnvMesh
|
||||
}
|
||||
|
||||
func start() {
|
||||
guard ARBodyTrackingConfiguration.isSupported else {
|
||||
status = "ARBodyTracking unsupported (need A12+, iPhone XR/XS+)"
|
||||
return
|
||||
}
|
||||
let cfg = ARBodyTrackingConfiguration()
|
||||
var feats: [String] = []
|
||||
if ARWorldTrackingConfiguration.supportsFrameSemantics(.sceneDepth) {
|
||||
cfg.frameSemantics.insert(.sceneDepth)
|
||||
feats.append("LiDAR depth")
|
||||
}
|
||||
// NOTE: ARBodyTrackingConfiguration does not expose
|
||||
// sceneReconstruction (that's ARWorldTrackingConfiguration
|
||||
// territory). Env mesh capture requires a separate ARSession
|
||||
// with body tracking off — out of scope for this scaffold.
|
||||
if sendEnvMesh {
|
||||
feats.append("env-mesh: requires separate session (TODO)")
|
||||
}
|
||||
cfg.automaticImageScaleEstimationEnabled = true
|
||||
openUDP()
|
||||
session.run(cfg, options: [.resetTracking, .removeExistingAnchors])
|
||||
status = feats.isEmpty
|
||||
? "running (RGB only)"
|
||||
: "running (\(feats.joined(separator: ", ")))"
|
||||
running = true
|
||||
}
|
||||
|
||||
func stop() {
|
||||
session.pause()
|
||||
for c in conns { c.cancel() }
|
||||
conns.removeAll()
|
||||
running = false
|
||||
status = "stopped"
|
||||
}
|
||||
|
||||
// MARK: - UDP fanout
|
||||
|
||||
private func openUDP() {
|
||||
let ports: [UInt16] = [pythonPort, swiftPort]
|
||||
for p in ports where p != 0 {
|
||||
guard let nwPort = NWEndpoint.Port(rawValue: p) else { continue }
|
||||
let conn = NWConnection(
|
||||
to: .hostPort(host: NWEndpoint.Host(host), port: nwPort),
|
||||
using: .udp)
|
||||
conn.start(queue: .global(qos: .userInitiated))
|
||||
conns.append(conn)
|
||||
}
|
||||
}
|
||||
|
||||
private func sendDatagram(_ data: Data) {
|
||||
for c in conns {
|
||||
c.send(content: data, completion: .idempotent)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - ARSessionDelegate
|
||||
|
||||
nonisolated func session(_ s: ARSession, didUpdate frame: ARFrame) {
|
||||
let t = frame.timestamp
|
||||
Task { @MainActor in
|
||||
// Throttle to 30 fps max.
|
||||
if t - self.lastFrameTime < 1.0 / 30.0 { return }
|
||||
self.lastFrameTime = t
|
||||
|
||||
var bodyCount: Int = 0
|
||||
for anchor in frame.anchors {
|
||||
guard let body = anchor as? ARBodyAnchor else { continue }
|
||||
self.publishJoints(pid: bodyCount, body: body)
|
||||
bodyCount += 1
|
||||
}
|
||||
self.sendOSC(addr: "/body3d/count",
|
||||
args: [.int32(Int32(bodyCount))])
|
||||
self.framesSent &+= 1
|
||||
|
||||
let now = Date().timeIntervalSinceReferenceDate
|
||||
self.jointsInSecond &+= bodyCount * 91
|
||||
if now - self.lastSecond >= 1.0 {
|
||||
self.jointsPerSec = Double(self.jointsInSecond)
|
||||
/ max(0.001, now - self.lastSecond)
|
||||
self.jointsInSecond = 0
|
||||
self.lastSecond = now
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func publishJoints(pid: Int, body: ARBodyAnchor) {
|
||||
let skeleton = body.skeleton
|
||||
let transforms = skeleton.jointModelTransforms
|
||||
let root = body.transform
|
||||
for (idx, m) in transforms.enumerated() {
|
||||
let world = root * m
|
||||
sendOSC(addr: "/body3d/kp",
|
||||
args: [.int32(Int32(pid)),
|
||||
.int32(Int32(idx)),
|
||||
.float32(world.columns.3.x),
|
||||
.float32(world.columns.3.y),
|
||||
.float32(world.columns.3.z)])
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - OSC minimal encoder
|
||||
|
||||
enum OSCArg {
|
||||
case int32(Int32)
|
||||
case float32(Float)
|
||||
case string(String)
|
||||
}
|
||||
|
||||
private func sendOSC(addr: String, args: [OSCArg]) {
|
||||
var data = Data()
|
||||
appendOSCString(addr, into: &data)
|
||||
var types = ","
|
||||
for a in args {
|
||||
switch a {
|
||||
case .int32: types.append("i")
|
||||
case .float32: types.append("f")
|
||||
case .string: types.append("s")
|
||||
}
|
||||
}
|
||||
appendOSCString(types, into: &data)
|
||||
for a in args {
|
||||
switch a {
|
||||
case .int32(let v):
|
||||
var be = v.bigEndian
|
||||
withUnsafeBytes(of: &be) { data.append(contentsOf: $0) }
|
||||
case .float32(let v):
|
||||
var be = v.bitPattern.bigEndian
|
||||
withUnsafeBytes(of: &be) { data.append(contentsOf: $0) }
|
||||
case .string(let s):
|
||||
appendOSCString(s, into: &data)
|
||||
}
|
||||
}
|
||||
sendDatagram(data)
|
||||
}
|
||||
|
||||
private func appendOSCString(_ s: String, into data: inout Data) {
|
||||
let bytes = Array(s.utf8) + [0]
|
||||
data.append(contentsOf: bytes)
|
||||
let pad = (4 - data.count % 4) % 4
|
||||
if pad > 0 {
|
||||
data.append(contentsOf: [UInt8](repeating: 0, count: pad))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ARViewContainer: UIViewRepresentable {
|
||||
@ObservedObject var session: ARBodySession
|
||||
func makeUIView(context: Context) -> ARView { session.arView }
|
||||
func updateUIView(_ uiView: ARView, context: Context) {}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct ARBodyTrackerApp: App {
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
ContentView()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import SwiftUI
|
||||
import ARKit
|
||||
import RealityKit
|
||||
|
||||
struct ContentView: View {
|
||||
@StateObject private var session = ARBodySession()
|
||||
@State private var host: String = "192.168.0.159"
|
||||
@State private var pythonPort: String = "57128" // -> data_only_viz IphoneOSCListener
|
||||
@State private var swiftPort: String = "57129" // -> AVLiveBody ArkitOSCListener (diagnostic)
|
||||
@State private var sendEnvMesh: Bool = false
|
||||
|
||||
var body: some View {
|
||||
ZStack(alignment: .topLeading) {
|
||||
ARViewContainer(session: session)
|
||||
.ignoresSafeArea()
|
||||
VStack(alignment: .leading, spacing: 8) {
|
||||
Text("AR Body → AV-Live")
|
||||
.font(.headline)
|
||||
.foregroundColor(.white)
|
||||
HStack {
|
||||
Text("Host").foregroundColor(.white)
|
||||
TextField("GrosMac IP", text: $host)
|
||||
.keyboardType(.numbersAndPunctuation)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
}
|
||||
HStack {
|
||||
Text("Py").foregroundColor(.white)
|
||||
TextField("57128", text: $pythonPort)
|
||||
.keyboardType(.numberPad)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.frame(width: 70)
|
||||
Text("Swift").foregroundColor(.white)
|
||||
TextField("57129", text: $swiftPort)
|
||||
.keyboardType(.numberPad)
|
||||
.textFieldStyle(.roundedBorder)
|
||||
.frame(width: 70)
|
||||
}
|
||||
Toggle(isOn: $sendEnvMesh) {
|
||||
Text("Env mesh (LiDAR)").foregroundColor(.white)
|
||||
}
|
||||
HStack {
|
||||
Button(session.running ? "Stop" : "Start") {
|
||||
if session.running {
|
||||
session.stop()
|
||||
} else {
|
||||
session.configure(
|
||||
host: host,
|
||||
pythonPort: UInt16(pythonPort) ?? 57128,
|
||||
swiftPort: UInt16(swiftPort) ?? 57129,
|
||||
sendEnvMesh: sendEnvMesh)
|
||||
session.start()
|
||||
}
|
||||
}
|
||||
.buttonStyle(.borderedProminent)
|
||||
Spacer()
|
||||
Text(session.status)
|
||||
.font(.caption)
|
||||
.foregroundColor(.white)
|
||||
.padding(6)
|
||||
.background(.black.opacity(0.5))
|
||||
.cornerRadius(6)
|
||||
}
|
||||
Text("frames: \(session.framesSent) joints/s: \(Int(session.jointsPerSec))")
|
||||
.font(.caption2)
|
||||
.foregroundColor(.white)
|
||||
}
|
||||
.padding(12)
|
||||
.background(.black.opacity(0.5))
|
||||
.cornerRadius(10)
|
||||
.padding()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleDevelopmentRegion</key><string>en</string>
|
||||
<key>CFBundleDisplayName</key><string>ARBody Tracker</string>
|
||||
<key>CFBundleExecutable</key><string>$(EXECUTABLE_NAME)</string>
|
||||
<key>CFBundleIdentifier</key><string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key><string>6.0</string>
|
||||
<key>CFBundleName</key><string>ARBodyTracker</string>
|
||||
<key>CFBundlePackageType</key><string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key><string>0.1.0</string>
|
||||
<key>CFBundleVersion</key><string>1</string>
|
||||
<key>LSRequiresIPhoneOS</key><true/>
|
||||
<key>NSCameraUsageDescription</key>
|
||||
<string>Required for ARKit body tracking and LiDAR depth capture.</string>
|
||||
<key>UIApplicationSceneManifest</key>
|
||||
<dict>
|
||||
<key>UIApplicationSupportsMultipleScenes</key><false/>
|
||||
</dict>
|
||||
<key>UIRequiredDeviceCapabilities</key>
|
||||
<array>
|
||||
<string>arm64</string>
|
||||
<string>arkit</string>
|
||||
</array>
|
||||
<key>UIDeviceFamily</key>
|
||||
<array><integer>1</integer></array>
|
||||
<key>UIRequiresFullScreen</key><true/>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
<key>UILaunchScreen</key><dict/>
|
||||
</dict>
|
||||
</plist>
|
||||
Reference in New Issue
Block a user