diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift index 7bf93a3..298d67c 100644 --- a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift +++ b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift @@ -13,12 +13,27 @@ import SwiftUI /// LiDAR (sceneDepth + scene reconstruction mesh) is enabled when the /// device supports it (iPhone Pro / Pro Max). RGB-only fallback on /// non-LiDAR devices. +/// Lightweight 2D snapshot of the tracked skeleton, ready for SwiftUI +/// Canvas. Joint indices follow `ARSkeletonDefinition.defaultBody3D`. +struct SkeletonSnapshot: Equatable { + /// Projected joint positions in viewport coordinates, or nil if the + /// joint falls outside the view / is not finite. + let points: [CGPoint?] + /// Per-joint tracking flag (`ARSkeleton.isJointTracked`). + let tracked: [Bool] +} + @MainActor final class ARBodySession: NSObject, ObservableObject, ARSessionDelegate { @Published var running: Bool = false @Published var status: String = "idle" @Published var framesSent: Int = 0 @Published var jointsPerSec: Double = 0 + @Published var bodyCount: Int = 0 + @Published var skeleton2D: SkeletonSnapshot? + /// Set by the SwiftUI view via GeometryReader so the projection + /// matches the on-screen ARView size. + var viewportSize: CGSize = .zero private var host: String = "192.168.0.159" private var pythonPort: UInt16 = 57128 private var swiftPort: UInt16 = 57129 @@ -28,6 +43,8 @@ final class ARBodySession: NSObject, ObservableObject, ARSessionDelegate { private var lastFrameTime: TimeInterval = 0 private var jointsInSecond: Int = 0 private var lastSecond: TimeInterval = 0 + private let bodyParents: [Int] = + ARSkeletonDefinition.defaultBody3D.parentIndices let arView = ARView(frame: .zero) @@ -54,10 +71,10 @@ final class ARBodySession: NSObject, ObservableObject, ARSessionDelegate { } let cfg = ARBodyTrackingConfiguration() var feats: [String] = [] - if ARWorldTrackingConfiguration.supportsFrameSemantics(.sceneDepth) { - cfg.frameSemantics.insert(.sceneDepth) - feats.append("LiDAR depth") - } + // No extra frame semantics: `.sceneDepth` is reserved to + // ARWorldTracking, and `.personSegmentationWithDepth` is + // rejected per-frame by ABPKPersonIDTracker in this config + // (spams the console without producing usable depth). // NOTE: ARBodyTrackingConfiguration does not expose // sceneReconstruction (that's ARWorldTrackingConfiguration // territory). Env mesh capture requires a separate ARSession @@ -111,18 +128,22 @@ final class ARBodySession: NSObject, ObservableObject, ARSessionDelegate { if t - self.lastFrameTime < 1.0 / 30.0 { return } self.lastFrameTime = t - var bodyCount: Int = 0 + var count: Int = 0 + var firstBody: ARBodyAnchor? for anchor in frame.anchors { guard let body = anchor as? ARBodyAnchor else { continue } - self.publishJoints(pid: bodyCount, body: body) - bodyCount += 1 + self.publishJoints(pid: count, body: body) + if count == 0 { firstBody = body } + count += 1 } self.sendOSC(addr: "/body3d/count", - args: [.int32(Int32(bodyCount))]) + args: [.int32(Int32(count))]) self.framesSent &+= 1 + self.bodyCount = count + self.updateSkeleton2D(body: firstBody, camera: frame.camera) let now = Date().timeIntervalSinceReferenceDate - self.jointsInSecond &+= bodyCount * 91 + self.jointsInSecond &+= count * 91 if now - self.lastSecond >= 1.0 { self.jointsPerSec = Double(self.jointsInSecond) / max(0.001, now - self.lastSecond) @@ -132,6 +153,44 @@ final class ARBodySession: NSObject, ObservableObject, ARSessionDelegate { } } + private func currentInterfaceOrientation() -> UIInterfaceOrientation { + for scene in UIApplication.shared.connectedScenes { + if let ws = scene as? UIWindowScene { + return ws.interfaceOrientation + } + } + return .portrait + } + + private func updateSkeleton2D(body: ARBodyAnchor?, camera: ARCamera) { + guard let body, viewportSize.width > 1, viewportSize.height > 1 + else { + if skeleton2D != nil { skeleton2D = nil } + return + } + let xforms = body.skeleton.jointModelTransforms + let root = body.transform + let orient = currentInterfaceOrientation() + var pts: [CGPoint?] = Array(repeating: nil, count: xforms.count) + var tracked: [Bool] = Array(repeating: false, count: xforms.count) + for (i, m) in xforms.enumerated() { + let w = root * m + let p3 = simd_make_float3(w.columns.3.x, + w.columns.3.y, + w.columns.3.z) + let p2 = camera.projectPoint(p3, + orientation: orient, + viewportSize: viewportSize) + if p2.x.isFinite && p2.y.isFinite { pts[i] = p2 } + tracked[i] = body.skeleton.isJointTracked(i) + } + skeleton2D = SkeletonSnapshot(points: pts, tracked: tracked) + } + + /// Exposed for SwiftUI overlays that need to wire bone parent + /// indices without re-reading the ARKit skeleton definition. + var bodyParentIndices: [Int] { bodyParents } + private func publishJoints(pid: Int, body: ARBodyAnchor) { let skeleton = body.skeleton let transforms = skeleton.jointModelTransforms diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift index 844f356..44e5d7d 100644 --- a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift +++ b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift @@ -9,13 +9,26 @@ struct ContentView: View { @State private var swiftPort: String = "57129" // -> AVLiveBody ArkitOSCListener (diagnostic) @State private var sendEnvMesh: Bool = false + /// Replace the live ARView with a gradient placeholder. Camera is + /// unavailable inside Xcode previews; enable this flag there so the + /// panel + skeleton overlay can still be laid out. + var useMockBackground: Bool = false + /// Overlay a synthetic ARKit T-pose so the skeleton renderer can be + /// tuned without running on a device. + var useMockSkeleton: Bool = false + var body: some View { GeometryReader { geo in ZStack(alignment: .topLeading) { - ARViewContainer(session: session) + cameraBackground .ignoresSafeArea() - SkeletonOverlay(snapshot: session.skeleton2D, - parents: session.bodyParentIndices) + SkeletonOverlay( + snapshot: useMockSkeleton + ? SkeletonSnapshot.mockTPose(in: geo.size) + : session.skeleton2D, + parents: useMockSkeleton + ? SkeletonSnapshot.mockParents + : session.bodyParentIndices) .ignoresSafeArea() .allowsHitTesting(false) controlPanel @@ -27,6 +40,27 @@ struct ContentView: View { } } + @ViewBuilder + private var cameraBackground: some View { + if useMockBackground { + ZStack { + LinearGradient( + colors: [ + Color(red: 0.18, green: 0.20, blue: 0.24), + Color(red: 0.05, green: 0.05, blue: 0.08), + ], + startPoint: .top, + endPoint: .bottom) + Text("camera preview\n(unavailable in Xcode canvas)") + .font(.caption) + .multilineTextAlignment(.center) + .foregroundStyle(.white.opacity(0.35)) + } + } else { + ARViewContainer(session: session) + } + } + private var controlPanel: some View { VStack(alignment: .leading, spacing: 8) { Text("AR Body → AV-Live") @@ -86,6 +120,65 @@ struct ContentView: View { } } +extension SkeletonSnapshot { + /// Parent indices for the 16-joint preview stick figure. Each entry + /// is the parent joint index, or -1 for the root (head). + static let mockParents: [Int] = [ + -1, // 0 head + 0, // 1 neck + 1, // 2 lShoulder + 1, // 3 rShoulder + 2, // 4 lElbow + 3, // 5 rElbow + 4, // 6 lWrist + 5, // 7 rWrist + 1, // 8 spine + 8, // 9 pelvis + 9, // 10 lHip + 9, // 11 rHip + 10, // 12 lKnee + 11, // 13 rKnee + 12, // 14 lAnkle + 13, // 15 rAnkle + ] + + /// Synthetic 16-joint stick figure used by Xcode previews. ARKit + /// is not available in the preview canvas, so we cannot rely on + /// `ARSkeletonDefinition.neutralBodySkeleton3D` (returns nil). + static func mockTPose(in size: CGSize) -> SkeletonSnapshot { + // Normalized layout: origin at body center, +y down, ±1 spans + // roughly the full body height. + let layout: [CGPoint] = [ + CGPoint(x: 0.00, y: -0.45), + CGPoint(x: 0.00, y: -0.32), + CGPoint(x: -0.18, y: -0.30), + CGPoint(x: 0.18, y: -0.30), + CGPoint(x: -0.30, y: -0.12), + CGPoint(x: 0.30, y: -0.12), + CGPoint(x: -0.36, y: 0.08), + CGPoint(x: 0.36, y: 0.08), + CGPoint(x: 0.00, y: -0.10), + CGPoint(x: 0.00, y: 0.06), + CGPoint(x: -0.10, y: 0.09), + CGPoint(x: 0.10, y: 0.09), + CGPoint(x: -0.12, y: 0.30), + CGPoint(x: 0.12, y: 0.30), + CGPoint(x: -0.13, y: 0.46), + CGPoint(x: 0.13, y: 0.46), + ] + let scale = min(size.width * 0.9, size.height * 0.8) + let cx = size.width * 0.5 + let cy = size.height * 0.5 + let pts: [CGPoint?] = layout.map { + CGPoint(x: cx + $0.x * scale, + y: cy + $0.y * scale) + } + return SkeletonSnapshot( + points: pts, + tracked: Array(repeating: true, count: pts.count)) + } +} + /// Draws ARKit body joints + bones over the camera view. Bones are /// derived from `ARSkeletonDefinition.defaultBody3D` parent indices. struct SkeletonOverlay: View { @@ -120,3 +213,15 @@ struct SkeletonOverlay: View { } } } + +#Preview("iPhone 15 Pro — portrait") { + ContentView(useMockBackground: true, useMockSkeleton: true) +} + +#Preview("iPhone 15 Pro — landscape", traits: .landscapeLeft) { + ContentView(useMockBackground: true, useMockSkeleton: true) +} + +#Preview("Empty camera (no body)") { + ContentView(useMockBackground: true, useMockSkeleton: false) +}