diff --git a/.gitignore b/.gitignore index 8abfdc7..da5b073 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,19 @@ launcher/**/xcuserdata/ launcher/**/*.xcuserstate launcher/**/Package.resolved +# iphone-arbody — xcodeproj is generated by xcodegen, Local.xcconfig +# carries personal Apple Developer Team ID. Both are local-only. +iphone-arbody/ARBodyTracker.xcodeproj/ +iphone-arbody/Config/Local.xcconfig +iphone-arbody/**/xcuserdata/ +iphone-arbody/**/*.xcuserstate +iphone-arbody/build/ +iphone-arbody/DerivedData/ + +# SwiftPM build + editor artifacts +.build/ +.swiftpm/ + # openFrameworks — on garde les shaders + settings.json pour qu'ils # arrivent sur les autres machines, mais on ignore les binaires. oscope-of/bin/* @@ -44,3 +57,10 @@ sound_algo/*.log .vscode/ *.swp *~ + +# tool session state (claude-mem / remember skill) +.remember/ + +# macOS / iCloud collision artifacts (auto-created on rename) +*\ 2 +*\ 2.* diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..d08bca6 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,83 @@ +# AGENTS.md + +Guidance for AI coding agents (Claude Code, Aider, Cursor, etc.) working in this repo. + +## Project + +`AV-Live` — live-coding audio-visual performance system: SuperCollider sound engine, openFrameworks visualiser driven by a Hantek 6022BL oscilloscope, and a SwiftUI menubar launcher orchestrating everything. Public, GPL-3. Repo `electron-rare/AV-Live`, branch `main`. Multi-host: GrosMac (source), macm1 (sink / Multi-HMR + Apple Vision ANE), iPhone 16 Pro (ARKit/LiDAR pub). + +## Tech stack (per sub-project) + +| Sub-project | Stack | +|-------------|-------| +| `sound_algo/` | SuperCollider (sclang + scsynth), 1099 SynthDefs, 345 tracks | +| `oscope-of/` | openFrameworks C++, libusb (Hantek bulk), GLSL 150 / GL 3.2 core | +| `launcher/` | SwiftUI menubar app, Swift Package Manager | +| `data_only_viz/` | Python 3.11+ via `uv`, native Metal (pyobjc), multi-backend pose | +| `data_feeds/` | Python data ingestion | +| `web_realart/` | Node.js, Express, OSC bridge | +| `avlivebody-mac/` | SwiftUI body-tracking client (ARKit/SMPL-X mesh, ad-hoc signed for local dev) | +| `iphone-arbody/` | iOS app, ARBodyTracker, publishes `/body3d/kp` via OSC | + +## Commands + +```bash +# Python sub-projects (uv only) +cd data_only_viz && uv sync && uv run python -m data_only_viz +cd data_feeds && uv sync + +# openFrameworks +cd oscope-of && make -j + +# Web bridge +cd web_realart && npm install && npm start + +# Swift +open launcher/Package.swift # or xcodebuild from CLI +open avlivebody-mac/avlivebody.xcodeproj +``` + +## Conventions + +- Commits: subject ≤ 50 chars, body ≤ 72, no underscore in scope, no AI attribution, never `--no-verify` (hooks enforce). +- Branches: `feat/`, `fix/`, `docs/`, `refactor/`, `chore/`. +- Language: French to the user, English in code/comments/commits. +- No emojis in code/docs/commits unless explicitly requested. +- Python: **always `uv`** (never pip/poetry/conda directly). +- `.gitignore` already excludes `*.pt`, `*.ckpt`, `*.safetensors`, `*.mlpackage` at root — don't commit weights. +- License: GPL-3 (whole repo) — keep new files under a compatible license header when adding third-party code. + +## File layout + +- `sound_algo/` — SC sound engine (own `CLAUDE.md`) +- `oscope-of/` — visualiser +- `launcher/` — macOS menubar +- `data_only_viz/` — pose / mesh / body tracking pipeline (Metal) +- `data_feeds/` — data ingestion +- `web_realart/` — web UI + OSC bridge +- `avlivebody-mac/`, `iphone-arbody/` — body-tracking clients +- `shared/` — cross-sub-project assets +- `third_party/` — vendored deps (CHECK before adding to root deps) +- `tools/` — helper scripts +- `docs/superpowers/plans/` — in-flight plans/specs +- `AV-Live-corrupted-20260514/` — quarantined corrupted snapshot, do not touch + +## Domain-specific gotchas + +- **mDNS hostnames are required** (`grosmac.local`, `supra-m1.local`) for `AVBODY_HOST` / `MULTIHMR_REMOTE_HOST`. They resist DHCP changes (iPhone hotspot reassigns 172.20.10.x routinely). +- **`POSE_FILTER` chain ordering is load-bearing**: default is `median+kalman+lookahead+ik`. Extras must be inserted at the right stage — `one_euro_joints` BEFORE kalman, `one_euro_bones` AFTER SMPL-X fusion in `multi.py`. `arkit_fuse` overrides 14 body slots with ARKit ARSkeleton3D from iOS app via `/body3d/kp` on `:57128` (always-on listener). +- **`ICP_FUSION=1`** requires `ICP_LIDAR_HOST` (iPhone IP), `ICP_LIDAR_PORT` (default 5500, iPhone ARMesh TCP), and an extrinsic JSON at `~/.config/av-live/lidar_extrinsic.json`. See `docs/ICP_FUSION.md`. +- **iPhone OSC port `57128`** is hardcoded as the publish target for `/body3d/kp` — don't reassign. +- **`avlivebody-mac` requires ad-hoc signing for local dev** (fixed in `85589f2`). Don't strip the signing identity. +- **`onVideoFrame` retain cycle in avlivebody** was fixed in `3b5f29e` — when adding new frame callbacks, mind the strong-self capture. +- **AVLive-Body legacy** has been archived (`9e1482e`); the canonical client is `avlivebody-mac`. Don't reintroduce paths to the old project. +- **macm1 = sink** (Multi-HMR CoreML + Apple Vision ANE + SMPL-X TCP); GrosMac = source. Mind the direction when wiring new OSC topics. +- **Each major sub-project has its own `CLAUDE.md`** — closest wins. Put cross-cutting rules here, sub-project specifics in the nested file. + +## When in doubt + +- Read root `CLAUDE.md` and the nested `CLAUDE.md` of the sub-project you're editing. +- Recent commits: `git log --oneline -20`. +- Plans: `docs/superpowers/plans/`. +- Cluster context: `~/CLAUDE.md` (GrosMac / macm1 / iPhone topology). +- For sound: read `sound_algo/CLAUDE.md` before touching SynthDefs. diff --git a/CLAUDE.md b/CLAUDE.md index 11cdd7c..582f4b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,6 +27,27 @@ Toujours répondre en français à l'utilisateur. Code, commentaires de code, co | Bridge web / UI de live coding | `web_realart/` | | Plans / specs en cours | `docs/superpowers/plans/` | +## Network topology + +| Host | mDNS | IP (DHCP) | Role | +|------|------|-----------|------| +| GrosMac M5 | `grosmac.local` | LAN | Source + visualisation (AVLiveBody + data_only_viz + data_feeds) | +| macm1 M1 Max | `supra-m1.local` | `192.168.0.175` | Sink (Multi-HMR CoreML + Apple Vision ANE + SMPL-X TCP) | +| iPhone 16 Pro | (Personal Hotspot) | DHCP | ARKit/LiDAR pub via OSC `/body3d/kp` | + +`AVBODY_HOST` / `MULTIHMR_REMOTE_HOST` accept mDNS hostnames — résiste aux changements DHCP (notamment iPhone hotspot 172.20.10.x). + +## Environment variables + +| Env | Default | Effect | +|-----|---------|--------| +| `POSE_FILTER` | `median+kalman+lookahead+ik` | filter chain stages — extra: `one_euro_joints` (joint-space CHI 2012 One Euro, inserted before kalman), `one_euro_bones` (bone-vector One Euro applied after SMPL-X fusion in multi.py), `arkit_fuse` (overrides 14 body slots with ARKit ARSkeleton3D from the iOS app, expects /body3d/kp on :57128) | +| `IPHONE_OSC_PORT` | `57128` | UDP port the iPhone ARBodyTracker app pushes /body3d/kp to (always-on listener in data_only_viz) | +| `ICP_FUSION` | `0` | `1` to enable LiDAR↔SMPL-X ICP fusion (cf. `docs/ICP_FUSION.md`) | +| `ICP_LIDAR_HOST` | _(unset)_ | iPhone ARBodyTracker IP when `ICP_FUSION=1` | +| `ICP_LIDAR_PORT` | `5500` | iPhone ARMesh TCP port | +| `ICP_LIDAR_EXTRINSIC` | `~/.config/av-live/lidar_extrinsic.json` | extrinsic JSON path | + ## Conventions globales - Python : **uv** systématiquement (jamais pip/poetry/conda directs). diff --git a/avlivebody-mac/.gitignore b/avlivebody-mac/.gitignore new file mode 100644 index 0000000..952b3f6 --- /dev/null +++ b/avlivebody-mac/.gitignore @@ -0,0 +1,4 @@ +*.xcodeproj/ +Config/Local.xcconfig +.build/ +.swiftpm/ diff --git a/avlivebody-mac/Config/Local.xcconfig.example b/avlivebody-mac/Config/Local.xcconfig.example new file mode 100644 index 0000000..d6d5e1f --- /dev/null +++ b/avlivebody-mac/Config/Local.xcconfig.example @@ -0,0 +1,3 @@ +// Copy to Config/Local.xcconfig and set your Apple Developer Team ID. +// Config/Local.xcconfig is gitignored. +DEVELOPMENT_TEAM = YOUR_TEAM_ID diff --git a/avlivebody-mac/Config/Shared.xcconfig b/avlivebody-mac/Config/Shared.xcconfig new file mode 100644 index 0000000..055b7cf --- /dev/null +++ b/avlivebody-mac/Config/Shared.xcconfig @@ -0,0 +1,8 @@ +#include? "Local.xcconfig" + +MACOSX_DEPLOYMENT_TARGET = 15.0 +SWIFT_VERSION = 5.10 +// Manual ad-hoc signing for local dev (no Apple Mac Development cert +// required). Override here or via target settings for distribution. +CODE_SIGN_STYLE = Manual +CODE_SIGN_IDENTITY = - diff --git a/avlivebody-mac/README.md b/avlivebody-mac/README.md new file mode 100644 index 0000000..5c485b5 --- /dev/null +++ b/avlivebody-mac/README.md @@ -0,0 +1,62 @@ +# AVLiveBody (macOS) + +Native macOS Xcode app that renders SMPL-X body meshes in RealityKit +from the USB iPhone body-tracking pipeline (ARBodyTracker -> Multi-HMR +worker -> AVLiveBody scene). + +## Prerequisites + +- macOS 15+ +- Xcode 16+ +- `xcodegen` — `brew install xcodegen` + +## First-time setup + +1. Copy the CoreML model into the app resources (required, gitignored, + ~195 MB). Without it the app degrades to skeleton-only rendering: + + ``` + cp -R ~/.cache/av-live-multihmr/multihmr_full_672_s.mlpackage \ + avlivebody-mac/Sources/AVLiveBody/Resources/ + ``` + +2. Create your local xcconfig and set your signing team: + + ``` + cp Config/Local.xcconfig.example Config/Local.xcconfig + # Edit Config/Local.xcconfig: + # DEVELOPMENT_TEAM = + ``` + +## Build + +Generate the Xcode project (run after every `project.yml` change) then +open or build from the CLI: + +``` +cd avlivebody-mac +xcodegen generate +open AVLiveBody.xcodeproj +``` + +CLI build / test: + +``` +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' build + +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' test +``` + +## Runtime requirements + +A tethered iPhone running the matching `ARBodyTracker` iOS app over USB +is required for body input. See `iphone-arbody/` for the iOS side. + +## Architecture + +- Design spec: + `docs/superpowers/specs/2026-05-18-avlivebody-macos-rewrite-design.md` +- Implementation plan: + `docs/superpowers/plans/2026-05-18-avlivebody-macos-rewrite.md` diff --git a/avlivebody-mac/Sources/AVLiveBody/AVLiveBodyApp.swift b/avlivebody-mac/Sources/AVLiveBody/AVLiveBodyApp.swift new file mode 100644 index 0000000..379f7b1 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/AVLiveBodyApp.swift @@ -0,0 +1,68 @@ +import Cocoa +import CoreVideo +import SwiftUI + +/// Forces a regular, keyboard-focusable foreground app. +final class AppDelegate: NSObject, NSApplicationDelegate { + func applicationDidFinishLaunching(_ notification: Notification) { + NSApp.setActivationPolicy(.regular) + NSApp.activate() + } +} + +@main +struct AVLiveBodyApp: App { + @NSApplicationDelegateAdaptor(AppDelegate.self) + private var appDelegate + + var body: some Scene { + WindowGroup { + ContentView() + .frame(minWidth: 900, minHeight: 600) + } + } +} + +@MainActor +struct ContentView: View { + @StateObject private var consumer = USBSkeletonConsumer() + private let controller = SceneController() + private let multiHMR: MultiHMRCoreML? = MultiHMRCoreML() + /// Placeholder intrinsics until a `.meta` frame supplies real ones. + private let cameraK: [Float] = [ + 672, 0, 336, 0, 672, 336, 0, 0, 1, + ] + + var body: some View { + ZStack(alignment: .top) { + SceneView(controller: controller) + StatusBar(consumer: consumer) + } + .onAppear { wire() } + .onDisappear { consumer.stop() } + .onReceive(consumer.$skeletons) { skeletons in + controller.updateSkeleton(skeletons) + } + } + + private func wire() { + let controller = self.controller + let multiHMR = self.multiHMR + let cameraK = self.cameraK + consumer.onVideoFrame = { [weak consumer] pixelBuffer in + MainActor.assumeIsolated { + controller.updateVideo(pixelBuffer) + guard let consumer else { return } + if let hmr = multiHMR { + let raw = hmr.infer( + pixelBuffer, cameraK: cameraK) + let fused = BodyFusion.fuse( + persons: raw, + skeletons: consumer.skeletons) + controller.updateMesh(fused) + } + } + } + consumer.start() + } +} diff --git a/avlivebody-mac/Sources/AVLiveBody/AxisConvention.swift b/avlivebody-mac/Sources/AVLiveBody/AxisConvention.swift new file mode 100644 index 0000000..464c3b0 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/AxisConvention.swift @@ -0,0 +1,10 @@ +import Foundation +import simd + +/// ARKit/Multi-HMR world coords (y up, z back) -> RealityKit world +/// coords (y up, z forward). Apply to every vertex/translation that +/// crosses from source pipeline space into the scene. +@inline(__always) +func arkitToRealityKit(_ v: SIMD3) -> SIMD3 { + SIMD3(v.x, -v.y, -v.z) +} diff --git a/avlivebody-mac/Sources/AVLiveBody/Info.plist b/avlivebody-mac/Sources/AVLiveBody/Info.plist new file mode 100644 index 0000000..c9d68b1 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/Info.plist @@ -0,0 +1,17 @@ + + + + + CFBundleNameAVLiveBody + CFBundleIdentifier$(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleExecutable$(EXECUTABLE_NAME) + CFBundlePackageTypeAPPL + CFBundleShortVersionString1.0 + CFBundleVersion1 + LSMinimumSystemVersion15.0 + NSCameraUsageDescription + Receives the tethered iPhone camera over USB. + NSLocalNetworkUsageDescription + Connects to the tethered iPhone over USB (usbmuxd). + + diff --git a/avlivebody-mac/Sources/AVLiveBody/MeshEntity.swift b/avlivebody-mac/Sources/AVLiveBody/MeshEntity.swift new file mode 100644 index 0000000..8ee8029 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/MeshEntity.swift @@ -0,0 +1,68 @@ +import AppKit +import Foundation +import RealityKit +import simd + +/// Renders SMPL-X dense body meshes (10475 vertices) from Multi-HMR. +/// Triangle indices come from the bundled `smplx_faces.bin` +/// (flat UInt32 triplets). +@MainActor +final class MeshEntity { + let root = Entity() + + private static let vertexCount = 10475 + private let faces: [UInt32] + private var pools: [Int: ModelEntity] = [:] + private let material = SimpleMaterial( + color: NSColor(white: 0.8, alpha: 1.0), + roughness: 0.5, isMetallic: false) + + init() { + faces = MeshEntity.loadFaces() + } + + func update(_ persons: [MultiHMRPerson]) { + for (idx, person) in persons.enumerated() { + let entity = pools[idx] ?? { + let e = ModelEntity() + root.addChild(e) + pools[idx] = e + return e + }() + guard let mesh = buildMesh(person.vertices) else { continue } + entity.model = ModelComponent(mesh: mesh, + materials: [material]) + let t = person.translation + entity.transform.translation = arkitToRealityKit(t) + entity.isEnabled = true + } + for idx in pools.keys where idx >= persons.count { + pools[idx]?.isEnabled = false + } + } + + private func buildMesh(_ verts: [SIMD3]) + -> MeshResource? { + guard verts.count == Self.vertexCount, !faces.isEmpty else { + NSLog("MeshEntity: vertex count mismatch %d (expected %d), faces=%d", + verts.count, Self.vertexCount, faces.count) + return nil + } + var descriptor = MeshDescriptor(name: "smplx") + descriptor.positions = MeshBuffer(verts.map(arkitToRealityKit)) + descriptor.primitives = .triangles(faces) + return try? MeshResource.generate(from: [descriptor]) + } + + private static func loadFaces() -> [UInt32] { + guard let url = Bundle.main.url( + forResource: "smplx_faces", withExtension: "bin"), + let data = try? Data(contentsOf: url) else { + NSLog("MeshEntity: smplx_faces.bin missing") + return [] + } + return data.withUnsafeBytes { raw in + Array(raw.bindMemory(to: UInt32.self)) + } + } +} diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/Resources/smplx_faces.bin b/avlivebody-mac/Sources/AVLiveBody/Resources/smplx_faces.bin similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/Resources/smplx_faces.bin rename to avlivebody-mac/Sources/AVLiveBody/Resources/smplx_faces.bin diff --git a/avlivebody-mac/Sources/AVLiveBody/SceneController.swift b/avlivebody-mac/Sources/AVLiveBody/SceneController.swift new file mode 100644 index 0000000..b4254a6 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/SceneController.swift @@ -0,0 +1,117 @@ +import AppKit +import Foundation +import CoreVideo +import RealityKit +import simd +import AVLiveWire + +/// Owns the single RealityKit scene: the video quad, the body root, +/// and an orbital camera. The app calls `updateVideo/updateSkeleton/ +/// updateMesh` from the main queue. +@MainActor +final class SceneController { + let arView = ARView(frame: .zero) + + private let cameraAnchor = AnchorEntity(world: .zero) + private let camera = PerspectiveCamera() + private let worldAnchor = AnchorEntity(world: .zero) + + private(set) var videoQuad: VideoQuad? + private(set) var skeleton: SkeletonEntity? + private(set) var mesh: MeshEntity? + + /// Orbital camera state. + private var orbitYaw: Float = 0 + private var orbitPitch: Float = 0 + private var orbitRadius: Float = 3.0 + + private var didSetUp = false + + func setUp() { + guard !didSetUp else { return } + didSetUp = true + arView.environment.background = .color(.black) + arView.scene.addAnchor(worldAnchor) + + camera.camera.fieldOfViewInDegrees = 55 + cameraAnchor.addChild(camera) + arView.scene.addAnchor(cameraAnchor) + applyCamera() + + let q = VideoQuad() + worldAnchor.addChild(q.entity) + videoQuad = q + + let s = SkeletonEntity() + worldAnchor.addChild(s.root) + skeleton = s + + let m = MeshEntity() + worldAnchor.addChild(m.root) + mesh = m + + installOrbitGestures() + } + + func updateVideo(_ pixelBuffer: CVPixelBuffer) { + videoQuad?.update(pixelBuffer) + } + + func updateSkeleton(_ skeletons: [Int: SkeletonPayload]) { + skeleton?.update(skeletons) + } + + func updateMesh(_ persons: [MultiHMRPerson]) { + mesh?.update(persons) + } + + // MARK: - Orbital camera + + private func applyCamera() { + let cy = cos(orbitYaw), sy = sin(orbitYaw) + let cp = cos(orbitPitch), sp = sin(orbitPitch) + let pos = SIMD3(orbitRadius * cp * sy, + orbitRadius * sp, + orbitRadius * cp * cy) + cameraAnchor.transform.translation = pos + camera.look(at: .zero, from: pos, relativeTo: nil) + } + + private func installOrbitGestures() { + let pan = NSPanGestureRecognizer( + target: OrbitTarget.shared, action: #selector( + OrbitTarget.handlePan(_:))) + OrbitTarget.shared.controller = self + arView.addGestureRecognizer(pan) + } + + fileprivate func orbit(dx: Float, dy: Float) { + orbitYaw += dx * 0.01 + orbitPitch = max(-1.4, min(1.4, orbitPitch + dy * 0.01)) + applyCamera() + } +} + +/// Bridges the AppKit pan gesture to `SceneController.orbit`. +final class OrbitTarget: NSObject { + static let shared = OrbitTarget() + weak var controller: SceneController? + private var last: CGPoint = .zero + + @objc func handlePan(_ g: NSPanGestureRecognizer) { + switch g.state { + case .began: + last = g.translation(in: g.view) + case .changed: + let p = g.translation(in: g.view) + let dx = Float(p.x - last.x) + let dy = Float(p.y - last.y) + last = p + MainActor.assumeIsolated { + self.controller?.orbit(dx: dx, dy: -dy) + } + default: + break + } + } +} diff --git a/avlivebody-mac/Sources/AVLiveBody/SceneView.swift b/avlivebody-mac/Sources/AVLiveBody/SceneView.swift new file mode 100644 index 0000000..66aaf5c --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/SceneView.swift @@ -0,0 +1,15 @@ +import RealityKit +import SwiftUI + +/// SwiftUI bridge that hands the SceneController's ARView to the +/// window and runs `setUp()` once. +struct SceneView: NSViewRepresentable { + let controller: SceneController + + func makeNSView(context: Context) -> ARView { + controller.setUp() + return controller.arView + } + + func updateNSView(_ view: ARView, context: Context) {} +} diff --git a/avlivebody-mac/Sources/AVLiveBody/SkeletonEntity.swift b/avlivebody-mac/Sources/AVLiveBody/SkeletonEntity.swift new file mode 100644 index 0000000..748bbb2 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/SkeletonEntity.swift @@ -0,0 +1,56 @@ +import AVLiveWire +import AppKit +import Foundation +import RealityKit +import simd + +/// Renders 91-joint skeletons as yellow marker spheres. One marker +/// pool per pid. ARKit world coords -> RealityKit space (x, -y, -z). +@MainActor +final class SkeletonEntity { + let root = Entity() + + private static let jointCount = 91 + private static let markerRadius: Float = 0.012 + + private var pools: [Int: [ModelEntity]] = [:] + private let mesh = MeshResource.generateSphere(radius: markerRadius) + private let material = SimpleMaterial( + color: NSColor.systemYellow, roughness: 0.6, isMetallic: false) + + func update(_ skeletons: [Int: SkeletonPayload]) { + // Drop pools for pids no longer present. + for pid in pools.keys where skeletons[pid] == nil { + pools[pid]?.forEach { $0.removeFromParent() } + pools.removeValue(forKey: pid) + } + for (pid, payload) in skeletons { + let pool = pools[pid] ?? makePool() + pools[pid] = pool + let n = min(Self.jointCount, payload.joints.count, + payload.valid.count) + for i in 0.. [ModelEntity] { + var pool: [ModelEntity] = [] + pool.reserveCapacity(Self.jointCount) + for _ in 0..(0, 0, Self.zBack) + } + + /// Replace the plane's texture from a decoded camera frame. + func update(_ pixelBuffer: CVPixelBuffer) { + let ci = CIImage(cvPixelBuffer: pixelBuffer) + guard let cg = ciContext.createCGImage( + ci, from: ci.extent) else { return } + guard let texture = try? TextureResource( + image: cg, options: .init(semantic: .color)) else { + NSLog("VideoQuad: TextureResource creation failed (%dx%d)", + CVPixelBufferGetWidth(pixelBuffer), + CVPixelBufferGetHeight(pixelBuffer)) + return + } + var material = UnlitMaterial() + material.color = .init(tint: .white, + texture: .init(texture)) + entity.model?.materials = [material] + } +} diff --git a/avlivebody-mac/Sources/AVLiveBody/usb/BodyFusion.swift b/avlivebody-mac/Sources/AVLiveBody/usb/BodyFusion.swift new file mode 100644 index 0000000..cbfb357 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/usb/BodyFusion.swift @@ -0,0 +1,29 @@ +import AVLiveWire +import Foundation +import simd + +/// Overrides the highest-scoring Multi-HMR mesh's pelvis depth with +/// the first valid USB skeleton pelvis z. Single-person assumption: +/// with multiple skeletons in the dict the source pelvis is arbitrary +/// (dict iteration order). Pure, stateless — unit-testable. +enum BodyFusion { + /// ARSkeleton3D joint 0 = root (hips), per ARSkeletonDefinition.defaultBody3D. + static let pelvisJoint = 0 + + static func fuse(persons: [MultiHMRPerson], + skeletons: [Int: SkeletonPayload]) + -> [MultiHMRPerson] { + let pelvisZs: [Float] = skeletons.values.compactMap { s in + guard pelvisJoint < s.valid.count, + s.valid[pelvisJoint] else { return nil } + return s.joints[pelvisJoint].z + } + guard !pelvisZs.isEmpty, + let primaryIdx = persons.indices.max(by: { + persons[$0].score < persons[$1].score + }) else { return persons } + var out = persons + out[primaryIdx].translation.z = pelvisZs[0] + return out + } +} diff --git a/avlivebody-mac/Sources/AVLiveBody/usb/MultiHMRCoreML.swift b/avlivebody-mac/Sources/AVLiveBody/usb/MultiHMRCoreML.swift new file mode 100644 index 0000000..689bb92 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/usb/MultiHMRCoreML.swift @@ -0,0 +1,155 @@ +import CoreML +import CoreVideo +import CoreImage +import Foundation + +/// One detected SMPL-X body from Multi-HMR. +struct MultiHMRPerson { + var vertices: [SIMD3] // 10475 SMPL-X verts, model space + var translation: SIMD3 // pelvis translation + var score: Float +} + +/// CoreML wrapper around the bundled `multihmr_full_672_s.mlpackage`. +/// Mirrors `data_only_viz/multihmr_coreml.py`: two MLMultiArray inputs +/// (`image` 1x3x672x672 ImageNet-normalized, `cam_K` 1x3x3), fixed +/// K=4 person outputs. +final class MultiHMRCoreML { + static let inputSize = 672 + static let vertexCount = 10475 + static let maxPersons = 4 + private static let detThreshold: Float = 0.3 + private static let normMean: [Float] = [0.485, 0.456, 0.406] + private static let normStd: [Float] = [0.229, 0.224, 0.225] + + private let model: MLModel + private let ciContext = CIContext() + + /// Loads the bundled model. Returns nil if the resource or load + /// fails — callers fall back to skeleton-only rendering. + init?() { + guard let url = Bundle.main.url( + forResource: "multihmr_full_672_s", + withExtension: "mlmodelc") else { + NSLog("MultiHMRCoreML: mlpackage resource missing") + return nil + } + let cfg = MLModelConfiguration() + cfg.computeUnits = .cpuAndGPU + do { + model = try MLModel(contentsOf: url, configuration: cfg) + } catch { + NSLog("MultiHMRCoreML: load failed %@", + String(describing: error)) + return nil + } + } + + /// Run inference on one camera frame. `cameraK` is the 3x3 camera + /// intrinsics row-major. + func infer(_ pixelBuffer: CVPixelBuffer, + cameraK: [Float]) -> [MultiHMRPerson] { + guard let image = makeImageInput(pixelBuffer), + let k = makeKInput(cameraK) else { return [] } + let inputs: [String: MLFeatureValue] = [ + "image": MLFeatureValue(multiArray: image), + "cam_K": MLFeatureValue(multiArray: k), + ] + guard let provider = try? MLDictionaryFeatureProvider( + dictionary: inputs), + let out = try? model.prediction(from: provider) else { + return [] + } + return parse(out) + } + + // MARK: - Input preprocessing + + /// `CVPixelBuffer` -> [1,3,672,672] Float32, RGB, ImageNet-normed. + private func makeImageInput(_ pb: CVPixelBuffer) -> MLMultiArray? { + let n = Self.inputSize + // Resize to n x n BGRA via CoreImage. + let ci = CIImage(cvPixelBuffer: pb) + let sx = CGFloat(n) / ci.extent.width + let sy = CGFloat(n) / ci.extent.height + let scaled = ci.transformed( + by: CGAffineTransform(scaleX: sx, y: sy)) + var dst: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, n, n, + kCVPixelFormatType_32BGRA, nil, &dst) + guard let dst else { return nil } + ciContext.render(scaled, to: dst) + CVPixelBufferLockBaseAddress(dst, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(dst, .readOnly) } + guard let base = CVPixelBufferGetBaseAddress(dst) else { + return nil + } + let rowBytes = CVPixelBufferGetBytesPerRow(dst) + let px = base.assumingMemoryBound(to: UInt8.self) + guard let arr = try? MLMultiArray( + shape: [1, 3, NSNumber(value: n), NSNumber(value: n)], + dataType: .float32) else { return nil } + let ptr = arr.dataPointer.assumingMemoryBound(to: Float.self) + let plane = n * n + for y in 0.. [1,3,3] Float32. + private func makeKInput(_ k: [Float]) -> MLMultiArray? { + guard k.count == 9, + let arr = try? MLMultiArray( + shape: [1, 3, 3], dataType: .float32) else { return nil } + let ptr = arr.dataPointer.assumingMemoryBound(to: Float.self) + for i in 0..<9 { ptr[i] = k[i] } + return arr + } + + // MARK: - Output parsing + + private func parse(_ out: MLFeatureProvider) -> [MultiHMRPerson] { + guard let v3d = out.featureValue(for: "var_2420")? + .multiArrayValue, + let transl = out.featureValue(for: "var_2423")? + .multiArrayValue, + let scores = out.featureValue(for: "var_2436")? + .multiArrayValue else { return [] } + var persons: [MultiHMRPerson] = [] + let vc = Self.vertexCount + for k in 0..]( + repeating: .zero, count: vc) + let base = k * vc * 3 + for i in 0.. Data? + func close() +} + +/// usbmux client: device discovery + connect-to-port. After a +/// successful `connect`, the same transport carries the raw tunneled +/// byte stream from the device. +final class USBClient { + private let transport: MuxTransport + private var tag: UInt32 = 0 + + init(transport: MuxTransport) { + self.transport = transport + } + + func listDevices() -> [Int] { + tag += 1 + transport.send(USBMuxProtocol.encode( + plist: ["MessageType": "ListDevices"], tag: tag)) + guard let reply = transport.receivePacket(), + let plist = USBMuxProtocol.decode(reply), + let list = plist["DeviceList"] as? [[String: Any]] + else { return [] } + return list.compactMap { $0["DeviceID"] as? Int } + } + + /// Returns true once the transport is tunneled to `port` on the + /// device. usbmux wants the TCP port in big-endian order. + func connect(deviceID: Int, port: UInt16) -> Bool { + tag += 1 + let swapped = Int((port << 8) | (port >> 8)) + transport.send(USBMuxProtocol.encode(plist: [ + "MessageType": "Connect", + "DeviceID": deviceID, + "PortNumber": swapped, + ], tag: tag)) + guard let reply = transport.receivePacket(), + let plist = USBMuxProtocol.decode(reply), + let number = plist["Number"] as? Int + else { return false } + return number == 0 + } +} + +/// Production transport: blocking AF_UNIX socket to usbmuxd. +final class UnixMuxTransport: MuxTransport { + private var fd: Int32 = -1 + + init?(path: String = "/var/run/usbmuxd") { + fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { return nil } + var addr = sockaddr_un() + addr.sun_family = sa_family_t(AF_UNIX) + precondition(path.utf8.count < 104, + "usbmuxd socket path exceeds sun_path limit") + _ = path.withCString { src in + withUnsafeMutablePointer(to: &addr.sun_path) { + $0.withMemoryRebound(to: CChar.self, capacity: 104) { + strcpy($0, src) + } + } + } + let size = socklen_t(MemoryLayout.size) + let ok = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.connect(fd, $0, size) + } + } + if ok != 0 { Darwin.close(fd); return nil } + } + + func send(_ data: Data) { + guard fd >= 0 else { return } + data.withUnsafeBytes { buf in + guard let base = buf.baseAddress else { return } + var off = 0 + while off < data.count { + let w = Darwin.write(fd, base.advanced(by: off), + data.count - off) + if w <= 0 { + if w < 0 && errno == EINTR { continue } + break + } + off += w + } + } + } + + /// Read one usbmux packet: 4-byte LE length prefix then body. + func receivePacket() -> Data? { + guard let head = readN(4) else { return nil } + guard let len = USBMuxProtocol.readLE32(head, 0) else { return nil } + let total = Int(len) + guard total >= 16, let rest = readN(total - 4) else { return nil } + return head + rest + } + + /// Read raw tunneled bytes after a successful Connect. + func readStream(max: Int = 65536) -> Data? { + readN(max, exact: false) + } + + private func readN(_ n: Int, exact: Bool = true) -> Data? { + var buf = [UInt8](repeating: 0, count: n) + var got = 0 + while got < n { + let r = buf.withUnsafeMutableBytes { + Darwin.read(fd, $0.baseAddress!.advanced(by: got), n - got) + } + if r < 0 { + if errno == EINTR { continue } + return got > 0 && !exact ? Data(buf[0.. 0 && !exact ? Data(buf[0..= 0 { Darwin.close(fd); fd = -1 } + } +} diff --git a/avlivebody-mac/Sources/AVLiveBody/usb/USBMuxProtocol.swift b/avlivebody-mac/Sources/AVLiveBody/usb/USBMuxProtocol.swift new file mode 100644 index 0000000..40c17f6 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/usb/USBMuxProtocol.swift @@ -0,0 +1,38 @@ +import Foundation + +/// Codec for the usbmuxd request/response protocol. 16-byte +/// little-endian header (length, version=1, message=8, tag) then an +/// XML property list. +enum USBMuxProtocol { + static func encode(plist: [String: Any], tag: UInt32) -> Data { + let body = (try? PropertyListSerialization.data( + fromPropertyList: plist, format: .xml, options: 0)) + ?? Data() + var d = Data() + appendLE32(&d, UInt32(16 + body.count)) // length + appendLE32(&d, 1) // version + appendLE32(&d, 8) // message: plist + appendLE32(&d, tag) + d.append(body) + return d + } + + static func decode(_ packet: Data) -> [String: Any]? { + guard packet.count >= 16 else { return nil } + let body = packet.dropFirst(16) + return (try? PropertyListSerialization.propertyList( + from: body, options: [], format: nil)) as? [String: Any] + } + + static func appendLE32(_ d: inout Data, _ v: UInt32) { + for i in 0..<4 { d.append(UInt8((v >> (8 * i)) & 0xFF)) } + } + + static func readLE32(_ d: Data, _ offset: Int) -> UInt32? { + guard offset >= 0, d.count >= offset + 4 else { return nil } + let b = [UInt8](d) + var v: UInt32 = 0 + for i in 0..<4 { v |= UInt32(b[offset + i]) << (8 * i) } + return v + } +} diff --git a/avlivebody-mac/Sources/AVLiveBody/usb/USBSkeletonConsumer.swift b/avlivebody-mac/Sources/AVLiveBody/usb/USBSkeletonConsumer.swift new file mode 100644 index 0000000..601740a --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/usb/USBSkeletonConsumer.swift @@ -0,0 +1,107 @@ +import AVLiveWire +import Combine +import CoreVideo +import Foundation + +/// Connects to the tethered iPhone over USB (usbmuxd), demuxes the +/// AVLiveWire stream, republishes skeleton payloads (keyed by pid) +/// and forwards decoded camera frames. Blocking transport runs on a +/// dedicated background thread; only `@Published` writes hop to main. +final class USBSkeletonConsumer: ObservableObject { + /// 91-joint skeleton payloads keyed by pid. + @Published var skeletons: [Int: SkeletonPayload] = [:] + @Published var connected = false + + /// Called on the main queue for every decoded camera frame. + var onVideoFrame: ((CVPixelBuffer) -> Void)? + + /// TCP port the iPhone `USBServer` listens on. + static let devicePort: UInt16 = 7000 + + private let videoDecoder = VideoDecoder() + private let stateLock = NSLock() + private var running = false + private var thread: Thread? + + init() { + videoDecoder.onFrame = { [weak self] pixelBuffer in + DispatchQueue.main.async { + self?.onVideoFrame?(pixelBuffer) + } + } + } + + private var isRunning: Bool { + stateLock.lock(); defer { stateLock.unlock() } + return running + } + + func start() { + stateLock.lock() + if running { stateLock.unlock(); return } + running = true + let t = Thread { [weak self] in self?.loop() } + t.name = "cc.avlive.usbconsumer" + thread = t + stateLock.unlock() + t.start() + } + + func stop() { + stateLock.lock(); running = false; stateLock.unlock() + } + + private func loop() { + while isRunning { + guard let transport = UnixMuxTransport() else { + NSLog("USBSkeletonConsumer: no usbmuxd; retry") + Thread.sleep(forTimeInterval: 1.0); continue + } + let client = USBClient(transport: transport) + let devices = client.listDevices() + guard let dev = devices.first, + client.connect(deviceID: dev, + port: Self.devicePort) else { + NSLog("USBSkeletonConsumer: no device; retry") + transport.close() + Thread.sleep(forTimeInterval: 1.0); continue + } + NSLog("USBSkeletonConsumer: connected to device %d", dev) + publishConnected(true) + var demux = StreamDemuxer() + while isRunning { + guard let chunk = transport.readStream(), + !chunk.isEmpty else { break } + for frame in demux.feed(chunk) { route(frame) } + } + transport.close() + publishConnected(false) + NSLog("USBSkeletonConsumer: disconnected") + if isRunning { Thread.sleep(forTimeInterval: 1.0) } + } + } + + private func route(_ frame: StreamDemuxer.Frame) { + switch frame.header.tag { + case .skeleton: + guard let payload = + SkeletonPayload(decoding: frame.payload) else { return } + let pid = Int(frame.header.pid) + DispatchQueue.main.async { [weak self] in + self?.skeletons[pid] = payload + } + case .video: + guard let payload = + VideoPayload(decoding: frame.payload) else { return } + videoDecoder.decode(payload) + case .meta: + break + } + } + + private func publishConnected(_ value: Bool) { + DispatchQueue.main.async { [weak self] in + self?.connected = value + } + } +} diff --git a/avlivebody-mac/Sources/AVLiveBody/usb/VideoDecoder.swift b/avlivebody-mac/Sources/AVLiveBody/usb/VideoDecoder.swift new file mode 100644 index 0000000..5e18be8 --- /dev/null +++ b/avlivebody-mac/Sources/AVLiveBody/usb/VideoDecoder.swift @@ -0,0 +1,184 @@ +import AVLiveWire +import CoreMedia +import CoreVideo +import Foundation +import VideoToolbox + +/// HEVC decoder. Feed `VideoPayload`s in; receive `CVPixelBuffer`s via +/// `onFrame`. Keyframe payloads must carry the VPS/SPS/PPS parameter +/// sets prepended as 4-byte-length-prefixed NAL units (the layout the +/// iOS `VideoEncoder` emits); the decoder (re)builds its format +/// description from those. +final class VideoDecoder { + var onFrame: ((CVPixelBuffer) -> Void)? + + private var session: VTDecompressionSession? + private var formatDesc: CMVideoFormatDescription? + + /// Decode one access unit. + func decode(_ payload: VideoPayload) { + var au = payload.data + if payload.isKeyframe { + let (params, rest) = Self.splitParameterSets(au) + if !params.isEmpty { + rebuildFormat(params) + } + au = rest + } + guard let fmt = formatDesc, !au.isEmpty else { return } + if session == nil { makeSession(fmt) } + guard let session, let block = Self.blockBuffer(au) else { + return + } + var sample: CMSampleBuffer? + var sampleSize = au.count + guard CMSampleBufferCreateReady( + allocator: kCFAllocatorDefault, dataBuffer: block, + formatDescription: fmt, sampleCount: 1, + sampleTimingEntryCount: 0, sampleTimingArray: nil, + sampleSizeEntryCount: 1, sampleSizeArray: &sampleSize, + sampleBufferOut: &sample) == noErr, let sample else { + return + } + VTDecompressionSessionDecodeFrame( + session, sampleBuffer: sample, flags: [], + infoFlagsOut: nil) { [weak self] status, _, image, _, _ in + guard status == noErr, let image else { return } + self?.onFrame?(image) + } + } + + func stop() { + if let session { VTDecompressionSessionInvalidate(session) } + session = nil + formatDesc = nil + } + + deinit { stop() } + + // MARK: - Helpers + + /// Leading 4-byte-length-prefixed NAL units of HEVC parameter-set + /// type (VPS=32, SPS=33, PPS=34) are split from the frame data. + /// Returns (parameterSetData, frameData). + private static func splitParameterSets(_ data: Data) + -> (Data, Data) { + let bytes = [UInt8](data) + var offset = 0 + var paramEnd = 0 + while offset + 4 <= bytes.count { + let len = (Int(bytes[offset]) << 24) + | (Int(bytes[offset + 1]) << 16) + | (Int(bytes[offset + 2]) << 8) + | Int(bytes[offset + 3]) + let nalStart = offset + 4 + guard len > 0, nalStart + len <= bytes.count else { break } + let nalType = (Int(bytes[nalStart]) >> 1) & 0x3F + if nalType == 32 || nalType == 33 || nalType == 34 { + offset = nalStart + len + paramEnd = offset + } else { + break + } + } + return (data.prefix(paramEnd), + data.suffix(from: data.startIndex + .advanced(by: paramEnd))) + } + + private func rebuildFormat(_ paramData: Data) { + var sets: [[UInt8]] = [] + let bytes = [UInt8](paramData) + var offset = 0 + while offset + 4 <= bytes.count { + let len = (Int(bytes[offset]) << 24) + | (Int(bytes[offset + 1]) << 16) + | (Int(bytes[offset + 2]) << 8) + | Int(bytes[offset + 3]) + let start = offset + 4 + guard len > 0, start + len <= bytes.count else { break } + sets.append(Array(bytes[start..= 3 else { return } + var fmt: CMFormatDescription? + let status = withParameterSetPointers(sets) { pBuf, sBuf in + CMVideoFormatDescriptionCreateFromHEVCParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: sets.count, + parameterSetPointers: pBuf, + parameterSetSizes: sBuf, + nalUnitHeaderLength: 4, extensions: nil, + formatDescriptionOut: &fmt) + } + if status == noErr, let fmt { + formatDesc = fmt + if let session { VTDecompressionSessionInvalidate(session) } + session = nil + } + } + + /// Build the C-style parallel arrays of parameter-set pointers and + /// sizes that `CMVideoFormatDescriptionCreateFromHEVCParameterSets` + /// requires, keeping the backing storage alive for the call. + private func withParameterSetPointers( + _ sets: [[UInt8]], + _ body: (UnsafePointer>, + UnsafePointer) -> OSStatus) -> OSStatus { + func recurse(_ index: Int, + _ ptrs: inout [UnsafePointer], + _ sizes: inout [Int]) -> OSStatus { + if index == sets.count { + return ptrs.withUnsafeBufferPointer { pBuf in + sizes.withUnsafeBufferPointer { sBuf in + body(pBuf.baseAddress!, sBuf.baseAddress!) + } + } + } + return sets[index].withUnsafeBufferPointer { buf in + ptrs.append(buf.baseAddress!) + sizes.append(buf.count) + return recurse(index + 1, &ptrs, &sizes) + } + } + var ptrs: [UnsafePointer] = [] + var sizes: [Int] = [] + ptrs.reserveCapacity(sets.count) + sizes.reserveCapacity(sets.count) + return recurse(0, &ptrs, &sizes) + } + + private func makeSession(_ fmt: CMVideoFormatDescription) { + let attrs: [CFString: Any] = [ + kCVPixelBufferPixelFormatTypeKey: + kCVPixelFormatType_32BGRA, + ] + VTDecompressionSessionCreate( + allocator: kCFAllocatorDefault, formatDescription: fmt, + decoderSpecification: nil, + imageBufferAttributes: attrs as CFDictionary, + outputCallback: nil, decompressionSessionOut: &session) + } + + private static func blockBuffer(_ data: Data) -> CMBlockBuffer? { + var block: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, memoryBlock: nil, + blockLength: data.count, + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, offsetToData: 0, + dataLength: data.count, flags: 0, + blockBufferOut: &block) == noErr, let block else { + return nil + } + var ok = false + data.withUnsafeBytes { raw in + if let base = raw.baseAddress, + CMBlockBufferReplaceDataBytes( + with: base, blockBuffer: block, + offsetIntoDestination: 0, + dataLength: data.count) == noErr { ok = true } + } + return ok ? block : nil + } +} diff --git a/avlivebody-mac/Tests/AVLiveBodyTests/.gitkeep b/avlivebody-mac/Tests/AVLiveBodyTests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/avlivebody-mac/Tests/AVLiveBodyTests/BodyFusionTests.swift b/avlivebody-mac/Tests/AVLiveBodyTests/BodyFusionTests.swift new file mode 100644 index 0000000..f719aae --- /dev/null +++ b/avlivebody-mac/Tests/AVLiveBodyTests/BodyFusionTests.swift @@ -0,0 +1,29 @@ +import XCTest +import AVLiveWire +@testable import AVLiveBody + +final class BodyFusionTests: XCTestCase { + private func skeleton(pelvisZ: Float) -> SkeletonPayload { + var p = SkeletonPayload() + p.joints[0] = SIMD3(0, 0, pelvisZ) + p.valid[0] = true + return p + } + + func testPelvisDepthOverride() { + let mesh = MultiHMRPerson( + vertices: [SIMD3](repeating: .zero, count: 1), + translation: SIMD3(0, 0, -1.0), score: 0.9) + let fused = BodyFusion.fuse( + persons: [mesh], skeletons: [0: skeleton(pelvisZ: -2.5)]) + XCTAssertEqual(fused[0].translation.z, -2.5, accuracy: 1e-4) + } + + func testPassthroughWhenNoSkeleton() { + let mesh = MultiHMRPerson( + vertices: [SIMD3](repeating: .zero, count: 1), + translation: SIMD3(0, 0, -1.0), score: 0.9) + let fused = BodyFusion.fuse(persons: [mesh], skeletons: [:]) + XCTAssertEqual(fused[0].translation.z, -1.0, accuracy: 1e-4) + } +} diff --git a/avlivebody-mac/Tests/AVLiveBodyTests/USBClientTests.swift b/avlivebody-mac/Tests/AVLiveBodyTests/USBClientTests.swift new file mode 100644 index 0000000..c663931 --- /dev/null +++ b/avlivebody-mac/Tests/AVLiveBodyTests/USBClientTests.swift @@ -0,0 +1,49 @@ +import XCTest +@testable import AVLiveBody + +/// In-memory stand-in for the usbmuxd Unix socket. +final class MockMuxTransport: MuxTransport { + var sent: [Data] = [] + var canned: [Data] = [] + func send(_ data: Data) { sent.append(data) } + func receivePacket() -> Data? { + canned.isEmpty ? nil : canned.removeFirst() + } + func close() {} +} + +final class USBClientTests: XCTestCase { + func testListDevicesParsesDeviceIDs() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode(plist: [ + "DeviceList": [ + ["DeviceID": 42, + "Properties": ["ConnectionType": "USB"]], + ]], tag: 0)] + let client = USBClient(transport: mock) + let devices = client.listDevices() + XCTAssertEqual(devices, [42]) + } + + func testConnectSendsConnectRequest() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode( + plist: ["MessageType": "Result", "Number": 0], tag: 0)] + let client = USBClient(transport: mock) + let ok = client.connect(deviceID: 42, port: 7000) + XCTAssertTrue(ok) + let req = USBMuxProtocol.decode(mock.sent.last!) + XCTAssertEqual(req?["MessageType"] as? String, "Connect") + XCTAssertEqual(req?["DeviceID"] as? Int, 42) + XCTAssertEqual(req?["PortNumber"] as? Int, + Int((UInt16(7000) << 8) | (UInt16(7000) >> 8))) + } + + func testConnectFailsOnNonZeroResult() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode( + plist: ["MessageType": "Result", "Number": 3], tag: 0)] + let client = USBClient(transport: mock) + XCTAssertFalse(client.connect(deviceID: 1, port: 7000)) + } +} diff --git a/avlivebody-mac/Tests/AVLiveBodyTests/USBMuxProtocolTests.swift b/avlivebody-mac/Tests/AVLiveBodyTests/USBMuxProtocolTests.swift new file mode 100644 index 0000000..7fbf2c9 --- /dev/null +++ b/avlivebody-mac/Tests/AVLiveBodyTests/USBMuxProtocolTests.swift @@ -0,0 +1,27 @@ +import XCTest +@testable import AVLiveBody + +final class USBMuxProtocolTests: XCTestCase { + func testEncodeWrapsPlistWith16ByteHeader() { + let body: [String: Any] = ["MessageType": "ListDevices"] + let packet = USBMuxProtocol.encode(plist: body, tag: 3) + XCTAssertGreaterThan(packet.count, 16) + XCTAssertEqual(USBMuxProtocol.readLE32(packet, 0).map(Int.init), + packet.count) + XCTAssertEqual(USBMuxProtocol.readLE32(packet, 4), 1) + XCTAssertEqual(USBMuxProtocol.readLE32(packet, 8), 8) + XCTAssertEqual(USBMuxProtocol.readLE32(packet, 12), 3) + } + + func testDecodeRoundTrip() { + let packet = USBMuxProtocol.encode( + plist: ["MessageType": "Result", "Number": 0], tag: 1) + let decoded = USBMuxProtocol.decode(packet) + XCTAssertEqual(decoded?["MessageType"] as? String, "Result") + XCTAssertEqual(decoded?["Number"] as? Int, 0) + } + + func testDecodeRejectsShortPacket() { + XCTAssertNil(USBMuxProtocol.decode(Data([0, 1, 2]))) + } +} diff --git a/avlivebody-mac/project.yml b/avlivebody-mac/project.yml new file mode 100644 index 0000000..d734cb5 --- /dev/null +++ b/avlivebody-mac/project.yml @@ -0,0 +1,54 @@ +name: AVLiveBody +options: + bundleIdPrefix: cc.saillant + deploymentTarget: + macOS: "15.0" + createIntermediateGroups: true + +configFiles: + Debug: Config/Shared.xcconfig + Release: Config/Shared.xcconfig + +packages: + AVLiveWire: + path: ../shared/AVLiveWire + +targets: + AVLiveBody: + type: application + platform: macOS + deploymentTarget: "15.0" + sources: + - path: Sources/AVLiveBody + excludes: + - Info.plist + dependencies: + - package: AVLiveWire + product: AVLiveWire + configFiles: + Debug: Config/Shared.xcconfig + Release: Config/Shared.xcconfig + settings: + base: + PRODUCT_NAME: AVLiveBody + PRODUCT_BUNDLE_IDENTIFIER: cc.saillant.AVLiveBody + INFOPLIST_FILE: Sources/AVLiveBody/Info.plist + GENERATE_INFOPLIST_FILE: NO + CODE_SIGN_STYLE: Manual + CODE_SIGN_IDENTITY: "-" + CODE_SIGNING_REQUIRED: NO + CODE_SIGNING_ALLOWED: NO + SWIFT_VERSION: "5.10" + ENABLE_HARDENED_RUNTIME: NO + AVLiveBodyTests: + type: bundle.unit-test + platform: macOS + sources: + - path: Tests/AVLiveBodyTests + dependencies: + - target: AVLiveBody + - package: AVLiveWire + product: AVLiveWire + settings: + base: + GENERATE_INFOPLIST_FILE: YES diff --git a/data_feeds/__init__.py b/data_feeds/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_feeds/config.avlivedata.toml b/data_feeds/config.avlivedata.toml new file mode 100644 index 0000000..ebecaea --- /dev/null +++ b/data_feeds/config.avlivedata.toml @@ -0,0 +1,37 @@ +[osc] +host = "127.0.0.1" +port = 57127 + +[feeds.eco2mix] +enabled = true +interval_sec = 60 + +[feeds.velib] +enabled = true +interval_sec = 120 +station_codes = [] + +[feeds.hubeau] +enabled = true +interval_sec = 300 +codes = ["F050001001"] + +[feeds.gbfs] +enabled = false +interval_sec = 120 +url = "https://velib-metropole-opendata.smoove.pro/opendata/Velib_Metropole/station_status.json" + +[feeds.ais] +enabled = false + +[feeds.carburants] +enabled = false + +[feeds.prim] +enabled = false + +[feeds.sytadin] +enabled = false + +[feeds.teleray] +enabled = false diff --git a/data_feeds/feeds/__init__.py b/data_feeds/feeds/__init__.py index e69de29..aeba3e5 100644 --- a/data_feeds/feeds/__init__.py +++ b/data_feeds/feeds/__init__.py @@ -0,0 +1,27 @@ +"""Registry of available feed classes (auto-discovery on import).""" +from __future__ import annotations + +from .base import Feed +from .eco2mix import Eco2MixFeed +from .gbfs import GBFSFeed +from .hubeau import HubeauFeed +from .velib import VelibFeed +from .ais import AISFeed +from .carburants import CarburantsFeed +from .prim import PRIMFeed +from .sytadin import SytadinFeed +from .teleray import TelerayFeed + +REGISTRY: dict[str, type[Feed]] = { + "eco2mix": Eco2MixFeed, + "gbfs": GBFSFeed, + "hubeau": HubeauFeed, + "velib": VelibFeed, + "ais": AISFeed, + "carburants": CarburantsFeed, + "prim": PRIMFeed, + "sytadin": SytadinFeed, + "teleray": TelerayFeed, +} + +__all__ = ["Feed", "REGISTRY"] diff --git a/data_feeds/feeds/ais.py b/data_feeds/feeds/ais.py new file mode 100644 index 0000000..bf51df8 --- /dev/null +++ b/data_feeds/feeds/ais.py @@ -0,0 +1,22 @@ +"""AIS vessel positions feed — STUB. + +TODO: needs aisstream.io API key + websocket subscription. +""" +from __future__ import annotations + +import logging + +from .base import Feed + +LOG = logging.getLogger("data_feeds.ais") + + +class AISFeed(Feed): + name = "ais" + interval_sec = 60.0 + + def fetch(self): + return None + + def publish(self, payload) -> None: + LOG.info("stub") diff --git a/data_feeds/feeds/base.py b/data_feeds/feeds/base.py new file mode 100644 index 0000000..9446c1e --- /dev/null +++ b/data_feeds/feeds/base.py @@ -0,0 +1,55 @@ +"""Abstract base class for data feeds.""" +from __future__ import annotations + +import abc +import logging +import time +import threading +from typing import Any + +LOG = logging.getLogger("data_feeds.base") + + +class Feed(abc.ABC): + name: str = "feed" + interval_sec: float = 60.0 + + def __init__(self, osc_send, **cfg) -> None: + self.osc_send = osc_send + self.cfg = cfg + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self.last_t: float = 0.0 + + def configure(self, **kwargs) -> None: + self.cfg.update(kwargs) + if "interval_sec" in kwargs: + self.interval_sec = float(kwargs["interval_sec"]) + + @abc.abstractmethod + def fetch(self) -> Any: ... + + @abc.abstractmethod + def publish(self, payload: Any) -> None: ... + + def tick(self) -> None: + try: + payload = self.fetch() + self.publish(payload) + self.last_t = time.time() + self.osc_send(f"/data/{self.name}/heartbeat", [self.last_t]) + except Exception as e: # noqa: BLE001 + LOG.warning("%s fetch failed: %s", self.name, e) + + def start(self) -> None: + self._thread = threading.Thread( + target=self._run, name=f"feed-{self.name}", daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + + def _run(self) -> None: + while not self._stop.is_set(): + self.tick() + self._stop.wait(self.interval_sec) diff --git a/data_feeds/feeds/carburants.py b/data_feeds/feeds/carburants.py new file mode 100644 index 0000000..fc06443 --- /dev/null +++ b/data_feeds/feeds/carburants.py @@ -0,0 +1,22 @@ +"""Prix carburants feed — STUB. + +TODO: needs prix-carburants.gouv.fr GeoJSON cache + station selection. +""" +from __future__ import annotations + +import logging + +from .base import Feed + +LOG = logging.getLogger("data_feeds.carburants") + + +class CarburantsFeed(Feed): + name = "carburants" + interval_sec = 3600.0 + + def fetch(self): + return None + + def publish(self, payload) -> None: + LOG.info("stub") diff --git a/data_feeds/feeds/eco2mix.py b/data_feeds/feeds/eco2mix.py new file mode 100644 index 0000000..23a6627 --- /dev/null +++ b/data_feeds/feeds/eco2mix.py @@ -0,0 +1,58 @@ +"""RTE eco2mix feed — France electricity production mix in MW. + +Uses the public OpenDataSoft mirror of RTE eco2mix-national-tr (temps reel, +15-min resolution). Stdlib HTTP only. +""" +from __future__ import annotations + +import json +import logging +import urllib.parse +import urllib.request +from typing import Any + +from .base import Feed + +LOG = logging.getLogger("data_feeds.eco2mix") + +# OpenDataSoft public mirror — no key required. +URL = ( + "https://odre.opendatasoft.com/api/records/1.0/search/" + "?dataset=eco2mix-national-tr&rows=1&sort=-date_heure" +) + + +class Eco2MixFeed(Feed): + name = "eco2mix" + interval_sec = 60.0 + + def fetch(self) -> Any: + req = urllib.request.Request(URL, headers={"User-Agent": "av-live/0.1"}) + with urllib.request.urlopen(req, timeout=10) as r: + data = json.loads(r.read().decode("utf-8")) + records = data.get("records") or [] + if not records: + return None + return records[0].get("fields") or {} + + def publish(self, payload: Any) -> None: + if not isinstance(payload, dict): + return + # Pick a representative subset (MW). Keys per eco2mix-national-tr. + keys = [ + "consommation", "nucleaire", "gaz", "charbon", "fioul", + "eolien", "solaire", "hydraulique", "bioenergies", + "ech_physiques", + ] + count = 0 + for k in keys: + v = payload.get(k) + if v is None: + continue + try: + fv = float(v) + except (TypeError, ValueError): + continue + self.osc_send(f"/data/{self.name}/sample", [k, fv]) + count += 1 + self.osc_send(f"/data/{self.name}/count", [count]) diff --git a/data_feeds/feeds/gbfs.py b/data_feeds/feeds/gbfs.py new file mode 100644 index 0000000..ab7db7b --- /dev/null +++ b/data_feeds/feeds/gbfs.py @@ -0,0 +1,53 @@ +"""Generic GBFS (General Bikeshare Feed Specification) reader. + +Reads a `station_status.json` URL and publishes aggregate counters. +""" +from __future__ import annotations + +import json +import logging +import urllib.request +from typing import Any + +from .base import Feed + +LOG = logging.getLogger("data_feeds.gbfs") + + +class GBFSFeed(Feed): + name = "gbfs" + interval_sec = 120.0 + + def fetch(self) -> Any: + url = self.cfg.get("url") + if not url: + LOG.info("gbfs disabled: no url configured") + return None + req = urllib.request.Request(url, headers={"User-Agent": "av-live/0.1"}) + with urllib.request.urlopen(req, timeout=10) as r: + return json.loads(r.read().decode("utf-8")) + + def publish(self, payload: Any) -> None: + if not isinstance(payload, dict): + return + stations = (payload.get("data") or {}).get("stations") or [] + if not stations: + return + codes = set(self.cfg.get("station_codes") or []) + bikes = 0 + docks = 0 + operative = 0 + sampled = 0 + for s in stations: + sid = str(s.get("station_id", "")) + if codes and sid not in codes: + continue + bikes += int(s.get("num_bikes_available") or 0) + docks += int(s.get("num_docks_available") or 0) + if s.get("is_renting") or s.get("is_installed"): + operative += 1 + sampled += 1 + self.osc_send(f"/data/{self.name}/sample", ["bikes_available", float(bikes)]) + self.osc_send(f"/data/{self.name}/sample", ["docks_available", float(docks)]) + self.osc_send(f"/data/{self.name}/sample", ["stations_active", float(operative)]) + self.osc_send(f"/data/{self.name}/count", [sampled]) diff --git a/data_feeds/feeds/hubeau.py b/data_feeds/feeds/hubeau.py new file mode 100644 index 0000000..c6d65a3 --- /dev/null +++ b/data_feeds/feeds/hubeau.py @@ -0,0 +1,66 @@ +"""Hub'Eau hydrometrie feed — water level and flow rate for French rivers. + +API: https://hubeau.eaufrance.fr/api/v1/hydrometrie/observations_tr +Open, no API key required. +""" +from __future__ import annotations + +import json +import logging +import urllib.parse +import urllib.request +from typing import Any + +from .base import Feed + +LOG = logging.getLogger("data_feeds.hubeau") + +BASE = "https://hubeau.eaufrance.fr/api/v1/hydrometrie/observations_tr" + + +class HubeauFeed(Feed): + name = "hubeau" + interval_sec = 300.0 + + def fetch(self) -> Any: + codes = self.cfg.get("codes") or ["F050001001"] + out: dict[str, dict[str, float]] = {} + for code in codes: + params = { + "code_entite": code, + "size": 1, + "sort": "desc", + "fields": "code_station,grandeur_hydro,resultat_obs,date_obs", + } + url = BASE + "?" + urllib.parse.urlencode(params) + try: + req = urllib.request.Request( + url, headers={"User-Agent": "av-live/0.1"}) + with urllib.request.urlopen(req, timeout=10) as r: + data = json.loads(r.read().decode("utf-8")) + except Exception as e: # noqa: BLE001 + LOG.warning("hubeau %s failed: %s", code, e) + continue + for obs in data.get("data") or []: + station = obs.get("code_station") or code + gr = obs.get("grandeur_hydro") or "X" + v = obs.get("resultat_obs") + if v is None: + continue + try: + fv = float(v) + except (TypeError, ValueError): + continue + out.setdefault(station, {})[gr] = fv + return out + + def publish(self, payload: Any) -> None: + if not isinstance(payload, dict) or not payload: + return + count = 0 + for station, vals in payload.items(): + for gr, v in vals.items(): + key = f"{station}_{gr}" + self.osc_send(f"/data/{self.name}/sample", [key, float(v)]) + count += 1 + self.osc_send(f"/data/{self.name}/count", [count]) diff --git a/data_feeds/feeds/prim.py b/data_feeds/feeds/prim.py new file mode 100644 index 0000000..54c8389 --- /dev/null +++ b/data_feeds/feeds/prim.py @@ -0,0 +1,22 @@ +"""PRIM Ile-de-France Mobilites feed — STUB. + +TODO: needs API key (https://prim.iledefrance-mobilites.fr/). +""" +from __future__ import annotations + +import logging + +from .base import Feed + +LOG = logging.getLogger("data_feeds.prim") + + +class PRIMFeed(Feed): + name = "prim" + interval_sec = 60.0 + + def fetch(self): + return None + + def publish(self, payload) -> None: + LOG.info("stub") diff --git a/data_feeds/feeds/sytadin.py b/data_feeds/feeds/sytadin.py new file mode 100644 index 0000000..6470044 --- /dev/null +++ b/data_feeds/feeds/sytadin.py @@ -0,0 +1,22 @@ +"""Sytadin Paris traffic feed — STUB. + +TODO: needs sytadin.fr scraping / cumulative km of congestion. +""" +from __future__ import annotations + +import logging + +from .base import Feed + +LOG = logging.getLogger("data_feeds.sytadin") + + +class SytadinFeed(Feed): + name = "sytadin" + interval_sec = 300.0 + + def fetch(self): + return None + + def publish(self, payload) -> None: + LOG.info("stub") diff --git a/data_feeds/feeds/teleray.py b/data_feeds/feeds/teleray.py new file mode 100644 index 0000000..8a52164 --- /dev/null +++ b/data_feeds/feeds/teleray.py @@ -0,0 +1,22 @@ +"""IRSN Teleray radiation feed — STUB. + +TODO: needs https://teleray.irsn.fr/data endpoint research. +""" +from __future__ import annotations + +import logging + +from .base import Feed + +LOG = logging.getLogger("data_feeds.teleray") + + +class TelerayFeed(Feed): + name = "teleray" + interval_sec = 600.0 + + def fetch(self): + return None + + def publish(self, payload) -> None: + LOG.info("stub") diff --git a/data_feeds/feeds/velib.py b/data_feeds/feeds/velib.py new file mode 100644 index 0000000..79088ad --- /dev/null +++ b/data_feeds/feeds/velib.py @@ -0,0 +1,25 @@ +"""Velib Metropole feed — specialization of GBFS against the Paris URL.""" +from __future__ import annotations + +import logging + +from .gbfs import GBFSFeed + +LOG = logging.getLogger("data_feeds.velib") + +VELIB_URL = ( + "https://velib-metropole-opendata.smoove.pro/opendata/" + "Velib_Metropole/station_status.json" +) + + +class VelibFeed(GBFSFeed): + name = "velib" + interval_sec = 120.0 + + def configure(self, **kwargs) -> None: + # Force the URL if caller did not provide one. + kwargs.setdefault("url", VELIB_URL) + super().configure(**kwargs) + if not self.cfg.get("url"): + self.cfg["url"] = VELIB_URL diff --git a/data_feeds/main.py b/data_feeds/main.py new file mode 100644 index 0000000..2dacae9 --- /dev/null +++ b/data_feeds/main.py @@ -0,0 +1,57 @@ +"""Run all enabled feeds, publish OSC to AVLiveBody.""" +from __future__ import annotations + +import argparse +import logging +import sys +import time +import tomllib +from pathlib import Path + +from .feeds import REGISTRY +from .osc_sender import OscSender + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser() + p.add_argument("--config", default="data_feeds/config.avlivedata.toml") + p.add_argument("--osc-host") + p.add_argument("--osc-port", type=int) + p.add_argument("-v", "--verbose", action="store_true") + args = p.parse_args(argv) + logging.basicConfig( + level=logging.INFO if args.verbose else logging.WARNING, + format="%(asctime)s %(levelname)s %(name)s %(message)s") + cfg = tomllib.loads(Path(args.config).read_text()) + osc_cfg = cfg.get("osc", {}) + host = args.osc_host or osc_cfg.get("host", "127.0.0.1") + port = args.osc_port or osc_cfg.get("port", 57127) + sender = OscSender(host, port) + feeds = [] + for name, kwargs in (cfg.get("feeds") or {}).items(): + if not kwargs.get("enabled", False): + continue + cls = REGISTRY.get(name) + if cls is None: + logging.warning("Unknown feed: %s", name) + continue + f = cls(sender.send) + f.configure(**kwargs) + f.start() + feeds.append(f) + logging.info("started feed %s (interval %.0fs)", name, f.interval_sec) + if not feeds: + logging.warning("No feeds enabled. Exiting.") + return 1 + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + return 0 + finally: + for f in feeds: + f.stop() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/data_feeds/osc_sender.py b/data_feeds/osc_sender.py new file mode 100644 index 0000000..5c20101 --- /dev/null +++ b/data_feeds/osc_sender.py @@ -0,0 +1,22 @@ +"""Wrapper around python-osc SimpleUDPClient with per-route helpers.""" +from __future__ import annotations + +import logging +from typing import Any + +from pythonosc.udp_client import SimpleUDPClient + +LOG = logging.getLogger("data_feeds.osc") + + +class OscSender: + def __init__(self, host: str, port: int) -> None: + self.host = host + self.port = port + self._client = SimpleUDPClient(host, port) + + def send(self, addr: str, args: list[Any]) -> None: + try: + self._client.send_message(addr, args) + except OSError as e: + LOG.warning("send %s failed: %s", addr, e) diff --git a/data_only_viz/CLAUDE.md b/data_only_viz/CLAUDE.md index f252139..97f3963 100644 --- a/data_only_viz/CLAUDE.md +++ b/data_only_viz/CLAUDE.md @@ -31,6 +31,12 @@ Python **3.11+** requis. `pyproject.toml` est la source de vérité — ne jamai - État partagé multi-thread : `state.py` expose `State.lock()` — toujours mutationner sous lock. - Filtrage temporel : `euro_filter.py` (One Euro Filter) sur les keypoints avant tracker. +- ARKit fusion : `iphone_osc_listener.py` consume /body3d/kp UDP :57128 + → `state.persons_arkit_joints`. `pose_filter.py::ArkitFuse` (stage + `arkit_fuse`) splices the 14 mapped body slots into MediaPipe pose + before kalman ; `multi_hmr_worker::arkit_pelvis_z_override` locks the + SMPL-X cam translation z to the ARKit pelvis. Mapping in + `arkit_joint_map.py`. - Association multi-personne : `tracker.py` IoU-based, `scipy.optimize.linear_sum_assignment`. - Shaders Metal dans `shaders/` (`.metal`), recompilés au runtime ; topologie mesh (SMPL faces) en binaire dans `mesh_topology.py`. - OSC out : `osc_listener.py` / `pose_bridge.py` — destination `oscope-of` sur `:57123`. diff --git a/data_only_viz/arkit_joint_map.py b/data_only_viz/arkit_joint_map.py new file mode 100644 index 0000000..e4fddd5 --- /dev/null +++ b/data_only_viz/arkit_joint_map.py @@ -0,0 +1,45 @@ +"""ARKit ARSkeleton3D 91-joint indices → MediaPipe Pose 33 indices. + +The ARKit ARSkeleton.JointName enum (Apple SDK) orders 91 joints +starting with the root, hips, spine chain, shoulders, etc. We pick +only the joints with a clear 1:1 anatomical correspondence to the +MediaPipe Pose 33 landmark set (which is what AVLiveBody renders). +Face/hand sub-joints (fingers, eyes) are skipped — those keep their +existing data sources (MediaPipe Face/Hand + HaMeR MANO). + +Reference for ARKit joint order : Apple developer docs +"ARSkeleton.JointName" — the canonical 91-joint list runs from +root_joint=0 down to right_handThumbEndJoint=90. + +The selection here mirrors `multi.py::SMPLX_TO_MP33` so the same 14 +body slots are overridden by ARKit when fresh. Confidence comes +from ARKit's tracking state but is not currently fanned out — we +trust ARKit body tracking when its OSC frame is present. +""" +from __future__ import annotations + +# MediaPipe Pose 33 cardinality (cf. mediapipe pose_world_landmarks). +MP33_NUM_LANDMARKS = 33 + +# Pelvis = ARKit hips_joint, slot 1 in the canonical enum order. +# Used by multi_hmr_worker for cam-translation z lock. +ARKIT_PELVIS_IDX = 1 + +# (arkit_joint_idx, mediapipe_pose_idx). Match the body slots used +# by the SMPL-X body fusion in multi.py. +ARKIT91_TO_MP33: tuple[tuple[int, int], ...] = ( + (50, 11), # left_shoulder_1_joint -> L_SHOULDER + (32, 12), # right_shoulder_1_joint -> R_SHOULDER + (53, 13), # left_arm_joint -> L_ELBOW + (35, 14), # right_arm_joint -> R_ELBOW + (54, 15), # left_forearm_joint -> L_WRIST + (36, 16), # right_forearm_joint -> R_WRIST + (62, 23), # left_upLeg_joint -> L_HIP + (57, 24), # right_upLeg_joint -> R_HIP + (63, 25), # left_leg_joint -> L_KNEE + (58, 26), # right_leg_joint -> R_KNEE + (64, 27), # left_foot_joint -> L_ANKLE + (59, 28), # right_foot_joint -> R_ANKLE + (65, 31), # left_toes_joint -> L_FOOT_INDEX + (60, 32), # right_toes_joint -> R_FOOT_INDEX +) diff --git a/data_only_viz/dino_reid.py b/data_only_viz/dino_reid.py new file mode 100644 index 0000000..8241c6c --- /dev/null +++ b/data_only_viz/dino_reid.py @@ -0,0 +1,204 @@ +"""DINOv2 ViT-S/14 person re-id backend (CoreML via pyobjc). + +Loads the .mlpackage produced by ``scripts/convert_dinov2.py`` and runs +inference one crop at a time (pyobjc + MLDictionaryFeatureProvider). +Same pattern as ``multihmr_coreml.py`` so Python 3.14 works (no +coremltools dependency at runtime). + +Embeddings are L2-normalised inside the CoreML graph, so cosine sim +between two outputs is a plain dot product. + +Public API:: + + reid = DinoReid(mlpackage_path) # optional path + emb = reid.embed_crops(list_of_uint8_HWC) # -> np.ndarray (N, 384) + DinoReid.is_available() # bool +""" +from __future__ import annotations + +import logging +import time +from pathlib import Path +from typing import Sequence + +import numpy as np + +LOG = logging.getLogger("dino_reid") + +DEFAULT_MLPACKAGE = ( + Path.home() / ".cache" / "av-live-multihmr" / "dinov2_vits14.mlpackage" +) + +EMBED_DIM = 384 +INPUT_SIZE = 224 + +# MLMultiArrayDataType raw values (from CoreML headers). +ML_DTYPE_FLOAT32 = 65568 +ML_DTYPE_FLOAT16 = 65552 +ML_DTYPE_DOUBLE = 65600 + + +def _resize_crop(crop_uint8: np.ndarray) -> np.ndarray: + """Resize an HxWx3 uint8 crop to (3, 224, 224) float32 in [0, 1]. + + Uses ``cv2.resize`` when available, falls back to a simple stride + sampler otherwise (avoids hard cv2 dep in test envs).""" + if crop_uint8.ndim != 3 or crop_uint8.shape[2] != 3: + raise ValueError(f"crop must be HxWx3 uint8, got {crop_uint8.shape}") + if crop_uint8.shape[0] == INPUT_SIZE and crop_uint8.shape[1] == INPUT_SIZE: + rgb = crop_uint8 + else: + try: + import cv2 + rgb = cv2.resize(crop_uint8, (INPUT_SIZE, INPUT_SIZE), + interpolation=cv2.INTER_AREA) + except ImportError: + h, w = crop_uint8.shape[:2] + ys = (np.linspace(0, h - 1, INPUT_SIZE)).astype(np.int32) + xs = (np.linspace(0, w - 1, INPUT_SIZE)).astype(np.int32) + rgb = crop_uint8[ys][:, xs] + return (rgb.astype(np.float32) / 255.0).transpose(2, 0, 1) + + +class DinoReid: + """Forward DINOv2 ViT-S/14 over RGB crops, return L2-normalised + embeddings (N, 384).""" + + def __init__(self, mlpackage_path: Path | str | None = None) -> None: + self.path = Path(mlpackage_path) if mlpackage_path else DEFAULT_MLPACKAGE + if not self.path.exists(): + raise FileNotFoundError(f"mlpackage missing: {self.path}") + + import objc + from Foundation import NSURL + + self._objc = objc + self._NSURL = NSURL + + ns: dict = {} + objc.loadBundle("CoreML", ns, + "/System/Library/Frameworks/CoreML.framework") + self._ns = ns + + MLModel = ns["MLModel"] + MLModelConfiguration = ns["MLModelConfiguration"] + cfg = MLModelConfiguration.alloc().init() + try: + # 2 = MLComputeUnitsAll (CPU+GPU+ANE). DINOv2 ViT-S/14 + # converts cleanly and ANE serves it well. + cfg.setComputeUnits_(2) + except Exception: # noqa: BLE001 + pass + + url = NSURL.fileURLWithPath_(str(self.path)) + compiled = MLModel.compileModelAtURL_error_(url, None) + if compiled is None: + raise RuntimeError(f"compile failed for {self.path}") + model = MLModel.modelWithContentsOfURL_configuration_error_( + compiled, cfg, None) + if model is None: + raise RuntimeError(f"load failed for {compiled}") + self._model = model + + # Discover the output feature name (single tensor). + desc = model.modelDescription() + out_names = [str(n) for n in desc.outputDescriptionsByName().keys()] + self._out_name = out_names[0] if out_names else "embedding" + LOG.info("dino_reid loaded (%s, out=%s)", self.path.name, + self._out_name) + + @classmethod + def is_available(cls, mlpackage_path: Path | str | None = None) -> bool: + p = Path(mlpackage_path) if mlpackage_path else DEFAULT_MLPACKAGE + if not p.exists(): + return False + try: + import objc # noqa: F401 + from Foundation import NSURL # noqa: F401 + return True + except Exception: # noqa: BLE001 + return False + + # ------------------------------------------------------------------ + # MLMultiArray plumbing — mirrors multihmr_coreml._np_to_mlarray / + # _mlarray_to_np. Float32 in, float32-or-float16 out. + # ------------------------------------------------------------------ + def _np_to_mlarray(self, arr: np.ndarray): + import ctypes + MLMultiArray = self._ns["MLMultiArray"] + arr = np.ascontiguousarray(arr, dtype=np.float32) + shape = [int(s) for s in arr.shape] + ml = MLMultiArray.alloc().initWithShape_dataType_error_( + shape, ML_DTYPE_FLOAT32, None) + if ml is None: + raise RuntimeError("MLMultiArray alloc failed") + ptr = ml.dataPointer() + addr = int(ptr) if isinstance(ptr, int) else ctypes.cast( + ptr, ctypes.c_void_p).value + if addr is None: + raise RuntimeError("dataPointer null") + ctypes.memmove(addr, arr.ctypes.data, arr.nbytes) + return ml + + def _mlarray_to_np(self, ml) -> np.ndarray: + import ctypes + shape = tuple(int(s) for s in ml.shape()) + dtype_id = int(ml.dataType()) + count = 1 + for s in shape: + count *= s + ptr = ml.dataPointer() + addr = int(ptr) if isinstance(ptr, int) else ctypes.cast( + ptr, ctypes.c_void_p).value + if addr is None: + raise RuntimeError("dataPointer null") + if dtype_id == ML_DTYPE_FLOAT16: + raw = (ctypes.c_uint16 * count).from_address(addr) + arr = np.ctypeslib.as_array(raw).view(np.float16).astype(np.float32) + elif dtype_id == ML_DTYPE_FLOAT32: + raw = (ctypes.c_float * count).from_address(addr) + arr = np.ctypeslib.as_array(raw).copy() + elif dtype_id == ML_DTYPE_DOUBLE: + raw = (ctypes.c_double * count).from_address(addr) + arr = np.ctypeslib.as_array(raw).astype(np.float32) + else: + raise RuntimeError(f"unsupported dtype {dtype_id}") + return arr.reshape(shape) + + def _predict_one(self, image_chw: np.ndarray) -> np.ndarray: + MLDictionaryFeatureProvider = self._ns["MLDictionaryFeatureProvider"] + MLFeatureValue = self._ns["MLFeatureValue"] + x4 = image_chw[np.newaxis, ...] if image_chw.ndim == 3 else image_chw + img_ml = self._np_to_mlarray(x4) + feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml)} + provider = MLDictionaryFeatureProvider.alloc( + ).initWithDictionary_error_(feats, None) + if provider is None: + raise RuntimeError("provider alloc failed") + out = self._model.predictionFromFeatures_error_(provider, None) + if out is None: + raise RuntimeError("predict failed") + fv = out.featureValueForName_(self._out_name) + ml = fv.multiArrayValue() + return self._mlarray_to_np(ml).reshape(-1) + + def embed_crops( + self, crops_uint8: Sequence[np.ndarray], + ) -> np.ndarray: + """Embed a list of HxWx3 uint8 RGB crops -> (N, 384) float32. + + Loops one crop at a time (the CoreML model is traced for B=1). + For typical N <= 4 this is still 10-15 ms total on M5.""" + if not crops_uint8: + return np.zeros((0, EMBED_DIM), dtype=np.float32) + t0 = time.perf_counter() + out = np.zeros((len(crops_uint8), EMBED_DIM), dtype=np.float32) + for i, c in enumerate(crops_uint8): + chw = _resize_crop(c) + out[i] = self._predict_one(chw) + dt_ms = (time.perf_counter() - t0) * 1e3 + if LOG.isEnabledFor(logging.DEBUG) or dt_ms > 50.0: + LOG.log( + logging.DEBUG if dt_ms <= 50.0 else logging.INFO, + "embedded %d crops in %.1f ms", len(crops_uint8), dt_ms) + return out diff --git a/data_only_viz/icp_fusion.py b/data_only_viz/icp_fusion.py new file mode 100644 index 0000000..f668aca --- /dev/null +++ b/data_only_viz/icp_fusion.py @@ -0,0 +1,215 @@ +"""ICP fusion between Multi-HMR SMPL-X meshes and iPhone LiDAR point clouds. + +All operations happen in the **webcam camera frame** (meters, OpenCV +convention: +X right, +Y down, +Z forward). LiDAR points must be +pre-transformed via `Extrinsic.T_arkit_to_cam`. +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass + +import numpy as np + +try: + import open3d as o3d +except ImportError: # pragma: no cover - exercised via skipif at import sites + o3d = None # type: ignore[assignment] + +_LOG = logging.getLogger(__name__) + +MIN_LIDAR_POINTS = 200 +MIN_FITNESS = 0.30 +MAX_RMSE_M = 0.05 +CROP_MARGIN_M = 0.30 + + +@dataclass +class IcpConfig: + voxel_size_m: float = 0.02 + max_correspondence_m: float = 0.05 + max_iterations: int = 30 + + +@dataclass +class IcpResult: + vertices_registered: np.ndarray + accepted: bool + fitness: float + rmse_m: float + iterations: int + + +def register_mesh_to_lidar( + smplx_verts_cam: np.ndarray, + lidar_points_cam: np.ndarray, + config: IcpConfig | None = None, +) -> IcpResult: + """Register SMPL-X verts onto a cropped LiDAR neighborhood.""" + if o3d is None: + raise RuntimeError("open3d not installed — install with `uv sync --extra lidar`") + + cfg = config or IcpConfig() + src = np.ascontiguousarray(smplx_verts_cam, dtype=np.float32) + + if not np.isfinite(src).all(): + _LOG.debug("ICP rejected: NaN/Inf in SMPL-X verts") + return IcpResult(src, False, 0.0, float("inf"), 0) + + lidar = _crop_to_bbox(lidar_points_cam, src, margin_m=CROP_MARGIN_M) + if lidar.shape[0] < MIN_LIDAR_POINTS or not np.isfinite(lidar).all(): + _LOG.debug("ICP rejected: insufficient LiDAR points (%d)", lidar.shape[0]) + return IcpResult(src, False, 0.0, float("inf"), 0) + + src_pcd = _to_pcd(src, cfg.voxel_size_m, estimate_normals=True) + tgt_pcd = _to_pcd(lidar, cfg.voxel_size_m, estimate_normals=True) + + if len(src_pcd.points) < 10 or len(tgt_pcd.points) < 10: + return IcpResult(src, False, 0.0, float("inf"), 0) + + criteria = o3d.pipelines.registration.ICPConvergenceCriteria( + max_iteration=cfg.max_iterations, + relative_fitness=1e-6, + relative_rmse=1e-6, + ) + # Coarse-to-fine: a wide first pass handles translations larger than the + # final correspondence threshold, then the strict pass refines and gates. + coarse = o3d.pipelines.registration.registration_icp( + src_pcd, tgt_pcd, max(cfg.max_correspondence_m * 5.0, 0.20), + np.eye(4), + o3d.pipelines.registration.TransformationEstimationPointToPlane(), + criteria, + ) + result = o3d.pipelines.registration.registration_icp( + src_pcd, tgt_pcd, cfg.max_correspondence_m, + coarse.transformation, + o3d.pipelines.registration.TransformationEstimationPointToPlane(), + criteria, + ) + + accepted = (result.fitness >= MIN_FITNESS) and (result.inlier_rmse <= MAX_RMSE_M) + if not accepted: + _LOG.debug("ICP rejected: fitness=%.3f rmse=%.4f", result.fitness, result.inlier_rmse) + return IcpResult(src, False, float(result.fitness), float(result.inlier_rmse), 0) + + T = np.asarray(result.transformation, dtype=np.float32) + homog = np.concatenate([src, np.ones((src.shape[0], 1), dtype=np.float32)], axis=1) + fused = (homog @ T.T)[:, :3] + if not np.isfinite(fused).all(): + return IcpResult(src, False, float(result.fitness), float(result.inlier_rmse), 0) + + return IcpResult( + vertices_registered=np.ascontiguousarray(fused, dtype=np.float32), + accepted=True, + fitness=float(result.fitness), + rmse_m=float(result.inlier_rmse), + iterations=cfg.max_iterations, + ) + + +def _crop_to_bbox(points: np.ndarray, anchor: np.ndarray, margin_m: float) -> np.ndarray: + if points.size == 0: + return points.astype(np.float32, copy=False) + lo = anchor.min(axis=0) - margin_m + hi = anchor.max(axis=0) + margin_m + mask = np.all((points >= lo) & (points <= hi), axis=1) + return points[mask].astype(np.float32, copy=False) + + +def _to_pcd(points: np.ndarray, voxel_size_m: float, estimate_normals: bool): + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(points.astype(np.float64, copy=False)) + if voxel_size_m > 0: + pcd = pcd.voxel_down_sample(voxel_size_m) + if estimate_normals: + pcd.estimate_normals( + search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=voxel_size_m * 2, max_nn=30), + ) + return pcd + + +def partition_lidar_by_pid( + lidar_points_cam: np.ndarray, + pelvises: dict[int, np.ndarray], + max_dist_m: float = 1.0, +) -> dict[int, np.ndarray]: + """Assign each LiDAR point to the closest pelvis within ``max_dist_m``. + + Points beyond ``max_dist_m`` from every pelvis (background, furniture) + are dropped. Returns ``{pid: (M, 3) float32}`` — pids with zero assigned + points are omitted. + """ + if not pelvises or lidar_points_cam.size == 0: + return {} + pids = list(pelvises.keys()) + centers = np.stack([pelvises[p] for p in pids]).astype(np.float32) + pts = np.ascontiguousarray(lidar_points_cam, dtype=np.float32) + + diff = pts[:, None, :] - centers[None, :, :] + d2 = np.einsum("npk,npk->np", diff, diff) + nearest = d2.argmin(axis=1) + nearest_d = np.sqrt(d2[np.arange(d2.shape[0]), nearest]) + + mask = nearest_d <= max_dist_m + out: dict[int, np.ndarray] = {} + for idx, pid in enumerate(pids): + sel = mask & (nearest == idx) + if not sel.any(): + continue + out[pid] = pts[sel] + return out + + +PELVIS_VERT_INDEX = 5559 # SMPL-X canonical pelvis vertex + + +@dataclass +class FusionMetadata: + applied: set[int] + fitness: dict[int, float] + rmse_m: dict[int, float] + n_lidar_points_used: int + + +class FusionWorker: + """Per-frame ICP fusion orchestrator (caller-driven, no internal thread).""" + + def __init__(self, extrinsic, config: IcpConfig | None = None) -> None: + self._extrinsic = extrinsic + self._config = config or IcpConfig() + + def set_extrinsic(self, extrinsic) -> None: + self._extrinsic = extrinsic + + def run_once(self, state) -> FusionMetadata: + applied: set[int] = set() + fitness: dict[int, float] = {} + rmse: dict[int, float] = {} + + lidar = getattr(state, "lidar_points", None) + if lidar is None or getattr(lidar, "size", 0) == 0 or not state.persons_smplx: + return FusionMetadata(applied, fitness, rmse, 0) + + T = np.asarray(self._extrinsic.T_arkit_to_cam, dtype=np.float32) + homog = np.concatenate([lidar, np.ones((lidar.shape[0], 1), dtype=np.float32)], axis=1) + lidar_cam = (homog @ T.T)[:, :3] + + pelvises = { + p.pid: p.vertices_3d[PELVIS_VERT_INDEX] + for p in state.persons_smplx + if p.vertices_3d is not None + } + parts = partition_lidar_by_pid(lidar_cam, pelvises, max_dist_m=1.0) + + for person in state.persons_smplx: + pts = parts.get(person.pid) + if pts is None: + continue + result = register_mesh_to_lidar(person.vertices_3d, pts, self._config) + fitness[person.pid] = result.fitness + rmse[person.pid] = result.rmse_m + if result.accepted: + person.vertices_3d = result.vertices_registered + applied.add(person.pid) + + return FusionMetadata(applied, fitness, rmse, lidar_cam.shape[0]) diff --git a/data_only_viz/icp_fusion_worker.py b/data_only_viz/icp_fusion_worker.py new file mode 100644 index 0000000..5f4921c --- /dev/null +++ b/data_only_viz/icp_fusion_worker.py @@ -0,0 +1,72 @@ +"""Threaded wrapper that polls State and calls FusionWorker.run_once. + +ICP fusion runs as a background thread parallel to the autonomous +Multi-HMR worker. It pulls the latest LiDAR frame from a +LidarTCPReader, stages it into State, and applies in-place ICP +registration to ``state.persons_smplx[*].vertices_3d``. + +Opt-in via ``ICP_FUSION=1`` from main.py. +""" +from __future__ import annotations + +import logging +import threading +import time +from typing import Optional + +from .icp_fusion import FusionWorker, IcpConfig +from .lidar_calib import load_extrinsic +from .lidar_receiver import LidarTCPReader + +_LOG = logging.getLogger(__name__) + + +class IcpFusionThread: + """Background thread: pull LiDAR frames, run FusionWorker on state.""" + + def __init__(self, state, host: str, port: int, + target_hz: float = 8.0) -> None: + self._state = state + self._reader = LidarTCPReader(host=host, port=port) + self._worker = FusionWorker(extrinsic=load_extrinsic(), + config=IcpConfig()) + self._period_s = 1.0 / max(target_hz, 0.5) + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + + def start(self) -> None: + if self._thread is not None: + return + self._reader.start() + self._thread = threading.Thread( + target=self._run, name="icp-fusion", daemon=True) + self._thread.start() + _LOG.info("icp-fusion thread started") + + def stop(self) -> None: + self._stop.set() + self._reader.stop() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + def _run(self) -> None: + while not self._stop.is_set(): + t0 = time.monotonic() + frame = self._reader.latest() + if frame is not None and self._state.persons_smplx: + # State doesn't expose a fine-grained lock for these + # fields here; rely on FusionWorker.run_once being + # write-only on persons_smplx[*].vertices_3d (replace in + # place) and the readers being tolerant of mid-update. + self._state.lidar_points = frame.points + self._state.lidar_timestamp_ns = frame.timestamp_ns + try: + self._state.icp_metadata = self._worker.run_once( + self._state) + except Exception as exc: # noqa: BLE001 + _LOG.warning("icp fusion failed: %s", exc) + self._state.icp_metadata = None + elapsed = time.monotonic() - t0 + if self._stop.wait(max(0.0, self._period_s - elapsed)): + return diff --git a/data_only_viz/iphone_osc_listener.py b/data_only_viz/iphone_osc_listener.py new file mode 100644 index 0000000..72c33e9 --- /dev/null +++ b/data_only_viz/iphone_osc_listener.py @@ -0,0 +1,118 @@ +"""OSC UDP listener for the iOS ARBodyTracker app. + +Subscribes to /body3d/kp on UDP :57128 (distinct from MediaPipe +output :57126). Each /body3d/kp pid joint_idx x y z message stores +one joint of ARKit's 91-joint ARSkeleton3D into +state.persons_arkit_joints[pid] (np.ndarray shape (91, 3), float32). +A background GC drops pids whose last_t is older than 1.0 s. + +Worker pattern mirrors osc_listener.OscListener. +""" +from __future__ import annotations + +import logging +import threading +import time +from typing import Any + +import numpy as np +from pythonosc import dispatcher, osc_server + +from .state import State + +LOG = logging.getLogger("iphone_osc") + +IPHONE_OSC_PORT = 57128 +ARKIT_NUM_JOINTS = 91 +STALE_SEC = 1.0 + + +class IphoneOSCListener: + def __init__(self, state: State, host: str = "0.0.0.0", + port: int = IPHONE_OSC_PORT) -> None: + self.state = state + self.host = host + self.port = port + self._server: osc_server.ThreadingOSCUDPServer | None = None + self._server_thread: threading.Thread | None = None + self._gc_thread: threading.Thread | None = None + self._stop = threading.Event() + self._last_hb: float = 0.0 + + def start(self) -> None: + d = dispatcher.Dispatcher() + d.map("/body3d/kp", self._on_kp) + d.map("/body3d/count", self._on_count) + self._server = osc_server.ThreadingOSCUDPServer( + (self.host, self.port), d) + self._server_thread = threading.Thread( + target=self._server.serve_forever, + name="iphone_osc", daemon=True) + self._server_thread.start() + self._gc_thread = threading.Thread( + target=self._gc_loop, name="iphone_gc", daemon=True) + self._gc_thread.start() + LOG.info("iphone OSC listening on %s:%d", self.host, self.port) + + def stop(self) -> None: + self._stop.set() + if self._server is not None: + self._server.shutdown() + self._server.server_close() + self._server = None + if self._server_thread is not None: + self._server_thread.join(timeout=2.0) + self._server_thread = None + if self._gc_thread is not None: + self._gc_thread.join(timeout=2.0) + self._gc_thread = None + + def _on_kp(self, _addr: str, *args: Any) -> None: + if len(args) < 5: + return + try: + pid = int(args[0]) + joint_idx = int(args[1]) + x = float(args[2]) + y = float(args[3]) + z = float(args[4]) + except (TypeError, ValueError): + return + if not (0 <= joint_idx < ARKIT_NUM_JOINTS): + return + with self.state.lock(): + arr = self.state.persons_arkit_joints.get(pid) + if arr is None or arr.shape != (ARKIT_NUM_JOINTS, 3): + arr = np.zeros((ARKIT_NUM_JOINTS, 3), dtype=np.float32) + self.state.persons_arkit_joints[pid] = arr + arr[joint_idx] = (x, y, z) + self.state.persons_arkit_last_t[pid] = time.perf_counter() + + def _on_count(self, _addr: str, *args: Any) -> None: + # Optional : we currently don't gate on count, but parse for log. + if not args: + return + try: + n = int(args[0]) + except (TypeError, ValueError): + return + now = time.monotonic() + if now - self._last_hb > 5.0: + self._last_hb = now + LOG.info("hb: %d ARKit bodies live", n) + + def _gc_stale(self) -> None: + cutoff = time.perf_counter() - STALE_SEC + with self.state.lock(): + drop = [ + pid for pid, t in self.state.persons_arkit_last_t.items() + if t < cutoff + ] + for pid in drop: + self.state.persons_arkit_joints.pop(pid, None) + self.state.persons_arkit_last_t.pop(pid, None) + + def _gc_loop(self) -> None: + while not self._stop.is_set(): + self._gc_stale() + time.sleep(0.5) diff --git a/data_only_viz/lidar_calib.py b/data_only_viz/lidar_calib.py new file mode 100644 index 0000000..ea0513e --- /dev/null +++ b/data_only_viz/lidar_calib.py @@ -0,0 +1,83 @@ +"""iPhone LiDAR (ARKit world) <-> webcam (Multi-HMR camera) extrinsic. + +Persisted as a small JSON document so calibration survives across launches. +The default location is ``~/.config/av-live/lidar_extrinsic.json``; override +with the ``ICP_LIDAR_EXTRINSIC`` env var. +""" +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np + +DEFAULT_EXTRINSIC_PATH = Path.home() / ".config" / "av-live" / "lidar_extrinsic.json" + + +@dataclass +class Extrinsic: + """4x4 rigid transform from ARKit world frame to Multi-HMR camera frame.""" + + T_arkit_to_cam: np.ndarray = field(default_factory=lambda: np.eye(4)) + confidence: float = 0.0 + captured_at_iso: str = "" + + @staticmethod + def identity() -> "Extrinsic": + return Extrinsic(T_arkit_to_cam=np.eye(4), confidence=0.0, captured_at_iso="") + + +def save_extrinsic(e: Extrinsic, path: Path | None = None) -> Path: + path = Path(path) if path is not None else _path_from_env() + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "T_arkit_to_cam": e.T_arkit_to_cam.astype(float).tolist(), + "confidence": float(e.confidence), + "captured_at_iso": e.captured_at_iso, + } + path.write_text(json.dumps(payload, indent=2)) + return path + + +def load_extrinsic(path: Path | None = None) -> Extrinsic: + path = Path(path) if path is not None else _path_from_env() + if not path.exists(): + return Extrinsic.identity() + payload = json.loads(path.read_text()) + return Extrinsic( + T_arkit_to_cam=np.array(payload["T_arkit_to_cam"], dtype=np.float64), + confidence=float(payload.get("confidence", 0.0)), + captured_at_iso=str(payload.get("captured_at_iso", "")), + ) + + +def _path_from_env() -> Path: + p = os.environ.get("ICP_LIDAR_EXTRINSIC") + return Path(p) if p else DEFAULT_EXTRINSIC_PATH + + +def kabsch_rigid(src: np.ndarray, tgt: np.ndarray) -> np.ndarray: + """Closed-form rigid alignment (Kabsch via SVD). + + Returns a 4x4 transform T such that ``tgt ≈ (src @ R.T) + t``. + """ + src = np.asarray(src, dtype=np.float64) + tgt = np.asarray(tgt, dtype=np.float64) + if src.shape != tgt.shape: + raise ValueError(f"shape mismatch: src={src.shape} tgt={tgt.shape}") + if src.shape[0] < 3 or src.shape[1] != 3: + raise ValueError("kabsch_rigid needs at least 3 paired 3D points") + src_c = src.mean(axis=0) + tgt_c = tgt.mean(axis=0) + H = (src - src_c).T @ (tgt - tgt_c) + U, _, Vt = np.linalg.svd(H) + d = np.linalg.det(Vt.T @ U.T) + D = np.diag([1.0, 1.0, np.sign(d)]) + R = Vt.T @ D @ U.T + t = tgt_c - R @ src_c + T = np.eye(4) + T[:3, :3] = R + T[:3, 3] = t + return T diff --git a/data_only_viz/lidar_receiver.py b/data_only_viz/lidar_receiver.py new file mode 100644 index 0000000..1263fe5 --- /dev/null +++ b/data_only_viz/lidar_receiver.py @@ -0,0 +1,130 @@ +"""TCP receiver for iPhone ARBodyTracker LiDAR ARMeshAnchor stream. + +Wire format (per frame, after the 4-byte big-endian length prefix consumed +by the socket reader): + + [uint64 BE timestamp_ns] + [uint32 BE vertex_count] + [float32 LE x y z] * vertex_count + +The decoder is pure and side-effect-free so it can be unit-tested without a +socket. The socket reader lives in a separate class (LidarTCPReader) so its +threading model is independently testable. +""" +from __future__ import annotations + +import struct +from dataclasses import dataclass + +import numpy as np + +_HEADER = struct.Struct(">QI") # timestamp_ns, vertex_count + + +@dataclass(frozen=True) +class LidarFrame: + """One decoded LiDAR frame from the iPhone.""" + + timestamp_ns: int + points: np.ndarray # shape (N, 3), float32, ARKit world frame (meters) + + +def decode_frame(body: bytes) -> LidarFrame: + """Decode a frame body (length prefix already stripped).""" + if len(body) < _HEADER.size: + raise ValueError(f"truncated frame: header needs {_HEADER.size} bytes, got {len(body)}") + timestamp_ns, vertex_count = _HEADER.unpack_from(body, 0) + if vertex_count == 0: + raise ValueError("vertex_count must be > 0") + expected = _HEADER.size + vertex_count * 12 + if len(body) < expected: + raise ValueError(f"truncated frame: need {expected} bytes for {vertex_count} verts, got {len(body)}") + raw = body[_HEADER.size : expected] + pts = np.frombuffer(raw, dtype="I") + + +class LidarTCPReader: + """Background TCP reader producing a single-slot latest-frame mailbox. + + Reconnects on transient failures with linear backoff up to 5s. + """ + + def __init__(self, host: str, port: int, connect_timeout_s: float = 2.0) -> None: + self._host = host + self._port = port + self._connect_timeout_s = connect_timeout_s + self._stop = threading.Event() + self._lock = threading.Lock() + self._latest: Optional[LidarFrame] = None + self._thread: Optional[threading.Thread] = None + + def start(self) -> None: + if self._thread is not None: + return + self._thread = threading.Thread(target=self._run, name="lidar-tcp", daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + def latest(self) -> Optional[LidarFrame]: + with self._lock: + return self._latest + + def _run(self) -> None: + backoff_s = 0.5 + while not self._stop.is_set(): + try: + with socket.create_connection((self._host, self._port), timeout=self._connect_timeout_s) as sock: + sock.settimeout(1.0) + backoff_s = 0.5 + self._read_loop(sock) + except (OSError, ValueError) as exc: + _LOG.warning("lidar reader: %s; reconnecting in %.1fs", exc, backoff_s) + if self._stop.wait(backoff_s): + return + backoff_s = min(backoff_s * 2.0, 5.0) + + def _read_loop(self, sock: socket.socket) -> None: + while not self._stop.is_set(): + header = self._recv_exact(sock, _LEN_PREFIX.size) + if header is None: + return + (length,) = _LEN_PREFIX.unpack(header) + if length <= 0 or length > 8_000_000: # sanity cap: 8 MB per frame + raise ValueError(f"implausible frame length {length}") + body = self._recv_exact(sock, length) + if body is None: + return + frame = decode_frame(body) + with self._lock: + self._latest = frame + + def _recv_exact(self, sock: socket.socket, n: int) -> Optional[bytes]: + buf = bytearray(n) + view = memoryview(buf) + got = 0 + while got < n: + if self._stop.is_set(): + return None + try: + k = sock.recv_into(view[got:]) + except socket.timeout: + continue + if k == 0: + return None + got += k + return bytes(buf) diff --git a/data_only_viz/main.py b/data_only_viz/main.py index 0be7474..dbdadaa 100644 --- a/data_only_viz/main.py +++ b/data_only_viz/main.py @@ -249,6 +249,40 @@ class AppDelegate(NSObject): # 2. Apple Vision body pose (fallback si MediaPipe casse) # 3. CoreML pose, DETRPose, Holistic, YOLO — fallbacks import os as _os + # iPhone ARBodyTracker (option 2 LiDAR fusion) : always-on + # listener on :57128. Harmless if no iPhone is broadcasting ; + # state.persons_arkit_joints stays empty and the arkit_fuse + # stage no-ops. Activated via POSE_FILTER=...+arkit_fuse. + try: + from .iphone_osc_listener import IphoneOSCListener + self._iphone_osc = IphoneOSCListener(self._state) + self._iphone_osc.start() + LOG.info("worker: + iPhone OSC listener :57128") + except Exception as e: # noqa: BLE001 + LOG.warning("iphone OSC listener start failed (%s)", e) + # ICP LiDAR fusion (opt-in via ICP_FUSION=1). Parallel to the + # ARKit pelvis fuse: ICP operates on SMPL-X dense vertices, not + # joints. Requires a calibrated extrinsic on disk (see + # scripts/calibrate_lidar.py) and an iPhone LiDAR stream + # broadcasting on ICP_LIDAR_HOST:ICP_LIDAR_PORT. + if _os.environ.get("ICP_FUSION", "0") == "1": + host = _os.environ.get("ICP_LIDAR_HOST") + if not host: + LOG.warning("ICP_FUSION=1 but ICP_LIDAR_HOST unset — " + "fusion disabled") + else: + try: + from .icp_fusion_worker import IcpFusionThread + self._icp_fusion = IcpFusionThread( + self._state, + host=host, + port=int(_os.environ.get("ICP_LIDAR_PORT", "5500")), + ) + self._icp_fusion.start() + LOG.info("worker: + ICP LiDAR fusion -> %s:%s", host, + _os.environ.get("ICP_LIDAR_PORT", "5500")) + except Exception as e: # noqa: BLE001 + LOG.warning("icp fusion start failed (%s)", e) # 0. Multi-HMR (SMPL-X 10475 verts mesh dense) — opt-in via flag if getattr(self._opts, "multi_hmr", False): try: @@ -585,6 +619,12 @@ class AppDelegate(NSObject): self._listener.stop() if self._pose_worker is not None: self._pose_worker.stop() + icp = getattr(self, "_icp_fusion", None) + if icp is not None: + try: + icp.stop() + except Exception as e: # noqa: BLE001 + LOG.warning("icp fusion stop failed (%s)", e) LOG.info("bye") diff --git a/data_only_viz/mesh_rigger.py b/data_only_viz/mesh_rigger.py index 3eda2fc..7638be5 100644 --- a/data_only_viz/mesh_rigger.py +++ b/data_only_viz/mesh_rigger.py @@ -14,6 +14,8 @@ Limitations connues (premiere iteration) : """ from __future__ import annotations +import collections +import logging import math import threading import time @@ -21,8 +23,16 @@ from dataclasses import dataclass, field import numpy as np +try: + from scipy.optimize import linear_sum_assignment + _HAVE_SCIPY = True +except ImportError: # noqa: BLE001 + _HAVE_SCIPY = False + from .state import PoseKp, SMPLXPerson, State +LOG = logging.getLogger("mesh_rigger") + # Indices MediaPipe POSE_LANDMARKS pour les hanches (pelvis 2D = midpoint). _LEFT_HIP = 23 @@ -55,6 +65,70 @@ def _pelvis_2d_from_body(body: list[PoseKp]) -> tuple[float, float] | None: return (0.5 * (lh.x + rh.x), 0.5 * (lh.y + rh.y)) +def _body_bbox_norm( + body: list[PoseKp], +) -> tuple[float, float, float, float] | None: + """Bbox image-normalized [0,1] from a list of body landmarks + (Vision 19 joints OR MediaPipe 33). None if not enough confident + points.""" + if not body: + return None + xs = [kp.x for kp in body if kp.c > 0.05] + ys = [kp.y for kp in body if kp.c > 0.05] + if len(xs) < 4 or len(ys) < 4: + return None + x0, x1 = max(0.0, min(xs)), min(1.0, max(xs)) + y0, y1 = max(0.0, min(ys)), min(1.0, max(ys)) + # Pad 10% to capture full body silhouette. + dx = (x1 - x0) * 0.10 + dy = (y1 - y0) * 0.10 + x0 = max(0.0, x0 - dx); x1 = min(1.0, x1 + dx) + y0 = max(0.0, y0 - dy); y1 = min(1.0, y1 + dy) + if x1 - x0 < 0.02 or y1 - y0 < 0.02: + return None + return (x0, y0, x1, y1) + + +def _mesh_bbox_norm(p: SMPLXPerson) -> tuple[float, float, float, float] | None: + """Project SMPL-X mesh vertices to image-normalized bbox. + + Multi-HMR uses focal = IMG_SIZE camera intrinsics. World verts + have z>0 (in front of camera).""" + v = np.asarray(p.vertices_3d, dtype=np.float32) + if v.size == 0 or v.shape[0] < 100: + return None + z = v[:, 2] + valid = z > 1e-3 + if not np.any(valid): + return None + x_img = (v[valid, 0] * _FOCAL / z[valid]) / _IMG_SIZE + 0.5 + y_img = (v[valid, 1] * _FOCAL / z[valid]) / _IMG_SIZE + 0.5 + x0, x1 = float(x_img.min()), float(x_img.max()) + y0, y1 = float(y_img.min()), float(y_img.max()) + x0 = max(0.0, x0); x1 = min(1.0, x1) + y0 = max(0.0, y0); y1 = min(1.0, y1) + if x1 - x0 < 0.02 or y1 - y0 < 0.02: + return None + return (x0, y0, x1, y1) + + +def _iou_norm( + a: tuple[float, float, float, float], + b: tuple[float, float, float, float], +) -> float: + ax0, ay0, ax1, ay1 = a + bx0, by0, bx1, by1 = b + ix0 = max(ax0, bx0); iy0 = max(ay0, by0) + ix1 = min(ax1, bx1); iy1 = min(ay1, by1) + iw = max(0.0, ix1 - ix0); ih = max(0.0, iy1 - iy0) + inter = iw * ih + if inter <= 0: + return 0.0 + a_area = (ax1 - ax0) * (ay1 - ay0) + b_area = (bx1 - bx0) * (by1 - by0) + return float(inter / (a_area + b_area - inter + 1e-9)) + + def _vision_pid_match( keyframe_pelvis_2d: tuple[float, float] | None, vision_bodies: list[list[PoseKp]], @@ -89,14 +163,22 @@ class MeshRigger: Thread-safe : ne mute pas le state, retourne une nouvelle liste. """ - def __init__(self, state: State, hold_window_s: float = 1.5) -> None: + def __init__(self, state: State, hold_window_s: float = 1.5, + dino_weight: float = 0.5, + dino_reid=None) -> None: self.state = state self.hold_window_s = hold_window_s + self.dino_weight = float(dino_weight) + self.dino_reid = dino_reid self._lock = threading.Lock() # pid Multi-HMR -> keyframe self._keyframes: dict[int, _Keyframe] = {} # pid Multi-HMR -> pid Vision matched (sticky across frames) self._vision_pid_map: dict[int, int] = {} + # pid Multi-HMR -> recent DINO embeddings (mean -> reid signature) + self._pid_embeddings: dict[int, collections.deque] = {} + # Cached log throttle + self._next_dino_log = 0.0 def apply( self, @@ -114,6 +196,14 @@ class MeshRigger: if old_pid not in current_pids: self._keyframes.pop(old_pid, None) self._vision_pid_map.pop(old_pid, None) + self._pid_embeddings.pop(old_pid, None) + + # 2) DINO fusion: if a reid backend is wired, try Hungarian + # over (mesh keyframe pids) x (Vision body pids) using + # alpha*IoU + (1-alpha)*cosine. This only kicks in when a + # keyframe is detected this call AND we have an RGB frame. + self._dino_match(persons_smplx, persons_body, + persons_body_ids) out: list[SMPLXPerson] = [] for person in persons_smplx: @@ -199,6 +289,136 @@ class MeshRigger: )) return out + # ------------------------------------------------------------------ + # DINOv2 reid hooks + # ------------------------------------------------------------------ + def _dino_match( + self, + persons_smplx: list[SMPLXPerson], + persons_body: list[list[PoseKp]], + persons_body_ids: list[int], + ) -> None: + """Update self._vision_pid_map and self._pid_embeddings by + matching mesh pids against Vision pids on alpha*IoU + + (1-alpha)*DINO cosine. No-op if any prerequisite missing. + + Caller must hold self._lock.""" + if self.dino_reid is None or not _HAVE_SCIPY: + return + if not persons_smplx or not persons_body: + return + # Need at least one new keyframe to be worth running DINO. + new_kf_pids: list[int] = [] + for p in persons_smplx: + kf = self._keyframes.get(p.pid) + if kf is None or not np.allclose( + kf.translation, p.translation, atol=1e-4): + new_kf_pids.append(int(p.pid)) + if not new_kf_pids: + return + + # Acquire current RGB frame (best effort, no double lock). + frame = self.state.last_frame_rgb + if frame is None: + return + H, W = frame.shape[:2] + + # Build Vision bboxes (image-normalized) and pixel crops. + v_bboxes_norm: list[tuple[float, float, float, float]] = [] + v_crops: list[np.ndarray] = [] + v_pids: list[int] = [] + for body, vpid in zip(persons_body, persons_body_ids): + bb = _body_bbox_norm(body) + if bb is None: + continue + x0, y0, x1, y1 = bb + px0 = max(0, int(x0 * W)) + py0 = max(0, int(y0 * H)) + px1 = min(W, int(x1 * W)) + py1 = min(H, int(y1 * H)) + if px1 <= px0 + 4 or py1 <= py0 + 4: + continue + v_bboxes_norm.append(bb) + v_crops.append(frame[py0:py1, px0:px1].copy()) + v_pids.append(int(vpid)) + + if not v_crops: + return + + # Build mesh bboxes (image-normalized) from world pelvis proj. + m_bboxes_norm: list[tuple[float, float, float, float]] = [] + m_pids_keep: list[int] = [] + m_crops: list[np.ndarray] = [] + for p in persons_smplx: + bb = _mesh_bbox_norm(p) + if bb is None: + continue + m_bboxes_norm.append(bb) + m_pids_keep.append(int(p.pid)) + x0, y0, x1, y1 = bb + px0 = max(0, int(x0 * W)) + py0 = max(0, int(y0 * H)) + px1 = min(W, int(x1 * W)) + py1 = min(H, int(y1 * H)) + if px1 > px0 + 4 and py1 > py0 + 4: + m_crops.append(frame[py0:py1, px0:px1].copy()) + else: + m_crops.append(None) # type: ignore[arg-type] + + if not m_bboxes_norm: + return + + # Embed Vision crops in one batch (still loops internally). + t0 = time.perf_counter() + try: + v_emb = self.dino_reid.embed_crops(v_crops) + except Exception as e: # noqa: BLE001 + LOG.warning("dino_reid embed failed: %s", e) + return + + # Build cost matrix mesh x vision : 1 - (alpha*IoU + (1-alpha)*cos) + n_m = len(m_bboxes_norm) + n_v = len(v_bboxes_norm) + alpha = float(np.clip(self.dino_weight, 0.0, 1.0)) + cost = np.ones((n_m, n_v), dtype=np.float32) + for i, mbb in enumerate(m_bboxes_norm): + hist = self._pid_embeddings.get(m_pids_keep[i]) + mean_emb = None + if hist: + stack = np.stack(list(hist), axis=0) + mean_emb = stack.mean(axis=0) + n = np.linalg.norm(mean_emb) + 1e-8 + mean_emb = mean_emb / n + for j, vbb in enumerate(v_bboxes_norm): + iou = _iou_norm(mbb, vbb) + if mean_emb is not None: + cos = float(np.dot(mean_emb, v_emb[j])) + else: + cos = iou # no history -> trust IoU + score = alpha * iou + (1.0 - alpha) * max(0.0, cos) + cost[i, j] = 1.0 - score + + rr, cc = linear_sum_assignment(cost) + for i, j in zip(rr, cc): + if cost[i, j] >= 0.95: + continue # weak match, ignore + mpid = m_pids_keep[i] + self._vision_pid_map[mpid] = v_pids[j] + # Update embedding history for THIS mesh pid using the + # Vision crop (most recent visual evidence). + dq = self._pid_embeddings.setdefault( + mpid, collections.deque(maxlen=10)) + dq.append(v_emb[j].copy()) + + now = time.monotonic() + dt_ms = (time.perf_counter() - t0) * 1e3 + if now >= self._next_dino_log: + LOG.info( + "dino_reid: embedded %d crops in %.1f ms (alpha=%.2f, " + "matched %d mesh<->vision pairs)", + len(v_crops), dt_ms, alpha, min(n_m, n_v)) + self._next_dino_log = now + 5.0 + @staticmethod def _project_pelvis( translation: np.ndarray, diff --git a/data_only_viz/multi.py b/data_only_viz/multi.py index 3f6ed0c..f543069 100644 --- a/data_only_viz/multi.py +++ b/data_only_viz/multi.py @@ -22,6 +22,8 @@ from pathlib import Path from .action_head_pub import ActionHeadPublisher from .euro_filter import SkeletonFilter from .pose_bridge import PoseSoundBridge +from .pose_filter import PoseFilterChain +from .pose_filter import _is_finite # noqa: PLC2701 (intentional internal use) from .state import Kp3D, PoseKp, State from .tracker import IoUTracker @@ -96,6 +98,179 @@ class MultiWorker: self._sound_bridge = PoseSoundBridge(throttle_hz=30.0) self._action_pub = ActionHeadPublisher(state=self.state, bridge=self._sound_bridge) self._action_pub.start() + # 3D pose filter chain : median, Kalman CV, lookahead, IK clamps. + self._filter_chain = PoseFilterChain(state=self.state) + # Discrimination state : per-pid frame counters for hysteresis. + # _pid_lifetime : frames since pid created (visible). + # _pid_last_bbox : last bbox seen for active pid (for re-association). + # _pid_missing : frames since pid disappeared (None when active). + self._pid_lifetime: dict[int, int] = {} + self._pid_missing: dict[int, int] = {} + self._pid_last_bbox: dict[int, tuple[float, float, float, float]] = {} + # Discrimination thresholds — tunable via env. + import os as _os + self._ghost_min_visible = int(_os.environ.get("POSE_GHOST_MIN_VISIBLE", "10")) + self._ghost_min_conf = float(_os.environ.get("POSE_GHOST_MIN_CONF", "0.5")) + self._hand_min_visible = int(_os.environ.get("POSE_HAND_MIN_VISIBLE", "15")) + self._face_min_visible = int(_os.environ.get("POSE_FACE_MIN_VISIBLE", "50")) + self._nms_iou = float(_os.environ.get("POSE_NMS_IOU", "0.7")) + # Counters exposed for debug. + self._n_ghost_dropped = 0 + self._n_hand_dropped = 0 + self._n_face_dropped = 0 + + # ------------------------------------------------------------------ + # Discrimination helpers — body ghost rejection, NMS, pid hysteresis, + # face/hand visibility gates. All return filtered (kps, ids) lists. + # ------------------------------------------------------------------ + @staticmethod + def _bbox_from_kps(kps: list) -> tuple[float, float, float, float]: + if not kps: + return (0.0, 0.0, 0.0, 0.0) + xs = [kp.x for kp in kps] + ys = [kp.y for kp in kps] + return (min(xs), min(ys), max(xs), max(ys)) + + @staticmethod + def _iou(a: tuple[float, float, float, float], + b: tuple[float, float, float, float]) -> float: + ix1 = max(a[0], b[0]); iy1 = max(a[1], b[1]) + ix2 = min(a[2], b[2]); iy2 = min(a[3], b[3]) + iw = max(0.0, ix2 - ix1); ih = max(0.0, iy2 - iy1) + inter = iw * ih + aw = max(0.0, a[2] - a[0]) * max(0.0, a[3] - a[1]) + bw = max(0.0, b[2] - b[0]) * max(0.0, b[3] - b[1]) + u = aw + bw - inter + return inter / u if u > 1e-9 else 0.0 + + def _reject_ghosts_and_nms( + self, + bodies: list[list], + bodies3d: list[list[Kp3D]], + ids_body: list[int], + ) -> tuple[list[list], list[list[Kp3D]], list[int]]: + """Drop body detections with = self._ghost_min_conf + and _is_finite(kp.x) and _is_finite(kp.y)) + if n_visible < self._ghost_min_visible: + keep_mask[i] = False + self._n_ghost_dropped += 1 + scores.append( + sum(kp.c for kp in kps) / len(kps) if kps else 0.0) + # NMS on remaining bboxes. + bboxes = [self._bbox_from_kps(kps) for kps in bodies] + order = sorted( + [i for i in range(len(bodies)) if keep_mask[i]], + key=lambda i: -scores[i]) + kept_order: list[int] = [] + for i in order: + drop = False + for j in kept_order: + if self._iou(bboxes[i], bboxes[j]) > self._nms_iou: + drop = True + break + if drop: + keep_mask[i] = False + else: + kept_order.append(i) + new_bodies = [bodies[i] for i in range(len(bodies)) if keep_mask[i]] + new_ids = [ids_body[i] for i in range(len(bodies)) + if i < len(ids_body) and keep_mask[i]] + # bodies3d aligned 1:1 with bodies. + new_b3d: list[list[Kp3D]] = [] + if bodies3d: + for i in range(min(len(bodies), len(bodies3d))): + if keep_mask[i]: + new_b3d.append(bodies3d[i]) + return new_bodies, new_b3d, new_ids + + def _apply_pid_hysteresis( + self, + bodies: list[list], + ids_body: list[int], + ) -> list[int]: + """Reuse a recently-disappeared pid when a young pid lands near + its last bbox. Mutates self._pid_lifetime / _pid_missing / + _pid_last_bbox in place. Returns possibly-remapped ids. + """ + # Tick all known pids missing counter ; will reset for visible ones. + for pid in list(self._pid_missing.keys()): + self._pid_missing[pid] += 1 + if self._pid_missing[pid] > 60: # forget after 2 s @30 fps + self._pid_missing.pop(pid, None) + self._pid_last_bbox.pop(pid, None) + self._pid_lifetime.pop(pid, None) + new_ids = list(ids_body) + for i, pid in enumerate(ids_body): + if pid < 0 or i >= len(bodies): + continue + bbox_i = self._bbox_from_kps(bodies[i]) + # If this pid is brand new (<10 frames) and we have an absent + # older pid (>=30 frames lifetime, <30 frames missing) with a + # close bbox, remap. + age = self._pid_lifetime.get(pid, 0) + if age < 10: + best_old: int | None = None + best_iou = 0.0 + for old_pid, miss in self._pid_missing.items(): + if old_pid == pid: + continue + if self._pid_lifetime.get(old_pid, 0) < 30: + continue + if miss > 30: + continue + old_bbox = self._pid_last_bbox.get(old_pid) + if old_bbox is None: + continue + iou = self._iou(bbox_i, old_bbox) + if iou > 0.3 and iou > best_iou: + best_iou = iou + best_old = old_pid + if best_old is not None: + new_ids[i] = best_old + pid = best_old + # Bookkeeping for visible pid. + self._pid_lifetime[pid] = self._pid_lifetime.get(pid, 0) + 1 + self._pid_missing.pop(pid, None) + self._pid_last_bbox[pid] = bbox_i + # Pids previously visible but absent this frame -> mark missing. + visible = set(new_ids) + for pid in list(self._pid_lifetime.keys()): + if pid not in visible and pid not in self._pid_missing: + self._pid_missing[pid] = 1 + return new_ids + + def _drop_low_visibility( + self, + kps_list: list[list], + ids: list[int], + min_visible: int, + which: str, + ) -> tuple[list[list], list[int]]: + out_kps: list[list] = [] + out_ids: list[int] = [] + for i, kps in enumerate(kps_list): + n_ok = sum( + 1 for kp in kps + if _is_finite(kp.x) and _is_finite(kp.y) + and (kp.x != 0.0 or kp.y != 0.0)) + if n_ok < min_visible: + if which == "face": + self._n_face_dropped += 1 + else: + self._n_hand_dropped += 1 + continue + out_kps.append(kps) + out_ids.append(ids[i] if i < len(ids) else -1) + return out_kps, out_ids def start(self) -> None: self._thread = threading.Thread( @@ -238,6 +413,14 @@ class MultiWorker: ids_body = self._tracker_body.update(bodies) ids_face = self._tracker_face.update(faces) ids_hand = self._tracker_hand.update(hands) + # --- Discrimination : ghost reject + NMS + pid hysteresis -- + bodies, bodies3d, ids_body = self._reject_ghosts_and_nms( + bodies, bodies3d, ids_body) + ids_body = self._apply_pid_hysteresis(bodies, ids_body) + faces, ids_face = self._drop_low_visibility( + faces, ids_face, self._face_min_visible, "face") + hands, ids_hand = self._drop_low_visibility( + hands, ids_hand, self._hand_min_visible, "hand") # --- Lissage One Euro par keypoint ------------------------- t_now = time.monotonic() bodies = [_smooth_kps(self._smooth_body, ids_body[i], kps, t_now) @@ -246,11 +429,21 @@ class MultiWorker: for i, kps in enumerate(faces)] hands = [_smooth_kps(self._smooth_hand, ids_hand[i], kps, t_now) for i, kps in enumerate(hands)] + # --- Filter chain face + hands (median + Kalman 2D + lookahead) + faces = self._filter_chain.apply_face(faces, ids_face, t_now) + hands = self._filter_chain.apply_hand(hands, ids_hand, None, t_now) # Pont sonore : envoi OSC /pose/* a sclang (body + face + hands) # 3D world landmarks share ids with bodies (same MediaPipe # detection, just a different coordinate space). ids_body3d = ids_body[:len(bodies3d)] if bodies3d else [] + if bodies3d: + bodies3d = self._filter_chain.apply(bodies3d, ids_body3d, t_now) + # Debug : log body3d count once / 5 s so we know MediaPipe + # actually populates pose_world_landmarks. + if not hasattr(self, "_dbg_b3d_t") or t_now - self._dbg_b3d_t > 5.0: + LOG.info("body3d: n=%d (pose_world_landmarks)", len(bodies3d)) + self._dbg_b3d_t = t_now self._sound_bridge.send( bodies, ids_body, t_now, persons_face=faces, persons_face_ids=ids_face, diff --git a/data_only_viz/multi_hmr_worker.py b/data_only_viz/multi_hmr_worker.py index a00b6dc..e4c9a34 100644 --- a/data_only_viz/multi_hmr_worker.py +++ b/data_only_viz/multi_hmr_worker.py @@ -20,6 +20,7 @@ from pathlib import Path import numpy as np +from .arkit_joint_map import ARKIT_PELVIS_IDX from .euro_filter import OneEuroFilter from .state import PoseKp, SMPLXPerson, State from .tracker import IoUTracker @@ -30,12 +31,34 @@ CACHE = Path.home() / ".cache" / "av-live-multihmr" CKPT = CACHE / "checkpoints" / "multiHMR_672_S.pt" SMPLX_PATH = CACHE / "models" / "smplx" / "SMPLX_NEUTRAL.npz" MULTIHMR_REPO = CACHE / "multi-hmr" -COREML_MLPACKAGE = CACHE / "multihmr_full_672_s.mlpackage" +COREML_MLPACKAGE = Path( + os.environ.get("COREML_MLPACKAGE") + or str(CACHE / "multihmr_full_672_s.mlpackage")) IMG_SIZE = 672 N_VERTS = 10475 +def arkit_pelvis_z_override(state, pid: int, z_pred: float, + fresh_sec: float = 1.0) -> float: + """Return ARKit pelvis world-z if a fresh ARKit frame exists for + this pid, otherwise return the Multi-HMR predicted z unchanged. + + Used to resolve Multi-HMR's monocular scale ambiguity: ARKit's + LiDAR-anchored pelvis position is ground truth in the iPhone + world frame, which (after extrinsics calibration) is the same + metric scale as the SMPL-X cam-space output. + """ + with state.lock(): + arr = state.persons_arkit_joints.get(pid) + last_t = state.persons_arkit_last_t.get(pid, 0.0) + if arr is None: + return float(z_pred) + if time.perf_counter() - last_t > fresh_sec: + return float(z_pred) + return float(arr[ARKIT_PELVIS_IDX, 2]) + + class MultiHMRWorker: def __init__(self, state: State, num_persons: int = 4, target_fps: float = 10.0, device: str = "mps", @@ -77,6 +100,10 @@ class MultiHMRWorker: # (cf tracker.py) pour resister aux occlusions et au mouvement # rapide. Multi-HMR a 3 fps -> 30 frames = 10s de survie. self._tracker = IoUTracker(iou_threshold=0.15, max_miss=30) + # Lazily-loaded CoreML backend for predict_once (single-shot, + # off-thread). Independent of the worker thread's _run_coreml + # backend instance — predict_once must work even without start(). + self._coreml_backend_singleshot = None @staticmethod def is_available() -> bool: @@ -93,6 +120,83 @@ class MultiHMRWorker: def stop(self) -> None: self._stop.set() + def _get_or_load_coreml_backend(self): + """Lazily load the CoreML backend for single-shot inference. + + Returns the cached `MultiHMRCoreMLBackend` instance, or None if + the backend cannot be imported / the .mlpackage is missing. + Thread-safe enough for our use (calibration CLI is single- + threaded; the worker thread uses its own backend in _run_coreml). + """ + if self._coreml_backend_singleshot is not None: + return self._coreml_backend_singleshot + try: + from .multihmr_coreml import MultiHMRCoreMLBackend + backend = MultiHMRCoreMLBackend(COREML_MLPACKAGE) + except (ImportError, FileNotFoundError) as e: + LOG.info("predict_once: CoreML backend unavailable: %s", e) + return None + except Exception as e: # noqa: BLE001 + LOG.warning("predict_once: CoreML backend init failed: %s", e) + return None + self._coreml_backend_singleshot = backend + return backend + + def predict_once(self, rgb_image): + """Single-shot SMPL-X prediction on one RGB image. + + Args: + rgb_image: (H, W, 3) uint8 RGB array. Will be center- + cropped + resized to 672x672 internally. + + Returns: + First `SMPLXPerson` detection (pid=0) or None if no + humans pass the detection threshold. + + Raises: + NotImplementedError: if the CoreML backend is unavailable + (PyTorch single-shot path is TBD). + """ + backend = self._get_or_load_coreml_backend() + if backend is None: + raise NotImplementedError( + "CoreML backend unavailable; PyTorch single-shot path TBD") + + try: + import cv2 + except ImportError as e: + raise NotImplementedError( + "opencv-python required for predict_once: %s" % e) + + rgb = np.asarray(rgb_image) + if rgb.ndim != 3 or rgb.shape[2] != 3: + raise ValueError( + f"rgb_image must be (H,W,3), got {rgb.shape}") + h, w = rgb.shape[:2] + if (h, w) != (IMG_SIZE, IMG_SIZE): + side = min(h, w) + y0 = (h - side) // 2 + x0 = (w - side) // 2 + rgb = rgb[y0:y0 + side, x0:x0 + side] + rgb = cv2.resize(rgb, (IMG_SIZE, IMG_SIZE)) + + img = rgb.transpose(2, 0, 1).astype(np.float32) / 255.0 + focal = float(IMG_SIZE) + K_np = np.array([[focal, 0.0, IMG_SIZE / 2.0], + [0.0, focal, IMG_SIZE / 2.0], + [0.0, 0.0, 1.0]], dtype=np.float32) + + humans = backend.infer(img, K_np, det_thresh=self.det_thresh) + if not humans: + return None + + hh = humans[0] + v3d = hh["v3d"].detach().cpu().numpy() + return SMPLXPerson( + pid=0, + vertices_3d=np.ascontiguousarray(v3d, dtype=np.float32), + ) + def _run(self) -> None: if self.backend == "coreml": self._run_coreml() @@ -238,6 +342,10 @@ class MultiHMRWorker: prev_thumb = thumb frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + # Publish to state for DINOv2 reid in MeshRigger. + with self.state.lock(): + self.state.last_frame_rgb = frame_rgb + self.state.last_frame_rgb_t = time.monotonic() tensor = torch.from_numpy(frame_rgb).permute(2, 0, 1).float() tensor = (tensor / 255.0).unsqueeze(0).to(device) @@ -365,6 +473,10 @@ class MultiHMRWorker: v3d = hh["v3d"].detach().cpu().numpy() transl = hh.get("transl_pelvis", hh.get("transl")) transl_np = transl.detach().cpu().numpy().flatten() + if transl_np.size >= 3: + transl_np = transl_np.copy() + transl_np[2] = arkit_pelvis_z_override( + self.state, pid, float(transl_np[2])) shape_raw = hh["shape"].detach().cpu().numpy().flatten() expr_raw = hh["expression"].detach().cpu().numpy().flatten() @@ -517,6 +629,9 @@ class MultiHMRWorker: prev_thumb = thumb frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + with self.state.lock(): + self.state.last_frame_rgb = frame_rgb + self.state.last_frame_rgb_t = time.monotonic() img = frame_rgb.transpose(2, 0, 1).astype(np.float32) / 255.0 t_inf_start = time.monotonic() @@ -603,6 +718,10 @@ class MultiHMRWorker: continue v3d = hh["v3d"].detach().cpu().numpy() transl_np = hh["transl_pelvis"].detach().cpu().numpy().flatten() + if transl_np.size >= 3: + transl_np = transl_np.copy() + transl_np[2] = arkit_pelvis_z_override( + self.state, pid, float(transl_np[2])) shape_raw = hh["shape"].detach().cpu().numpy().flatten() expr_raw = hh["expression"].detach().cpu().numpy().flatten() diff --git a/data_only_viz/multihmr_coreml.py b/data_only_viz/multihmr_coreml.py index 3992800..a33d7f3 100644 --- a/data_only_viz/multihmr_coreml.py +++ b/data_only_viz/multihmr_coreml.py @@ -20,6 +20,7 @@ Public API: from __future__ import annotations import logging +import os from pathlib import Path from typing import Any @@ -38,12 +39,24 @@ DEFAULT_MLPACKAGE = ( N_PERSONS_FIXED = 4 N_VERTS = 10475 -# CoreML output names from the exported .mlpackage. -OUT_V3D = "var_2412" # (4, 10475, 3) -OUT_TRANSL = "var_2415" # (4, 1, 3) -OUT_SCORES = "var_2428" # (4,) -OUT_BETAS = "var_2431" # (4, 10) -OUT_EXPR = "var_2434" # (4, 10) +# CoreML output names from the exported .mlpackage. The exported +# `multihmr_full_672_s.mlpackage` (2026-05-14 re-convert) renumbered +# the MIL vars; verified against the on-disk artifact's spec. +OUT_V3D = "var_2420" # (4, 10475, 3) +OUT_TRANSL = "var_2423" # (4, 1, 3) +OUT_SCORES = "var_2436" # (4,) +OUT_BETAS = "var_2439" # (4, 10) +OUT_EXPR = "var_2442" # (4, 10) +# var_2445 (4, 127, 3) = j3d joints — present but unused here. + +# DINOv2 backbone was trained on ImageNet-normalized RGB; the public +# `infer()` contract takes [0,1] CHW input and applies this here so +# every caller stays normalization-agnostic. Feeding raw [0,1] to the +# model collapses all detection scores to ~0.01 ("0 detections" bug). +_IMG_NORM_MEAN = np.array([0.485, 0.456, 0.406], + dtype=np.float32).reshape(1, 3, 1, 1) +_IMG_NORM_STD = np.array([0.229, 0.224, 0.225], + dtype=np.float32).reshape(1, 3, 1, 1) # MLMultiArrayDataType raw values (from CoreML headers). ML_DTYPE_FLOAT32 = 65568 @@ -161,12 +174,22 @@ class MultiHMRCoreMLBackend: MLModel = ns["MLModel"] MLModelConfiguration = ns["MLModelConfiguration"] cfg = MLModelConfiguration.alloc().init() + # MLComputeUnits: 0=CPUOnly, 1=CPUAndGPU, 2=All (ANE+GPU+CPU), + # 3=CPUAndNeuralEngine. Bench M5 2026-05-14 (under live-worker + # contention, 30 iter median, full Multi-HMR predict+copy): + # CPU_AND_GPU = 252 ms (baseline) + # ALL = 246 ms (within noise, ANE doesn't help) + # CPU_AND_NE = 1301 ms (ANE solo catastrophic) + # CPU_ONLY = 1152 ms + # Standalone (no contention) FP32 = 139 ms = 7.2 fps. Default + # stays CPU+GPU. Override with COREML_COMPUTE_UNITS env var + # (`all`, `cpu_and_gpu`, `cpu_and_ne`, `cpu_only`) for A/B testing. + cu_env = os.environ.get("COREML_COMPUTE_UNITS", "").strip().lower() + cu_map = {"cpu_only": 0, "cpu_and_gpu": 1, "all": 2, + "cpu_and_ne": 3} + cu = cu_map.get(cu_env, 1) try: - # MLComputeUnits: 0=CPUOnly, 1=CPUAndGPU, 2=All (ANE+GPU+CPU), - # 3=CPUAndNeuralEngine. Multi-HMR's ANEF compile fails - # (validated 2026-05-13 on M5), and 'All' falls back to a - # slow path (~146ms). CPU+GPU = 28ms = ~35fps on M5. - cfg.setComputeUnits_(1) + cfg.setComputeUnits_(cu) except Exception: # noqa: BLE001 pass url = NSURL.fileURLWithPath_(str(self.path)) @@ -182,8 +205,10 @@ class MultiHMRCoreMLBackend: raise RuntimeError(f"MLModel load failed for {compiled_url}") self._model = model self._ns = ns - LOG.info("Multi-HMR CoreML model loaded (%s, computeUnits=CPU+GPU)", - self.path.name) + cu_name = {0: "CPU_ONLY", 1: "CPU+GPU", 2: "ALL", 3: "CPU+NE"}.get( + cu, str(cu)) + LOG.info("Multi-HMR CoreML model loaded (%s, computeUnits=%s)", + self.path.name, cu_name) @staticmethod def is_available(mlpackage_path: Path | None = None) -> bool: @@ -232,7 +257,8 @@ class MultiHMRCoreMLBackend: """Run a forward pass and return list of humans dicts. Args: - image_chw_float32: (3, 672, 672) or (1, 3, 672, 672) in [0,1]. + image_chw_float32: (3, 672, 672) or (1, 3, 672, 672), RGB in + [0,1]. ImageNet normalization is applied internally. K_33: (3, 3) or (1, 3, 3) camera intrinsics. det_thresh: scores threshold; CoreML forwards K=4 always. @@ -251,6 +277,7 @@ class MultiHMRCoreMLBackend: if K.shape != (1, 3, 3): raise ValueError(f"K shape {K.shape}, expected (1,3,3)") + img = (img - _IMG_NORM_MEAN) / _IMG_NORM_STD raw = self._predict(img, K) v3d = raw.get(OUT_V3D) transl = raw.get(OUT_TRANSL) diff --git a/data_only_viz/pose_filter.py b/data_only_viz/pose_filter.py new file mode 100644 index 0000000..6e508f5 --- /dev/null +++ b/data_only_viz/pose_filter.py @@ -0,0 +1,896 @@ +"""3D pose filtering chain : median spike removal, Kalman CV smoothing, +spring-damper organic inertia, lookahead extrapolation, IK angular clamps. + +Operates on lists of Kp3D (metric, hip-centered) keyed by track id. + +Stages are toggleable via the POSE_FILTER env var : + POSE_FILTER=median+kalman+lookahead+ik (default) + POSE_FILTER=median + POSE_FILTER=off +""" +from __future__ import annotations + +import logging +import math +import os +import time +from collections import deque +from dataclasses import dataclass, field +from typing import Iterable + +from .arkit_joint_map import ARKIT91_TO_MP33 +from .euro_filter import OneEuroFilter, SkeletonFilter +from .state import Kp3D, State + +LOG = logging.getLogger("pose_filter") + +NUM_JOINTS = 33 +DEFAULT_STAGES = ("median", "kalman", "lookahead", "ik") +ALL_STAGES = ( + "median", "kalman", "spring", "lookahead", "ik", + "one_euro_joints", "one_euro_bones", "arkit_fuse", +) + +# MediaPipe POSE_LANDMARKS indices used by IK constraints. +L_SHOULDER, R_SHOULDER = 11, 12 +L_ELBOW, R_ELBOW = 13, 14 +L_WRIST, R_WRIST = 15, 16 +L_HIP, R_HIP = 23, 24 +L_KNEE, R_KNEE = 25, 26 +L_ANKLE, R_ANKLE = 27, 28 +L_FOOT, R_FOOT = 31, 32 + +# (parent_idx, joint_idx, child_idx, min_deg, max_deg) +JOINT_LIMITS: tuple[tuple[int, int, int, float, float], ...] = ( + (L_SHOULDER, L_ELBOW, L_WRIST, 0.0, 175.0), + (R_SHOULDER, R_ELBOW, R_WRIST, 0.0, 175.0), + (L_HIP, L_KNEE, L_ANKLE, 0.0, 175.0), + (R_HIP, R_KNEE, R_ANKLE, 0.0, 175.0), + (L_KNEE, L_ANKLE, L_FOOT, 60.0, 135.0), + (R_KNEE, R_ANKLE, R_FOOT, 60.0, 135.0), +) + + +# ----------------------------- utilities -------------------------------- + +def _is_finite(v: float) -> bool: + return v == v and v not in (float("inf"), float("-inf")) + + +def _kp_finite(kp: Kp3D) -> bool: + return _is_finite(kp.x) and _is_finite(kp.y) and _is_finite(kp.z) + + +def _median(values: list[float]) -> float: + s = sorted(values) + n = len(s) + if n == 0: + return 0.0 + if n % 2 == 1: + return s[n // 2] + return 0.5 * (s[n // 2 - 1] + s[n // 2]) + + +def _std(values: list[float], mu: float) -> float: + if not values: + return 0.0 + var = sum((v - mu) ** 2 for v in values) / len(values) + return math.sqrt(var) + + +# ----------------------------- median filter ---------------------------- + +class MedianFilter: + """Per (pid, joint) ring buffer ; replaces spikes outside 3σ by median.""" + + def __init__(self, window: int = 3) -> None: + self.window = max(1, window) + self._buf: dict[tuple[int, int], deque[tuple[float, float, float]]] = {} + + def reset(self) -> None: + self._buf.clear() + + def apply(self, pid: int, joint_idx: int, x: float, y: float, z: float + ) -> tuple[float, float, float]: + key = (pid, joint_idx) + buf = self._buf.get(key) + if buf is None: + buf = deque(maxlen=self.window) + self._buf[key] = buf + + # Spike detection requires history. + out = (x, y, z) + if not (_is_finite(x) and _is_finite(y) and _is_finite(z)): + if buf: + med = (_median([v[0] for v in buf]), + _median([v[1] for v in buf]), + _median([v[2] for v in buf])) + out = med + else: + out = (0.0, 0.0, 0.0) + elif len(buf) >= self.window: + for axis_idx, val in enumerate(out): + col = [v[axis_idx] for v in buf] + med = _median(col) + sigma = _std(col, med) + if sigma > 1e-6 and abs(val - med) > 3.0 * sigma: + out = tuple(med if i == axis_idx else out[i] + for i in range(3)) # type: ignore[assignment] + buf.append(out) + return out + + +# ----------------------------- Kalman CV -------------------------------- + +@dataclass +class _KalmanState: + # State vector [x, y, z, vx, vy, vz] + x: list[float] = field(default_factory=lambda: [0.0] * 6) + # 6x6 covariance flattened + P: list[list[float]] = field(default_factory=lambda: [[0.0] * 6 for _ in range(6)]) + initialised: bool = False + last_t: float = 0.0 + + +def _mat_eye(n: int, s: float = 1.0) -> list[list[float]]: + return [[s if i == j else 0.0 for j in range(n)] for i in range(n)] + + +def _mat_mul(A: list[list[float]], B: list[list[float]]) -> list[list[float]]: + ra, ca = len(A), len(A[0]) + cb = len(B[0]) + out = [[0.0] * cb for _ in range(ra)] + for i in range(ra): + Ai = A[i] + for k in range(ca): + aik = Ai[k] + if aik == 0.0: + continue + Bk = B[k] + for j in range(cb): + out[i][j] += aik * Bk[j] + return out + + +def _mat_add(A: list[list[float]], B: list[list[float]]) -> list[list[float]]: + return [[A[i][j] + B[i][j] for j in range(len(A[0]))] for i in range(len(A))] + + +def _mat_sub(A: list[list[float]], B: list[list[float]]) -> list[list[float]]: + return [[A[i][j] - B[i][j] for j in range(len(A[0]))] for i in range(len(A))] + + +def _mat_T(A: list[list[float]]) -> list[list[float]]: + return [[A[i][j] for i in range(len(A))] for j in range(len(A[0]))] + + +def _mat_inv3(M: list[list[float]]) -> list[list[float]]: + a, b, c = M[0] + d, e, f = M[1] + g, h, i = M[2] + A = e * i - f * h + B = -(d * i - f * g) + C = d * h - e * g + det = a * A + b * B + c * C + if abs(det) < 1e-12: + return _mat_eye(3, 1.0) + inv_det = 1.0 / det + return [ + [A * inv_det, -(b * i - c * h) * inv_det, (b * f - c * e) * inv_det], + [B * inv_det, (a * i - c * g) * inv_det, -(a * f - c * d) * inv_det], + [C * inv_det, -(a * h - b * g) * inv_det, (a * e - b * d) * inv_det], + ] + + +class KalmanCV: + """Constant-velocity Kalman per (pid, joint_idx) on R^3.""" + + def __init__(self, q: float = 1e-3, r: float = 1e-2) -> None: + self.q = q + self.r = r + self._states: dict[tuple[int, int], _KalmanState] = {} + self._H = [ + [1.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0, 0.0, 0.0], + ] + + def reset(self) -> None: + self._states.clear() + + def get_velocity(self, pid: int, joint_idx: int) -> tuple[float, float, float]: + st = self._states.get((pid, joint_idx)) + if st is None or not st.initialised: + return (0.0, 0.0, 0.0) + return (st.x[3], st.x[4], st.x[5]) + + def step(self, pid: int, joint_idx: int, mx: float, my: float, mz: float, + t_now: float) -> tuple[float, float, float]: + key = (pid, joint_idx) + st = self._states.get(key) + if st is None: + st = _KalmanState() + self._states[key] = st + + if not st.initialised: + st.x = [mx, my, mz, 0.0, 0.0, 0.0] + st.P = _mat_eye(6, 1.0) + st.initialised = True + st.last_t = t_now + return (mx, my, mz) + + dt = max(1e-3, min(0.2, t_now - st.last_t)) + st.last_t = t_now + + # Predict + F = _mat_eye(6, 1.0) + F[0][3] = dt + F[1][4] = dt + F[2][5] = dt + x_pred = [ + st.x[0] + dt * st.x[3], + st.x[1] + dt * st.x[4], + st.x[2] + dt * st.x[5], + st.x[3], st.x[4], st.x[5], + ] + Q = _mat_eye(6, self.q) + P_pred = _mat_add(_mat_mul(_mat_mul(F, st.P), _mat_T(F)), Q) + + # Update + z = [mx, my, mz] + # y = z - H x_pred + Hx = [x_pred[0], x_pred[1], x_pred[2]] + y = [z[i] - Hx[i] for i in range(3)] + # S = H P H^T + R (3x3) + HP = _mat_mul(self._H, P_pred) + S = [[HP[i][j] for j in range(3)] for i in range(3)] + # add HP*H^T rest cols (cols 3..5) -> 0 contribution since H rest zero + for i in range(3): + S[i][i] += self.r + S_inv = _mat_inv3(S) + # K = P H^T S^-1 (6x3) + PHt = [[P_pred[i][j] for j in range(3)] for i in range(6)] + K = _mat_mul(PHt, S_inv) + # x = x_pred + K y + x_new = [x_pred[i] + sum(K[i][j] * y[j] for j in range(3)) + for i in range(6)] + # P = (I - K H) P_pred + KH = [[K[i][0] if j == 0 else (K[i][1] if j == 1 else (K[i][2] if j == 2 else 0.0)) + for j in range(6)] for i in range(6)] + I6 = _mat_eye(6, 1.0) + st.P = _mat_mul(_mat_sub(I6, KH), P_pred) + st.x = x_new + return (x_new[0], x_new[1], x_new[2]) + + +# --------------------------- spring damper ------------------------------ + +class SpringDamper: + """Critically-tunable spring-damper per (pid, joint_idx) on R^3.""" + + def __init__(self, stiffness: float = 200.0, damping: float = 15.0, + mass: float = 1.0, enabled: bool = True) -> None: + self.k = stiffness + self.c = damping + self.m = max(1e-3, mass) + self.enabled = enabled + self._pos: dict[tuple[int, int], list[float]] = {} + self._vel: dict[tuple[int, int], list[float]] = {} + self._last_t: dict[tuple[int, int], float] = {} + + def reset(self) -> None: + self._pos.clear() + self._vel.clear() + self._last_t.clear() + + def step(self, pid: int, joint_idx: int, tx: float, ty: float, tz: float, + t_now: float) -> tuple[float, float, float]: + if not self.enabled: + return (tx, ty, tz) + key = (pid, joint_idx) + pos = self._pos.get(key) + if pos is None: + self._pos[key] = [tx, ty, tz] + self._vel[key] = [0.0, 0.0, 0.0] + self._last_t[key] = t_now + return (tx, ty, tz) + dt = max(1e-3, min(0.1, t_now - self._last_t[key])) + self._last_t[key] = t_now + vel = self._vel[key] + target = (tx, ty, tz) + for i in range(3): + # F = k(target - pos) - c * vel + f = self.k * (target[i] - pos[i]) - self.c * vel[i] + a = f / self.m + vel[i] += a * dt + pos[i] += vel[i] * dt + return (pos[0], pos[1], pos[2]) + + +# --------------------------- lookahead ---------------------------------- + +class LookaheadPredictor: + """Linear extrapolation using Kalman velocities, capped to avoid blow-ups.""" + + def __init__(self, lookahead_ms: float = 50.0, max_velocity: float = 5.0 + ) -> None: + self.lookahead_s = lookahead_ms / 1000.0 + self.max_v = max_velocity + + def step(self, x: float, y: float, z: float, + vx: float, vy: float, vz: float) -> tuple[float, float, float]: + def clamp(v: float) -> float: + if v > self.max_v: + return self.max_v + if v < -self.max_v: + return -self.max_v + return v + dt = self.lookahead_s + return (x + clamp(vx) * dt, y + clamp(vy) * dt, z + clamp(vz) * dt) + + +# --------------------------- IK constraints ----------------------------- + +def _vec_sub(a: tuple[float, float, float], b: tuple[float, float, float] + ) -> tuple[float, float, float]: + return (a[0] - b[0], a[1] - b[1], a[2] - b[2]) + + +def _vec_add(a: tuple[float, float, float], b: tuple[float, float, float] + ) -> tuple[float, float, float]: + return (a[0] + b[0], a[1] + b[1], a[2] + b[2]) + + +def _vec_scale(a: tuple[float, float, float], s: float + ) -> tuple[float, float, float]: + return (a[0] * s, a[1] * s, a[2] * s) + + +def _vec_dot(a: tuple[float, float, float], b: tuple[float, float, float] + ) -> float: + return a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + + +def _vec_norm(a: tuple[float, float, float]) -> float: + return math.sqrt(_vec_dot(a, a)) + + +def _vec_normalize(a: tuple[float, float, float], eps: float = 1e-9 + ) -> tuple[float, float, float]: + n = _vec_norm(a) + if n < eps: + return (1.0, 0.0, 0.0) + return (a[0] / n, a[1] / n, a[2] / n) + + +def _slerp_dir(d_from: tuple[float, float, float], + d_to: tuple[float, float, float], + t: float) -> tuple[float, float, float]: + """Slerp between two unit-ish vectors.""" + a = _vec_normalize(d_from) + b = _vec_normalize(d_to) + cos_a = max(-1.0, min(1.0, _vec_dot(a, b))) + ang = math.acos(cos_a) + if ang < 1e-6: + return a + sa = math.sin(ang) + if abs(sa) < 1e-6: + # antiparallel : pick an arbitrary perpendicular, then rotate. + ortho = (1.0, 0.0, 0.0) if abs(a[0]) < 0.9 else (0.0, 1.0, 0.0) + # Gram-Schmidt + d = _vec_dot(ortho, a) + perp = (ortho[0] - d * a[0], ortho[1] - d * a[1], ortho[2] - d * a[2]) + perp = _vec_normalize(perp) + # rotate a by t*pi around perp axis : Rodrigues for angle = t*pi + theta = t * ang + cs, sn = math.cos(theta), math.sin(theta) + # cross(perp, a) + cx = perp[1] * a[2] - perp[2] * a[1] + cy = perp[2] * a[0] - perp[0] * a[2] + cz = perp[0] * a[1] - perp[1] * a[0] + dot_pa = _vec_dot(perp, a) + return (a[0] * cs + cx * sn + perp[0] * dot_pa * (1 - cs), + a[1] * cs + cy * sn + perp[1] * dot_pa * (1 - cs), + a[2] * cs + cz * sn + perp[2] * dot_pa * (1 - cs)) + w1 = math.sin((1.0 - t) * ang) / sa + w2 = math.sin(t * ang) / sa + return (a[0] * w1 + b[0] * w2, + a[1] * w1 + b[1] * w2, + a[2] * w1 + b[2] * w2) + + +class IKConstraints: + """Clamp interior joint angles for elbows, knees, ankles.""" + + def __init__(self, limits: Iterable[tuple[int, int, int, float, float]] + = JOINT_LIMITS) -> None: + self.limits = tuple(limits) + + def apply(self, kps: list[Kp3D]) -> list[Kp3D]: + if len(kps) < NUM_JOINTS: + return kps + out = list(kps) + for parent_i, joint_i, child_i, min_deg, max_deg in self.limits: + if max(parent_i, joint_i, child_i) >= len(out): + continue + p = (out[parent_i].x, out[parent_i].y, out[parent_i].z) + j = (out[joint_i].x, out[joint_i].y, out[joint_i].z) + c = (out[child_i].x, out[child_i].y, out[child_i].z) + v_pj = _vec_sub(p, j) # from joint to parent + v_cj = _vec_sub(c, j) # from joint to child + n_pj = _vec_norm(v_pj) + n_cj = _vec_norm(v_cj) + if n_pj < 1e-6 or n_cj < 1e-6: + continue + cos_a = max(-1.0, min(1.0, _vec_dot(v_pj, v_cj) / (n_pj * n_cj))) + ang_deg = math.degrees(math.acos(cos_a)) + min_r = math.radians(min_deg) + max_r = math.radians(max_deg) + target_r: float | None = None + if ang_deg < min_deg: + target_r = min_r + elif ang_deg > max_deg: + target_r = max_r + if target_r is None: + continue + # Interpolate child direction toward parent direction (or away) + # so the new angle matches target_r. + cur_r = math.acos(cos_a) + # t such that new_angle = (1-t)*cur + t*pi between dirs ; use slerp. + # Find t in [0,1] s.t. slerp(d_cj, d_pj, t) makes angle = target_r + # The angle between slerp result and d_pj is (1-t)*cur_r. + # So target_r = (1 - t) * cur_r -> t = 1 - target_r / cur_r + if cur_r < 1e-6: + continue + t = 1.0 - (target_r / cur_r) + t = max(0.0, min(1.0, t)) + d_cj = _vec_normalize(v_cj) + d_pj = _vec_normalize(v_pj) + new_dir = _slerp_dir(d_cj, d_pj, t) + new_child = _vec_add(j, _vec_scale(new_dir, n_cj)) + old = out[child_i] + out[child_i] = Kp3D(x=new_child[0], y=new_child[1], + z=new_child[2], c=old.c) + return out + + +# --------------------------- chain wrapper ------------------------------ + +def _parse_env_stages() -> tuple[str, ...]: + raw = os.environ.get("POSE_FILTER") + if raw is None: + return DEFAULT_STAGES + raw = raw.strip().lower() + if raw in ("off", "none", "0", "false"): + return () + parts = tuple(p.strip() for p in raw.replace(",", "+").split("+") if p.strip()) + return tuple(p for p in parts if p in ALL_STAGES) + + +class PoseFilterChain: + """Chain : median → kalman → spring → lookahead → ik.""" + + def __init__(self, state: State | None = None, + enabled_stages: Iterable[str] | None = None) -> None: + self.state = state + if enabled_stages is None: + stages = _parse_env_stages() + else: + stages = tuple(s for s in enabled_stages if s in ALL_STAGES) + self.enabled = stages + self.median = MedianFilter(window=3) + self.kalman = KalmanCV() + self.spring = SpringDamper(enabled="spring" in self.enabled) + self.lookahead = LookaheadPredictor() + self.ik = IKConstraints() + # One Euro filters (CHI 2012) — adaptive low-pass driven by speed. + # Joints variant: applied in joint-space, per (pid, joint_idx). + # Bones variant: applied to bone vectors (child - parent) along + # the MediaPipe Pose 33 subset that overlaps SMPL-X fused joints. + self.one_euro_joints = SkeletonFilter(min_cutoff=1.2, beta=0.08) + self.one_euro_bones = BoneOneEuroFilter(min_cutoff=1.0, beta=0.05) + self.arkit_fuse = ArkitFuse() + self.last_apply_ms: float = 0.0 + self.last_apply_bones_ms: float = 0.0 + LOG.info("PoseFilterChain stages=%s", self.enabled or ("off",)) + + def reset(self) -> None: + self.median.reset() + self.kalman.reset() + self.spring.reset() + self.one_euro_joints.reset_all() + self.one_euro_bones.reset_all() + + def apply(self, bodies3d: list[list[Kp3D]], ids: list[int], + t_now: float) -> list[list[Kp3D]]: + if not bodies3d or not self.enabled: + self.last_apply_ms = 0.0 + return bodies3d + t0 = time.perf_counter() + out: list[list[Kp3D]] = [] + use_median = "median" in self.enabled + use_kalman = "kalman" in self.enabled + use_spring = "spring" in self.enabled + use_lookahead = "lookahead" in self.enabled + use_ik = "ik" in self.enabled + use_one_euro_joints = "one_euro_joints" in self.enabled + use_arkit_fuse = "arkit_fuse" in self.enabled + + for body_i, kps in enumerate(bodies3d): + pid = ids[body_i] if body_i < len(ids) else -1 + if use_arkit_fuse and self.state is not None: + kps = self.arkit_fuse.apply(self.state, pid, kps, t_now) + new_kps: list[Kp3D] = [] + for j_idx, kp in enumerate(kps): + x, y, z, c = kp.x, kp.y, kp.z, kp.c + if use_median: + x, y, z = self.median.apply(pid, j_idx, x, y, z) + if use_one_euro_joints: + x, y, z = self.one_euro_joints.smooth( + pid, j_idx, x, y, z, t_now) + if use_kalman: + x, y, z = self.kalman.step(pid, j_idx, x, y, z, t_now) + if use_spring: + x, y, z = self.spring.step(pid, j_idx, x, y, z, t_now) + if use_lookahead and use_kalman: + vx, vy, vz = self.kalman.get_velocity(pid, j_idx) + x, y, z = self.lookahead.step(x, y, z, vx, vy, vz) + new_kps.append(Kp3D(x=x, y=y, z=z, c=c)) + if use_ik: + new_kps = self.ik.apply(new_kps) + out.append(new_kps) + + self.last_apply_ms = (time.perf_counter() - t0) * 1000.0 + return out + + # ---- Face / hand smoothing entry points --------------------------- + def apply_face(self, faces: list[list], ids: list[int], + t_now: float) -> list[list]: + if not hasattr(self, "_face_chain"): + self._face_chain = FaceFilterChain() + return self._face_chain.apply(faces, ids, t_now) + + def apply_hand(self, hands: list[list], ids: list[int], + handedness: list[str] | None, + t_now: float) -> list[list]: + if not hasattr(self, "_hand_chain"): + self._hand_chain = HandFilterChain() + return self._hand_chain.apply(hands, ids, handedness, t_now) + + # ---- Bone-space One Euro (Point B) -------------------------------- + def apply_bones(self, bodies3d: list[list[Kp3D]], ids: list[int], + t_now: float) -> list[list[Kp3D]]: + """Filter bone vectors (child - parent) for the body skeleton. + + Called *after* SMPL-X fusion in multi.py. No-op unless + ``one_euro_bones`` is in POSE_FILTER. Mutates the child slot + of each bone in-place — parents are walked in topological + order (root → leaves) so children always see updated parents. + """ + if not bodies3d or "one_euro_bones" not in self.enabled: + self.last_apply_bones_ms = 0.0 + return bodies3d + t0 = time.perf_counter() + for body_i, kps in enumerate(bodies3d): + pid = ids[body_i] if body_i < len(ids) else -1 + self.one_euro_bones.apply_body(pid, kps, t_now) + self.last_apply_bones_ms = (time.perf_counter() - t0) * 1000.0 + return bodies3d + + def forget_person(self, pid: int) -> None: + """Drop per-pid state on track loss (caller responsibility).""" + try: + self.one_euro_joints.forget(pid) + self.one_euro_bones.forget(pid) + except Exception: # noqa: BLE001 + pass + + +# ============================ bone One Euro =============================== + +# Body skeleton bones expressed as (parent_idx, child_idx) over the +# MediaPipe Pose 33 indexing — chosen to overlap with the 14 +# SMPL-X-fused slots (cf. multi.py SMPLX_TO_MP33). Topological order: +# legs first, then arms, then bridges (clavicle, pelvis), then torso. +BODY_BONES: tuple[tuple[int, int], ...] = ( + (L_HIP, L_KNEE), # 23 -> 25 + (L_KNEE, L_ANKLE), # 25 -> 27 + (L_ANKLE, L_FOOT), # 27 -> 31 + (R_HIP, R_KNEE), # 24 -> 26 + (R_KNEE, R_ANKLE), # 26 -> 28 + (R_ANKLE, R_FOOT), # 28 -> 32 + (L_SHOULDER, L_ELBOW), # 11 -> 13 + (L_ELBOW, L_WRIST), # 13 -> 15 + (R_SHOULDER, R_ELBOW), # 12 -> 14 + (R_ELBOW, R_WRIST), # 14 -> 16 + (L_SHOULDER, R_SHOULDER), # clavicle bridge + (L_HIP, R_HIP), # pelvis bridge + (L_SHOULDER, L_HIP), # left torso + (R_SHOULDER, R_HIP), # right torso +) + + +class BoneOneEuroFilter: + """One Euro filter applied to bone vectors of the body skeleton. + + For each bone (parent, child), the vector ``child - parent`` is + smoothed component-wise. Child position is then reconstructed as + ``parent + smoothed_bone``. This preserves bone *direction* + stability frame-to-frame while remaining responsive to genuine + pose changes (One Euro adaptive cutoff). + + State is keyed by ``(pid, bone_idx)`` and lives in three + OneEuroFilter instances per bone (one per axis). + """ + + def __init__(self, min_cutoff: float = 1.0, beta: float = 0.05) -> None: + self._min_cutoff = min_cutoff + self._beta = beta + # (pid, bone_idx) -> (fx, fy, fz) + self._table: dict[tuple[int, int], tuple[ + OneEuroFilter, OneEuroFilter, OneEuroFilter]] = {} + + def _filters_for(self, pid: int, bone_idx: int) -> tuple[ + OneEuroFilter, OneEuroFilter, OneEuroFilter]: + key = (pid, bone_idx) + f = self._table.get(key) + if f is None: + f = ( + OneEuroFilter(self._min_cutoff, self._beta), + OneEuroFilter(self._min_cutoff, self._beta), + OneEuroFilter(self._min_cutoff, self._beta), + ) + self._table[key] = f + return f + + def apply_body(self, pid: int, kps: list[Kp3D], t: float) -> None: + n = len(kps) + for bone_idx, (p_idx, c_idx) in enumerate(BODY_BONES): + if p_idx >= n or c_idx >= n: + continue + p = kps[p_idx] + c = kps[c_idx] + if not (_kp_finite(p) and _kp_finite(c)): + continue + dx = c.x - p.x + dy = c.y - p.y + dz = c.z - p.z + fx, fy, fz = self._filters_for(pid, bone_idx) + sx = fx(dx, t) + sy = fy(dy, t) + sz = fz(dz, t) + kps[c_idx] = Kp3D( + x=p.x + sx, y=p.y + sy, z=p.z + sz, c=c.c) + + def forget(self, pid: int) -> None: + self._table = {k: v for k, v in self._table.items() if k[0] != pid} + + def reset_all(self) -> None: + self._table.clear() + + +class ArkitFuse: + """Splice ARKit 91-joint world-space data into MediaPipe Pose 33. + + Reads ``state.persons_arkit_joints[pid]`` (shape (91, 3)) when fresh + (last_t within FRESH_SEC). Writes the 14 body slots covered by + ARKIT91_TO_MP33 ; everything else (face landmarks, finger tips) + stays MediaPipe-driven. + """ + + FRESH_SEC: float = 1.0 + + def apply(self, state: "State", pid: int, + kps: list[Kp3D], t_now: float) -> list[Kp3D]: + with state.lock(): + arr = state.persons_arkit_joints.get(pid) + last_t = state.persons_arkit_last_t.get(pid, 0.0) + if arr is None: + return kps + if t_now - last_t > self.FRESH_SEC: + return kps + out = list(kps) + n = len(out) + for arkit_idx, mp33_idx in ARKIT91_TO_MP33: + if mp33_idx >= n: + continue + x = float(arr[arkit_idx, 0]) + y = float(arr[arkit_idx, 1]) + z = float(arr[arkit_idx, 2]) + old = out[mp33_idx] + out[mp33_idx] = Kp3D(x=x, y=y, z=z, c=getattr(old, "c", 1.0)) + return out + + +# ============================ face / hand ================================= + +# Face and hand filtering operate on PoseKp lists (normalized x,y in [0,1] +# + z relative depth + confidence). We only apply temporal smoothing +# (median + Kalman 2D + lookahead) — no IK, no spring. + +def _parse_env_face_stages() -> tuple[str, ...]: + raw = os.environ.get("POSE_FILTER_FACE") + if raw is None: + return ("median", "kalman", "lookahead") + raw = raw.strip().lower() + if raw in ("off", "none", "0", "false"): + return () + parts = tuple(p.strip() for p in raw.replace(",", "+").split("+") if p.strip()) + return tuple(p for p in parts if p in ("median", "kalman", "lookahead")) + + +def _parse_env_hand_stages() -> tuple[str, ...]: + raw = os.environ.get("POSE_FILTER_HAND") + if raw is None: + return ("median", "kalman", "lookahead") + raw = raw.strip().lower() + if raw in ("off", "none", "0", "false"): + return () + parts = tuple(p.strip() for p in raw.replace(",", "+").split("+") if p.strip()) + return tuple(p for p in parts if p in ("median", "kalman", "lookahead")) + + +class AlphaBetaCV: + """Lightweight alpha-beta filter (scalar Kalman approximation). + + Far cheaper than the 6x6 KalmanCV : O(1) per joint per axis with no + matrix algebra. Suited to face/hand smoothing where the full CV + Kalman is overkill. + """ + + def __init__(self, alpha: float = 0.55, beta: float = 0.15) -> None: + self.alpha = alpha + self.beta = beta + # state[key] = [x, y, z, vx, vy, vz, last_t] + self._st: dict[tuple[int, int], list[float]] = {} + + def reset(self) -> None: + self._st.clear() + + def get_velocity(self, pid: int, joint_idx: int + ) -> tuple[float, float, float]: + s = self._st.get((pid, joint_idx)) + if s is None: + return (0.0, 0.0, 0.0) + return (s[3], s[4], s[5]) + + def step(self, pid: int, joint_idx: int, mx: float, my: float, + mz: float, t_now: float) -> tuple[float, float, float]: + key = (pid, joint_idx) + s = self._st.get(key) + if s is None: + self._st[key] = [mx, my, mz, 0.0, 0.0, 0.0, t_now] + return (mx, my, mz) + dt = max(1e-3, min(0.2, t_now - s[6])) + s[6] = t_now + # Predict + x_pred = s[0] + s[3] * dt + y_pred = s[1] + s[4] * dt + z_pred = s[2] + s[5] * dt + # Residual + rx = mx - x_pred + ry = my - y_pred + rz = mz - z_pred + # Update + s[0] = x_pred + self.alpha * rx + s[1] = y_pred + self.alpha * ry + s[2] = z_pred + self.alpha * rz + s[3] += (self.beta / dt) * rx + s[4] += (self.beta / dt) * ry + s[5] += (self.beta / dt) * rz + return (s[0], s[1], s[2]) + + +class FaceFilterChain: + """Per-pid temporal smoothing for face landmarks (median + Kalman + lookahead). + + Lookahead 30 ms ; max velocity in normalized units/s. + """ + + def __init__(self, lookahead_ms: float = 30.0, + enabled_stages: Iterable[str] | None = None) -> None: + if enabled_stages is None: + stages = _parse_env_face_stages() + else: + stages = tuple(s for s in enabled_stages + if s in ("median", "kalman", "lookahead")) + self.enabled = stages + self.median = MedianFilter(window=3) + self.kalman = AlphaBetaCV(alpha=0.55, beta=0.15) + self.lookahead = LookaheadPredictor( + lookahead_ms=lookahead_ms, max_velocity=2.0) + self.last_apply_ms: float = 0.0 + + def reset(self) -> None: + self.median.reset() + self.kalman.reset() + + def apply(self, faces: list[list], ids: list[int], + t_now: float) -> list[list]: + if not faces or not self.enabled: + self.last_apply_ms = 0.0 + return faces + t0 = time.perf_counter() + use_median = "median" in self.enabled + use_kalman = "kalman" in self.enabled + use_lookahead = "lookahead" in self.enabled + out: list[list] = [] + for f_i, kps in enumerate(faces): + pid = ids[f_i] if f_i < len(ids) else -1 + # Encode pid with a face-side namespace to avoid colliding with + # body and hand kalman/median caches. + key_pid = pid * 13 + 1 if pid >= 0 else pid + new_kps = [] + for j_idx, kp in enumerate(kps): + x, y, z, c = kp.x, kp.y, kp.z, kp.c + if use_median: + x, y, z = self.median.apply(key_pid, j_idx, x, y, z) + if use_kalman: + x, y, z = self.kalman.step(key_pid, j_idx, x, y, z, t_now) + if use_lookahead and use_kalman: + vx, vy, vz = self.kalman.get_velocity(key_pid, j_idx) + x, y, z = self.lookahead.step(x, y, z, vx, vy, vz) + new_kps.append(type(kp)(x=x, y=y, z=z, c=c)) + out.append(new_kps) + self.last_apply_ms = (time.perf_counter() - t0) * 1000.0 + return out + + +class HandFilterChain: + """Per-pid+side temporal smoothing for hand landmarks. + + Left and right hands keep independent filter state via a namespaced + pid (pid*2 for left, pid*2+1 for right). When handedness is not + provided, hands fall back to a side-agnostic namespace. + """ + + def __init__(self, lookahead_ms: float = 30.0, + enabled_stages: Iterable[str] | None = None) -> None: + if enabled_stages is None: + stages = _parse_env_hand_stages() + else: + stages = tuple(s for s in enabled_stages + if s in ("median", "kalman", "lookahead")) + self.enabled = stages + self.median = MedianFilter(window=3) + self.kalman = AlphaBetaCV(alpha=0.6, beta=0.2) + self.lookahead = LookaheadPredictor( + lookahead_ms=lookahead_ms, max_velocity=4.0) + self.last_apply_ms: float = 0.0 + + def reset(self) -> None: + self.median.reset() + self.kalman.reset() + + def apply(self, hands: list[list], ids: list[int], + handedness: list[str] | None, + t_now: float) -> list[list]: + if not hands or not self.enabled: + self.last_apply_ms = 0.0 + return hands + t0 = time.perf_counter() + use_median = "median" in self.enabled + use_kalman = "kalman" in self.enabled + use_lookahead = "lookahead" in self.enabled + out: list[list] = [] + for h_i, kps in enumerate(hands): + pid = ids[h_i] if h_i < len(ids) else -1 + side = (handedness[h_i] if handedness and h_i < len(handedness) + else "u").lower() + side_bit = 0 if side.startswith("l") else (1 if side.startswith("r") else 2) + # Namespace : (pid << 2) | side_bit — keeps L/R independent. + key_pid = (pid * 4 + side_bit + 7) if pid >= 0 else pid + new_kps = [] + for j_idx, kp in enumerate(kps): + x, y, z, c = kp.x, kp.y, kp.z, kp.c + if use_median: + x, y, z = self.median.apply(key_pid, j_idx, x, y, z) + if use_kalman: + x, y, z = self.kalman.step(key_pid, j_idx, x, y, z, t_now) + if use_lookahead and use_kalman: + vx, vy, vz = self.kalman.get_velocity(key_pid, j_idx) + x, y, z = self.lookahead.step(x, y, z, vx, vy, vz) + new_kps.append(type(kp)(x=x, y=y, z=z, c=c)) + out.append(new_kps) + self.last_apply_ms = (time.perf_counter() - t0) * 1000.0 + return out diff --git a/data_only_viz/pyproject.toml b/data_only_viz/pyproject.toml index 66574f5..b21cda2 100644 --- a/data_only_viz/pyproject.toml +++ b/data_only_viz/pyproject.toml @@ -38,6 +38,11 @@ detrpose = [ "iopath>=0.1.10", "opencv-python>=4.10", ] +# Open3D for ICP fusion between iPhone LiDAR and Multi-HMR SMPL-X meshes. +# CPU-only is sufficient at 5-10 Hz LiDAR cadence. +lidar = [ + "open3d>=0.18,<0.20", +] nlf = [ "torch>=2.4", "torchvision>=0.19", diff --git a/data_only_viz/scripts/bench_icp_fusion.py b/data_only_viz/scripts/bench_icp_fusion.py new file mode 100644 index 0000000..b89c3cd --- /dev/null +++ b/data_only_viz/scripts/bench_icp_fusion.py @@ -0,0 +1,77 @@ +"""Latency / convergence bench for the ICP fusion worker. + +Usage: + + cd data_only_viz + uv run --extra lidar python -m data_only_viz.scripts.bench_icp_fusion \ + --n-frames 200 --n-people 2 --seed 0 +""" +from __future__ import annotations + +import argparse +import json +import time + +import numpy as np + +from data_only_viz.icp_fusion import FusionWorker, IcpConfig +from data_only_viz.lidar_calib import Extrinsic +from data_only_viz.state import SMPLXPerson, State + + +def _synth_person(seed: int, offset_x: float) -> SMPLXPerson: + rng = np.random.RandomState(seed) + verts = np.zeros((10475, 3), dtype=np.float32) + pts = rng.randn(2000, 3).astype(np.float32) * 0.1 + verts[: pts.shape[0]] = pts + np.array([offset_x, 0, 1.5], dtype=np.float32) + verts[5559] = pts.mean(axis=0) + np.array([offset_x, 0, 1.5], dtype=np.float32) + return SMPLXPerson(pid=seed, vertices_3d=verts) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser() + p.add_argument("--n-frames", type=int, default=200) + p.add_argument("--n-people", type=int, default=2) + p.add_argument("--seed", type=int, default=0) + args = p.parse_args(argv) + + rng = np.random.RandomState(args.seed) + persons = [_synth_person(i, offset_x=-0.6 + 1.2 * i) for i in range(args.n_people)] + state = State() + state.persons_smplx = persons + + worker = FusionWorker(extrinsic=Extrinsic.identity(), config=IcpConfig()) + + latencies_ms: list[float] = [] + accepted = 0 + pelvis_delta_m: list[float] = [] + for _ in range(args.n_frames): + all_pts = np.concatenate([ + pers.vertices_3d[: 2000] + np.array([0, 0.05, 0], dtype=np.float32) + + 0.02 * rng.randn(2000, 3).astype(np.float32) + for pers in persons + ]) + state.lidar_points = all_pts + before = np.stack([p.vertices_3d[5559].copy() for p in state.persons_smplx]) + t0 = time.perf_counter() + meta = worker.run_once(state) + latencies_ms.append((time.perf_counter() - t0) * 1000.0) + accepted += len(meta.applied) + after = np.stack([p.vertices_3d[5559] for p in state.persons_smplx]) + pelvis_delta_m.extend(np.linalg.norm(after - before, axis=1).tolist()) + + report = { + "n_frames": args.n_frames, + "n_people": args.n_people, + "latency_ms_p50": float(np.percentile(latencies_ms, 50)), + "latency_ms_p95": float(np.percentile(latencies_ms, 95)), + "acceptance_rate": accepted / (args.n_frames * args.n_people), + "pelvis_delta_m_mean": float(np.mean(pelvis_delta_m)), + "pelvis_delta_m_max": float(np.max(pelvis_delta_m)), + } + print(json.dumps(report, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_only_viz/scripts/bench_multihmr_coreml.py b/data_only_viz/scripts/bench_multihmr_coreml.py new file mode 100644 index 0000000..fe33462 --- /dev/null +++ b/data_only_viz/scripts/bench_multihmr_coreml.py @@ -0,0 +1,235 @@ +"""Bench Multi-HMR CoreML — compute_units sweep + section split. + +Bench Multi-HMR `.mlpackage` inference latency on M5 (or any Apple +Silicon). Decomposes the per-frame cost into copy_in / predict / +copy_out so we can see where time goes, then sweeps compute_units +(CPU_AND_GPU vs ALL vs CPU_AND_NE vs CPU_ONLY) and tests the +"reused MLMultiArray buffer" optimization. + +Usage: + uv run --project data_only_viz \ + python -m data_only_viz.scripts.bench_multihmr_coreml + +The result reproduces the 2026-05-14 finding: predict() is ~99% of +latency, copy_in is <2 ms, copy_out is <1 ms. None of the I/O +micro-optims (reused buffer, vImage preprocess, async copy) can +help meaningfully — only changing the model itself does (INT8 quant +via `scripts/quantize_multihmr_int8.py`, lower resolution, or a +smaller architecture). + +Pause the live worker before running for clean numbers: + pgrep -f 'data_only_viz.main.*multi-hmr' | xargs kill -STOP + # ...run bench... + pgrep -f 'data_only_viz.main.*multi-hmr' | xargs kill -CONT +""" +from __future__ import annotations + +import ctypes +import sys +import time +from pathlib import Path + +import numpy as np +from Foundation import NSURL + +from data_only_viz.multihmr_coreml import ( + DEFAULT_MLPACKAGE, + _load_frameworks, + _mlarray_to_np, + _np_to_mlarray, +) + +H = W = 672 +NITER = 30 +NWARM = 5 + + +def _make_inputs(): + img = np.random.rand(1, 3, H, W).astype(np.float32) + focal = float(H) + K = np.array( + [[[focal, 0, H / 2], [0, focal, H / 2], [0, 0, 1.0]]], + dtype=np.float32, + ) + return img, K + + +def _load_model(compute_units: int, mlpackage: Path): + ns = _load_frameworks() + MLModel = ns["MLModel"] + MLModelConfiguration = ns["MLModelConfiguration"] + cfg = MLModelConfiguration.alloc().init() + cfg.setComputeUnits_(compute_units) + url = NSURL.fileURLWithPath_(str(mlpackage)) + compiled = MLModel.compileModelAtURL_error_(url, None) + if compiled is None: + raise RuntimeError(f"compile failed cu={compute_units}") + model = MLModel.modelWithContentsOfURL_configuration_error_( + compiled, cfg, None) + if model is None: + raise RuntimeError(f"load failed cu={compute_units}") + return model, ns + + +def _stats(ts): + ts = sorted(ts) + return (ts[len(ts) // 2], + ts[len(ts) // 10], + ts[(len(ts) * 9) // 10]) + + +def bench_basic(label: str, compute_units: int, mlpackage: Path): + try: + model, ns = _load_model(compute_units, mlpackage) + except Exception as e: # noqa: BLE001 + print(f"[{label}] LOAD FAILED: {e}") + return None + MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"] + MLFeatureValue = ns["MLFeatureValue"] + img, K = _make_inputs() + for _ in range(NWARM): + img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K) + feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml), + "cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)} + prov = MLDictionaryFeatureProvider.alloc( + ).initWithDictionary_error_(feats, None) + out = model.predictionFromFeatures_error_(prov, None) + if out is None: + print(f"[{label}] predict returned None") + return None + ts = [] + for _ in range(NITER): + t0 = time.perf_counter() + img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K) + feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml), + "cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)} + prov = MLDictionaryFeatureProvider.alloc( + ).initWithDictionary_error_(feats, None) + out = model.predictionFromFeatures_error_(prov, None) + for name in out.featureNames(): + fv = out.featureValueForName_(name) + ml = fv.multiArrayValue() + if ml is None: + continue + _ = _mlarray_to_np(ml) + ts.append((time.perf_counter() - t0) * 1e3) + med, p10, p90 = _stats(ts) + print(f"[{label:34s}] med={med:6.1f}ms p10={p10:6.1f} " + f"p90={p90:6.1f} fps={1000/med:5.1f}") + return med + + +def bench_reused_input(label: str, compute_units: int, mlpackage: Path): + try: + model, ns = _load_model(compute_units, mlpackage) + except Exception as e: # noqa: BLE001 + print(f"[{label}] LOAD FAILED: {e}") + return None + MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"] + MLFeatureValue = ns["MLFeatureValue"] + img, K = _make_inputs() + img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K) + ptr_img = img_ml.dataPointer() + addr_img = int(ptr_img) if isinstance(ptr_img, int) else \ + ctypes.cast(ptr_img, ctypes.c_void_p).value + ptr_k = k_ml.dataPointer() + addr_k = int(ptr_k) if isinstance(ptr_k, int) else \ + ctypes.cast(ptr_k, ctypes.c_void_p).value + img_bytes = img.nbytes + k_bytes = K.nbytes + feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml), + "cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)} + for _ in range(NWARM): + ctypes.memmove(addr_img, img.ctypes.data, img_bytes) + ctypes.memmove(addr_k, K.ctypes.data, k_bytes) + prov = MLDictionaryFeatureProvider.alloc( + ).initWithDictionary_error_(feats, None) + _ = model.predictionFromFeatures_error_(prov, None) + ts = [] + for _ in range(NITER): + t0 = time.perf_counter() + ctypes.memmove(addr_img, img.ctypes.data, img_bytes) + ctypes.memmove(addr_k, K.ctypes.data, k_bytes) + prov = MLDictionaryFeatureProvider.alloc( + ).initWithDictionary_error_(feats, None) + out = model.predictionFromFeatures_error_(prov, None) + for name in out.featureNames(): + fv = out.featureValueForName_(name) + ml = fv.multiArrayValue() + if ml is None: + continue + _ = _mlarray_to_np(ml) + ts.append((time.perf_counter() - t0) * 1e3) + med, p10, p90 = _stats(ts) + print(f"[{label:34s}] med={med:6.1f}ms p10={p10:6.1f} " + f"p90={p90:6.1f} fps={1000/med:5.1f}") + return med + + +def bench_section_split(compute_units: int, mlpackage: Path): + model, ns = _load_model(compute_units, mlpackage) + MLDictionaryFeatureProvider = ns["MLDictionaryFeatureProvider"] + MLFeatureValue = ns["MLFeatureValue"] + img, K = _make_inputs() + for _ in range(NWARM): + img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K) + feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml), + "cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)} + prov = MLDictionaryFeatureProvider.alloc( + ).initWithDictionary_error_(feats, None) + _ = model.predictionFromFeatures_error_(prov, None) + t_in, t_pred, t_out = [], [], [] + for _ in range(NITER): + t0 = time.perf_counter() + img_ml = _np_to_mlarray(img); k_ml = _np_to_mlarray(K) + feats = {"image": MLFeatureValue.featureValueWithMultiArray_(img_ml), + "cam_K": MLFeatureValue.featureValueWithMultiArray_(k_ml)} + prov = MLDictionaryFeatureProvider.alloc( + ).initWithDictionary_error_(feats, None) + t1 = time.perf_counter() + out = model.predictionFromFeatures_error_(prov, None) + t2 = time.perf_counter() + for name in out.featureNames(): + fv = out.featureValueForName_(name) + ml = fv.multiArrayValue() + if ml is None: + continue + _ = _mlarray_to_np(ml) + t3 = time.perf_counter() + t_in.append((t1 - t0) * 1e3) + t_pred.append((t2 - t1) * 1e3) + t_out.append((t3 - t2) * 1e3) + mi = lambda a: sorted(a)[len(a) // 2] + print("[section-split CPU_AND_GPU]") + print(f" copy_in : {mi(t_in):6.2f} ms") + print(f" predict : {mi(t_pred):6.2f} ms") + print(f" copy_out : {mi(t_out):6.2f} ms") + print(f" total : {mi(t_in)+mi(t_pred)+mi(t_out):6.2f} ms") + + +def main(argv: list[str]) -> int: + mlpackage = DEFAULT_MLPACKAGE + if len(argv) > 1: + mlpackage = Path(argv[1]) + if not mlpackage.exists(): + print(f"mlpackage missing: {mlpackage}", file=sys.stderr) + return 1 + print(f"bench target: {mlpackage}") + print("=" * 70) + print("Section split (alloc/predict/copy)") + print("=" * 70) + bench_section_split(1, mlpackage) + print() + print("=" * 70) + print("Compute-units sweep (30 iter median)") + print("=" * 70) + bench_basic("A. CPU_AND_GPU (baseline)", 1, mlpackage) + bench_basic("B. ALL (ANE+GPU+CPU)", 2, mlpackage) + bench_basic("C. CPU_AND_NE (ANE-only)", 3, mlpackage) + bench_basic("D. CPU_ONLY", 0, mlpackage) + bench_reused_input("E. CPU_AND_GPU + reused buffer", 1, mlpackage) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/data_only_viz/scripts/calibrate_lidar.py b/data_only_viz/scripts/calibrate_lidar.py new file mode 100644 index 0000000..9e82724 --- /dev/null +++ b/data_only_viz/scripts/calibrate_lidar.py @@ -0,0 +1,97 @@ +"""Interactive one-shot extrinsic calibration between iPhone LiDAR and webcam. + +Usage: + + cd data_only_viz + uv run --extra lidar python -m data_only_viz.scripts.calibrate_lidar \ + --lidar-host 192.168.0.42 --lidar-port 5500 --webcam-index 0 + +The script prompts the user to assume 4 stances (front, left, right, back), +captures paired pelvis points (webcam: Multi-HMR vertex 5559; LiDAR: centroid +of the largest mesh anchor), solves Kabsch, and writes the result to +ICP_LIDAR_EXTRINSIC or the default path. + +Multi-HMR worker is launched in-process for this script (single-shot mode). +""" +from __future__ import annotations + +import argparse +import datetime as dt +import logging +import sys +import time + +import numpy as np + +from data_only_viz.lidar_calib import Extrinsic, kabsch_rigid, save_extrinsic +from data_only_viz.lidar_receiver import LidarTCPReader + +_LOG = logging.getLogger("calibrate_lidar") +_PELVIS_VERT_INDEX = 5559 # SMPL-X canonical pelvis vertex + + +def _wait_for_lidar(reader: LidarTCPReader, timeout_s: float = 5.0): + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + latest = reader.latest() + if latest is not None and latest.points.shape[0] > 50: + return latest + time.sleep(0.05) + raise RuntimeError("LiDAR frame never arrived") + + +def _capture_one_pair(reader: LidarTCPReader, get_smplx_pelvis_cam) -> tuple[np.ndarray, np.ndarray]: + input("Hold still, then press ENTER to capture...") + lidar = _wait_for_lidar(reader) + pelvis_cam = get_smplx_pelvis_cam() + pelvis_arkit = lidar.points.mean(axis=0) + _LOG.info("captured: cam=%s arkit=%s", pelvis_cam, pelvis_arkit) + return pelvis_cam, pelvis_arkit + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser() + p.add_argument("--lidar-host", required=True) + p.add_argument("--lidar-port", type=int, default=5500) + p.add_argument("--webcam-index", type=int, default=0) + p.add_argument("--stances", type=int, default=4) + args = p.parse_args(argv) + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s") + + reader = LidarTCPReader(host=args.lidar_host, port=args.lidar_port) + reader.start() + + # Task 9 added the ``MultiHMRWorker.predict_once`` API surface but + # left the body as ``NotImplementedError`` — the existing PyTorch + # path is too coupled to the worker thread for a clean extraction. + # When ``predict_once`` is wired (follow-up task), replace this + # placeholder by opening cv2.VideoCapture(args.webcam_index), + # running ``worker.predict_once(rgb)`` and returning + # ``person.vertices_3d[_PELVIS_VERT_INDEX]``. + def _placeholder_pelvis_cam() -> np.ndarray: + raise SystemExit( + "calibrate_lidar needs MultiHMRWorker.predict_once to be " + "implemented (currently NotImplementedError)") + + pairs_cam, pairs_arkit = [], [] + try: + for i in range(args.stances): + _LOG.info("stance %d/%d", i + 1, args.stances) + cam, arkit = _capture_one_pair(reader, _placeholder_pelvis_cam) + pairs_cam.append(cam) + pairs_arkit.append(arkit) + finally: + reader.stop() + + T = kabsch_rigid(np.asarray(pairs_arkit), np.asarray(pairs_cam)) + path = save_extrinsic(Extrinsic( + T_arkit_to_cam=T, + confidence=1.0, + captured_at_iso=dt.datetime.now(dt.timezone.utc).isoformat(), + )) + _LOG.info("extrinsic saved to %s", path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/data_only_viz/scripts/convert_dinov2.py b/data_only_viz/scripts/convert_dinov2.py new file mode 100644 index 0000000..941ac64 --- /dev/null +++ b/data_only_viz/scripts/convert_dinov2.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Convert DINOv2 ViT-S/14 to a CoreML .mlpackage for ANE-friendly inference. + +The wrapped module takes (1, 3, 224, 224) RGB float32 in [0, 1], applies +ImageNet normalization internally, runs the ViT, and returns the CLS +embedding (1, 384) L2-normalised. We trace + convert with +``coremltools.convert(... compute_units=ComputeUnit.ALL, compute_precision=FP16)``. + +Run with the Python 3.12 venv that has coremltools and torch:: + + /tmp/coreml312/bin/python -m data_only_viz.scripts.convert_dinov2 [--force] + +Output: + ~/.cache/av-live-multihmr/dinov2_vits14.mlpackage +""" +from __future__ import annotations + +import argparse +import logging +import sys +import time +import types +from pathlib import Path + +import numpy as np + +LOG = logging.getLogger("convert_dinov2") + +OUT_DIR = Path.home() / ".cache" / "av-live-multihmr" +OUT_PATH = OUT_DIR / "dinov2_vits14.mlpackage" + +_IMAGENET_MEAN = (0.485, 0.456, 0.406) +_IMAGENET_STD = (0.229, 0.224, 0.225) + + +def _build_wrapper(): + import torch + import torch.nn as nn + import torch.nn.functional as F + + backbone = torch.hub.load( + "facebookresearch/dinov2", + "dinov2_vits14", + source="github", + trust_repo=True, + ) + backbone.eval() + + # Pretrained pos_embed is at 37x37 (518/14). We pre-resample to + # 16x16 (224/14) once so the traced graph never needs an upsample. + pe = backbone.pos_embed.data # (1, 1+37*37, 384) + cls_pe = pe[:, :1] + patch_pe = pe[:, 1:] + n_old = int(round((patch_pe.shape[1]) ** 0.5)) + dim = patch_pe.shape[-1] + patch_pe = patch_pe.reshape(1, n_old, n_old, dim).permute(0, 3, 1, 2) + patch_pe = F.interpolate(patch_pe, size=(16, 16), mode="bilinear", + align_corners=False) + patch_pe = patch_pe.permute(0, 2, 3, 1).reshape(1, 16 * 16, dim) + new_pe = torch.cat([cls_pe, patch_pe], dim=1).contiguous() + backbone.pos_embed = nn.Parameter(new_pe, requires_grad=False) + + mean = torch.tensor(_IMAGENET_MEAN, dtype=torch.float32).view(1, 3, 1, 1) + std = torch.tensor(_IMAGENET_STD, dtype=torch.float32).view(1, 3, 1, 1) + + class DinoV2Wrapper(nn.Module): + def __init__(self): + super().__init__() + self.backbone = backbone + self.register_buffer("mean", mean) + self.register_buffer("std", std) + + def forward(self, x): + x = (x - self.mean) / self.std + bb = self.backbone + x = bb.patch_embed(x) + # cls_token is (1,1,384). Concat directly (B=1 fixed). + x = torch.cat((bb.cls_token, x), dim=1) + x = x + bb.pos_embed + for blk in bb.blocks: + x = blk(x) + x = bb.norm(x) + cls = x[:, 0] + cls = cls / (cls.norm(dim=-1, keepdim=True) + 1e-8) + return cls + + return DinoV2Wrapper().eval() + + +def _patch_coremltools_cast(): + """coremltools 9.0 _cast assumes x.val is a 0-d scalar. With recent + torch (2.12) some aten::Int args land as 1-D length-1 arrays. Patch + the helper to flatten before scalar-casting.""" + from coremltools.converters.mil.frontend.torch import ops as _ops + from coremltools.converters.mil.mil import Builder as mb + + _orig = _ops._cast + + def _patched_cast(context, node, dtype, dtype_name): + # Inputs are read inside _orig from context; we wrap the failure + # path by checking the first input's val first. + inputs = _ops._get_inputs(context, node, expected=1) + x = inputs[0] + if x.can_be_folded_to_const(): + val = x.val + if hasattr(val, "shape") and getattr(val, "shape", ()) != (): + # 1-D length-1 (or all-ones shape) -> extract scalar + import numpy as _np + arr = _np.asarray(val).reshape(-1) + if arr.size == 1: + res = mb.const(val=dtype(arr[0]), name=node.name) + context.add(res, node.name) + return + return _orig(context, node, dtype, dtype_name) + + _ops._cast = _patched_cast + + +def convert(force: bool = False) -> Path: + import torch + import coremltools as ct + _patch_coremltools_cast() + + OUT_DIR.mkdir(parents=True, exist_ok=True) + if OUT_PATH.exists() and not force: + LOG.info("already converted: %s", OUT_PATH) + return OUT_PATH + + LOG.info("loading DINOv2 ViT-S/14 ...") + wrap = _build_wrapper() + example = torch.rand(1, 3, 224, 224, dtype=torch.float32) + with torch.no_grad(): + ref_out = wrap(example) + LOG.info("torch out shape=%s norm=%.4f", tuple(ref_out.shape), + float(ref_out.norm(dim=-1).mean())) + + LOG.info("tracing ...") + with torch.no_grad(): + traced = torch.jit.trace(wrap, example, strict=False) + + LOG.info("ct.convert (mlprogram FP16, computeUnits=ALL) ...") + mlmodel = ct.convert( + traced, + source="pytorch", + convert_to="mlprogram", + inputs=[ct.TensorType(name="image", shape=example.shape, + dtype=np.float32)], + outputs=[ct.TensorType(name="embedding", dtype=np.float32)], + compute_precision=ct.precision.FLOAT16, + compute_units=ct.ComputeUnit.ALL, + minimum_deployment_target=ct.target.macOS14, + ) + mlmodel.short_description = "DINOv2 ViT-S/14 person re-id (384-D, L2)" + mlmodel.save(str(OUT_PATH)) + LOG.info("saved %s", OUT_PATH) + + pred = mlmodel.predict({"image": example.numpy().astype(np.float32)}) + coreml_out = list(pred.values())[0].reshape(-1) + ref_np = ref_out.numpy().reshape(-1) + cos = float(np.dot(coreml_out, ref_np) / + (np.linalg.norm(coreml_out) * np.linalg.norm(ref_np) + 1e-8)) + LOG.info("CoreML vs Torch cosine on random input: %.4f", cos) + return OUT_PATH + + +def bench(n_iter: int = 30) -> None: + import coremltools as ct + LOG.info("bench: load mlpackage ...") + m = ct.models.MLModel(str(OUT_PATH), + compute_units=ct.ComputeUnit.ALL) + crop = np.random.rand(1, 3, 224, 224).astype(np.float32) + for _ in range(3): + m.predict({"image": crop}) + times = [] + for _ in range(n_iter): + t0 = time.perf_counter() + m.predict({"image": crop}) + times.append((time.perf_counter() - t0) * 1e3) + times.sort() + p50 = times[len(times) // 2] + p95 = times[int(len(times) * 0.95)] + LOG.info("bench %d iter: p50=%.2f ms p95=%.2f ms mean=%.2f ms (~%.1f fps)", + n_iter, p50, p95, sum(times) / len(times), 1000.0 / p50) + + +def main() -> int: + logging.basicConfig(level=logging.INFO, + format="%(asctime)s %(name)s %(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--force", action="store_true") + ap.add_argument("--bench-only", action="store_true") + ap.add_argument("--n-iter", type=int, default=30) + args = ap.parse_args() + + if not args.bench_only: + convert(force=args.force) + bench(n_iter=args.n_iter) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/data_only_viz/scripts/coreml_full_probe.py b/data_only_viz/scripts/coreml_full_probe.py index f1f4387..f752ca1 100644 --- a/data_only_viz/scripts/coreml_full_probe.py +++ b/data_only_viz/scripts/coreml_full_probe.py @@ -524,10 +524,10 @@ try: compute_units=ct.ComputeUnit.CPU_AND_GPU, minimum_deployment_target=ct.target.macOS15, convert_to="mlprogram", - # FP16 OK depuis le patch roma branchless (cf rapport bisection - # 2026-05-13) : la source du NaN etait torch.empty + index_put_ - # dans roma.rotmat_to_rotvec, pas la precision. - compute_precision=ct.precision.FLOAT16, + # FP32 mandatory : FP16 (global ou hybride op_selector) degrade + # visiblement le mesh sur poses extremes. INT8 weight quant + # teste 2026-05-14 : aucun gain sur GPU compute-bound. + compute_precision=ct.precision.FLOAT32, ) out_path = "/tmp/multihmr_full_672_s.mlpackage" mlmodel.save(out_path) diff --git a/data_only_viz/scripts/quantize_multihmr_int8.py b/data_only_viz/scripts/quantize_multihmr_int8.py new file mode 100644 index 0000000..45902f1 --- /dev/null +++ b/data_only_viz/scripts/quantize_multihmr_int8.py @@ -0,0 +1,81 @@ +"""Quantize Multi-HMR mlpackage to INT8 (weight-only) for M5 speedup. + +Run in the Python 3.12 conversion venv (coremltools cannot run on 3.14): + + /tmp/coreml312/.venv/bin/python \ + data_only_viz/scripts/quantize_multihmr_int8.py + +Produces `multihmr_full_672_s_int8.mlpackage` next to the FP32 file. +Bench after with `scripts/coreml_full_probe.py` or just load with +`MultiHMRCoreMLBackend(path=...new path...)`. + +Strategy: +- Linear 8-bit weight palettization (per-tensor symmetric). Activations + stay FP16 — that's the "weight-only quant" path, lowest accuracy + hit and what CoreML's GPU runtime accelerates best. +- Skip the SMPL-X decoder branch ops that are sensitive to numeric + drift (skipped by name pattern below — adjust if v3d shows mesh + artefacts after quantization). + +Validation: +- After producing the int8 mlpackage, run the live worker briefly + with COREML_MLPACKAGE pointing to the new file and visually check + the mesh. If v3d shows tearing on extreme poses, retry with + `granularity="per_channel"` instead of `per_tensor`. +""" +from __future__ import annotations + +import sys +from pathlib import Path + +try: + import coremltools as ct + from coremltools.optimize.coreml import ( + linear_quantize_weights, + OptimizationConfig, + OpLinearQuantizerConfig, + ) +except ImportError as e: + print(f"coremltools missing in this venv: {e}", file=sys.stderr) + print("Run from the Python 3.12 conversion venv (coremltools " + "is not available on 3.14).", file=sys.stderr) + sys.exit(1) + + +SRC = Path.home() / ".cache" / "av-live-multihmr" / \ + "multihmr_full_672_s.mlpackage" +DST = Path.home() / ".cache" / "av-live-multihmr" / \ + "multihmr_full_672_s_int8.mlpackage" + + +def main() -> int: + if not SRC.exists(): + print(f"source mlpackage missing: {SRC}", file=sys.stderr) + return 1 + print(f"loading FP32 model from {SRC}") + model = ct.models.MLModel(str(SRC)) + + # Per-tensor symmetric int8 weight quant. Per-tensor keeps the + # quantized model small and GPU-friendly; per-channel is a safer + # fallback if mesh quality degrades. + op_cfg = OpLinearQuantizerConfig( + mode="linear_symmetric", + dtype="int8", + granularity="per_tensor", + ) + cfg = OptimizationConfig(global_config=op_cfg) + print("running linear_quantize_weights (per_tensor int8)...") + quant = linear_quantize_weights(model, config=cfg) + print(f"saving quantized model to {DST}") + quant.save(str(DST)) + print("done. Test with:") + print(f" COREML_MLPACKAGE={DST} \\\n" + f" MULTIHMR_BACKEND=coreml \\\n" + f" uv run --project data_only_viz \\\n" + f" python -m data_only_viz.main --multi-hmr " + f"--motion-gate 0") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_only_viz/smplx_osc_sender.py b/data_only_viz/smplx_osc_sender.py index 18dbdd8..5b5f0a9 100644 --- a/data_only_viz/smplx_osc_sender.py +++ b/data_only_viz/smplx_osc_sender.py @@ -25,9 +25,16 @@ from typing import Sequence import numpy as np +import os + from .mesh_rigger import MeshRigger from .state import SMPLXPerson, State +try: + from .dino_reid import DinoReid +except Exception: # noqa: BLE001 + DinoReid = None # type: ignore[assignment] + LOG = logging.getLogger("smplx_tcp") MAGIC = b"SMPX" @@ -47,7 +54,25 @@ class SMPLXTCPSender: self._sock: socket.socket | None = None # Hybrid keyframe rigging : entre deux keyframes Multi-HMR (~3 fps), # on translate le mesh via le delta pelvis Apple Vision (30 fps). - self._rigger = MeshRigger(state) if enable_rigging else None + # MULTIHMR_REID: 'dino' (try DINOv2 + IoU fusion, fallback IoU) / + # 'iou' (pure IoU). Default: 'dino' if mlpackage exists. + reid_mode = os.environ.get("MULTIHMR_REID", "dino").lower() + dino = None + if enable_rigging and reid_mode == "dino" and DinoReid is not None: + try: + if DinoReid.is_available(): + dino = DinoReid() + LOG.info("MeshRigger: DINOv2 reid enabled") + else: + LOG.info( + "MeshRigger: dino mlpackage absent, IoU only") + except Exception as e: # noqa: BLE001 + LOG.warning("MeshRigger: dino load failed (%s), IoU only", e) + dino = None + dino_weight = float(os.environ.get("MULTIHMR_REID_ALPHA", "0.5")) + self._rigger = MeshRigger( + state, dino_weight=dino_weight, + dino_reid=dino) if enable_rigging else None def start(self) -> None: self._thread = threading.Thread( diff --git a/data_only_viz/state.py b/data_only_viz/state.py index 54523ac..af64d60 100644 --- a/data_only_viz/state.py +++ b/data_only_viz/state.py @@ -119,6 +119,43 @@ class State: # Multi-HMR (SMPL-X 10475 verts x N personnes) persons_smplx: list = field(default_factory=list) # list[SMPLXPerson] smplx_last_t: float = 0.0 + # SMPL-X joint positions (127 joints incl. body + jaw + eyes + hands) + # per pid, shape (127, 3) float32, camera coords (z>0 forward). + # Indices 25-39 = left hand 15 finger joints, 40-54 = right hand. + persons_smplx_joints: dict = field(default_factory=dict) + + # HaMeR MANO hand meshes (v1.2 task #26-28). Keyed by pid -> side + # (0=left, 1=right) -> ndarray shape (778, 3) in camera-space metres. + # Companion arrays per pid/side: + # persons_hands_mesh_t : last_update timestamp (perf_counter) + # persons_hands_mesh_cam_t : (3,) translation of the hand mesh root. + persons_hands_mesh: dict = field(default_factory=dict) + persons_hands_mesh_cam_t: dict = field(default_factory=dict) + persons_hands_mesh_last_t: float = 0.0 + + # ARKit body tracking (iOS ARBodyTracker app) : 91 joints world + # space per pid. Same units as MediaPipe pose_world_landmarks + # (metres, hip-centered). Fresh = updated within < 1 s. + persons_arkit_joints: dict = field(default_factory=dict) + persons_arkit_last_t: dict = field(default_factory=dict) + + # ---- LiDAR / ICP mesh fusion (Task 8 - 2026-05-14) ---- + # Set by the LidarTCPReader poller; consumed by FusionWorker.run_once. + # The mesh-level fusion is complementary to the ARKit *joint* fusion + # above: joints are sparse + 60 Hz, LiDAR is dense + 5-10 Hz. + lidar_points: object = None # np.ndarray (N, 3) float32 ARKit world; None if no frame + lidar_timestamp_ns: int = 0 + icp_metadata: object = None # FusionMetadata from icp_fusion or None + + # v1.3: centralised webcam source. WebcamSource owns the single + # cv2.VideoCapture on the host and writes BGR frames here so all + # consumers (MediaPipe Multi, Apple Vision, Multi-HMR worker, + # HaMeR) read from one shared buffer instead of fighting over the + # camera device. ``latest_bgr_id`` is a monotonic counter so a + # consumer can detect new frames vs. re-reads. + latest_bgr: object = None # np.ndarray (H, W, 3) BGR uint8 + latest_bgr_id: int = 0 + latest_bgr_t: float = 0.0 # Renderer width: int = 1280 @@ -140,6 +177,11 @@ class State: # Derniere frame webcam au format JPEG bytes (pour NSImageView overlay). # Le pose worker la met a jour ; le HUD timer lit et l'affiche. last_webcam_jpeg: bytes | None = None + # Last full RGB frame fed to Multi-HMR (uint8 HxWx3, typ. 672x672). + # Updated by multi_hmr_worker right before inference. Read by + # MeshRigger for DINOv2-based person re-id. None when absent. + last_frame_rgb: np.ndarray | None = None + last_frame_rgb_t: float = 0.0 _lock: threading.RLock = field(default_factory=threading.RLock, repr=False) diff --git a/data_only_viz/tests/test_arkit_fuse.py b/data_only_viz/tests/test_arkit_fuse.py new file mode 100644 index 0000000..fa2bb3b --- /dev/null +++ b/data_only_viz/tests/test_arkit_fuse.py @@ -0,0 +1,42 @@ +"""ArkitFuse stage overrides 14 body slots with ARKit data when fresh.""" +import time + +import numpy as np + +from data_only_viz.state import Kp3D, State +from data_only_viz.pose_filter import PoseFilterChain + + +def _mp33_zero_body(): + return [Kp3D(x=0.0, y=0.0, z=0.0, c=1.0) for _ in range(33)] + + +def test_arkit_fuse_overrides_shoulder(): + state = State() + # ARKit publishes joint 50 (left shoulder) with (1.0, 2.0, 3.0) + arr = np.zeros((91, 3), dtype=np.float32) + arr[50] = (1.0, 2.0, 3.0) + with state.lock(): + state.persons_arkit_joints[0] = arr + state.persons_arkit_last_t[0] = time.perf_counter() + chain = PoseFilterChain(state=state, enabled_stages=("arkit_fuse",)) + bodies = [_mp33_zero_body()] + out = chain.apply(bodies, ids=[0], t_now=time.perf_counter()) + # Slot 11 = L_SHOULDER (from ARKIT91_TO_MP33). + assert out[0][11].x == 1.0 + assert out[0][11].y == 2.0 + assert out[0][11].z == 3.0 + + +def test_arkit_fuse_skips_stale(): + state = State() + arr = np.zeros((91, 3), dtype=np.float32) + arr[50] = (9.0, 9.0, 9.0) + with state.lock(): + state.persons_arkit_joints[0] = arr + state.persons_arkit_last_t[0] = time.perf_counter() - 5.0 + chain = PoseFilterChain(state=state, enabled_stages=("arkit_fuse",)) + bodies = [_mp33_zero_body()] + out = chain.apply(bodies, ids=[0], t_now=time.perf_counter()) + # Stale -> not applied, MediaPipe zero left intact. + assert out[0][11].x == 0.0 diff --git a/data_only_viz/tests/test_arkit_joint_map.py b/data_only_viz/tests/test_arkit_joint_map.py new file mode 100644 index 0000000..2ee5011 --- /dev/null +++ b/data_only_viz/tests/test_arkit_joint_map.py @@ -0,0 +1,32 @@ +"""ARKit 91 joints → MediaPipe Pose 33 mapping integrity.""" +from data_only_viz.arkit_joint_map import ( + ARKIT91_TO_MP33, ARKIT_PELVIS_IDX, MP33_NUM_LANDMARKS, +) + + +def test_mapping_is_tuple_of_pairs(): + assert isinstance(ARKIT91_TO_MP33, tuple) + assert len(ARKIT91_TO_MP33) > 0 + for pair in ARKIT91_TO_MP33: + assert isinstance(pair, tuple) + assert len(pair) == 2 + + +def test_mapping_indices_in_range(): + for arkit_idx, mp33_idx in ARKIT91_TO_MP33: + assert 0 <= arkit_idx < 91, f"arkit idx out of range: {arkit_idx}" + assert 0 <= mp33_idx < MP33_NUM_LANDMARKS, \ + f"mp33 idx out of range: {mp33_idx}" + + +def test_pelvis_index_valid(): + assert 0 <= ARKIT_PELVIS_IDX < 91 + + +def test_no_duplicate_mp33_targets(): + """Each MediaPipe slot must be written by at most one ARKit joint.""" + mp33_seen = set() + for _, mp33_idx in ARKIT91_TO_MP33: + assert mp33_idx not in mp33_seen, \ + f"mp33 slot {mp33_idx} mapped twice" + mp33_seen.add(mp33_idx) diff --git a/data_only_viz/tests/test_dino_reid.py b/data_only_viz/tests/test_dino_reid.py new file mode 100644 index 0000000..27b28f9 --- /dev/null +++ b/data_only_viz/tests/test_dino_reid.py @@ -0,0 +1,77 @@ +"""Tests for the DINOv2 reid backend. + +These tests are skipped automatically if the .mlpackage is not present +(`scripts/convert_dinov2.py` was never run) or pyobjc is unavailable. +""" +from __future__ import annotations + +import time +from pathlib import Path + +import numpy as np +import pytest + +from data_only_viz.dino_reid import DEFAULT_MLPACKAGE, EMBED_DIM, DinoReid + + +pytestmark = pytest.mark.skipif( + not DEFAULT_MLPACKAGE.exists(), + reason=f"DINOv2 mlpackage missing at {DEFAULT_MLPACKAGE}; " + "run scripts/convert_dinov2.py first", +) + + +@pytest.fixture(scope="module") +def reid() -> DinoReid: + return DinoReid() + + +def test_is_available() -> None: + assert DinoReid.is_available() is True + + +def test_load(reid: DinoReid) -> None: + assert reid is not None + assert reid._out_name + + +def test_embed_random_crops_different(reid: DinoReid) -> None: + # Two crops with very different visual content. DINOv2 CLS tokens + # for two iid noise patches are surprisingly close (~0.98), so we + # build crops that are visually distinct: one is mostly red, the + # other is mostly green with a striped pattern. + a = np.zeros((224, 224, 3), dtype=np.uint8) + a[..., 0] = 220 # red + a[40:80, 40:180] = (240, 30, 30) + b = np.zeros((224, 224, 3), dtype=np.uint8) + b[..., 1] = 200 # green + for i in range(0, 224, 16): + b[i:i + 8] = (10, 30, 220) # blue stripes + embs = reid.embed_crops([a, b]) + assert embs.shape == (2, EMBED_DIM) + norms = np.linalg.norm(embs, axis=1) + assert np.allclose(norms, 1.0, atol=1e-3) + cos = float(np.dot(embs[0], embs[1])) + assert cos < 0.95, f"distinct crops too similar: cos={cos:.3f}" + + +def test_embed_identical_crops_same(reid: DinoReid) -> None: + rng = np.random.default_rng(7) + a = rng.integers(0, 255, size=(224, 224, 3), dtype=np.uint8) + embs = reid.embed_crops([a, a.copy()]) + assert embs.shape == (2, EMBED_DIM) + cos = float(np.dot(embs[0], embs[1])) + assert cos > 0.999, f"identical crops cos={cos:.4f} (expected ~1.0)" + + +def test_latency_batch4(reid: DinoReid) -> None: + rng = np.random.default_rng(0) + crops = [rng.integers(0, 255, size=(180, 90, 3), dtype=np.uint8) + for _ in range(4)] + # warmup + reid.embed_crops(crops) + t0 = time.perf_counter() + reid.embed_crops(crops) + dt_ms = (time.perf_counter() - t0) * 1e3 + # Spec target: < 30 ms for batch=4 on M5. + assert dt_ms < 80.0, f"batch=4 too slow: {dt_ms:.1f} ms" diff --git a/data_only_viz/tests/test_face_hand_filter.py b/data_only_viz/tests/test_face_hand_filter.py new file mode 100644 index 0000000..ed034bd --- /dev/null +++ b/data_only_viz/tests/test_face_hand_filter.py @@ -0,0 +1,214 @@ +"""Tests for FaceFilterChain, HandFilterChain, and multi.py discrimination.""" +from __future__ import annotations + +import random +import time + +import pytest + +from data_only_viz.pose_filter import ( + FaceFilterChain, + HandFilterChain, + PoseFilterChain, +) +from data_only_viz.state import Kp3D, PoseKp + + +def _jitter_face(n_pts: int, base_x: float, base_y: float, + amp: float, rng: random.Random) -> list[PoseKp]: + return [ + PoseKp( + x=base_x + rng.uniform(-amp, amp), + y=base_y + rng.uniform(-amp, amp), + z=rng.uniform(-amp, amp), + c=1.0, + ) + for _ in range(n_pts) + ] + + +def test_face_filter_reduces_jitter() -> None: + chain = FaceFilterChain() + rng = random.Random(42) + n_pts = 68 + base_x, base_y = 0.5, 0.5 + amp = 0.01 + outputs: list[list[PoseKp]] = [] + t = 0.0 + for k in range(8): + t += 1.0 / 30.0 + faces = [_jitter_face(n_pts, base_x, base_y, amp, rng)] + out = chain.apply(faces, [0], t) + outputs.append(out[0]) + # Compute variance on x of joint 0 across the last 5 frames. + last = outputs[-5:] + xs = [f[0].x for f in last] + mean = sum(xs) / len(xs) + var = sum((v - mean) ** 2 for v in xs) / len(xs) + assert var < 0.005, f"face filter variance too high: {var}" + + +def test_hand_filter_left_right_independent() -> None: + chain = HandFilterChain() + rng = random.Random(7) + n_pts = 21 + t = 0.0 + last_l: list[PoseKp] = [] + last_r: list[PoseKp] = [] + for k in range(6): + t += 1.0 / 30.0 + left_hand = _jitter_face(n_pts, 0.2, 0.5, 0.008, rng) + right_hand = _jitter_face(n_pts, 0.8, 0.5, 0.008, rng) + out = chain.apply([left_hand, right_hand], [0, 0], + ["Left", "Right"], t) + last_l, last_r = out[0], out[1] + # Left and right hands keep distinct positions despite same pid. + assert abs(last_l[0].x - last_r[0].x) > 0.4 + # Filter reduced jitter on each side. + assert 0.1 < last_l[0].x < 0.35 + assert 0.65 < last_r[0].x < 0.9 + + +def test_hand_filter_chain_wrapper_smoke() -> None: + chain = PoseFilterChain() + rng = random.Random(0) + hands = [_jitter_face(21, 0.5, 0.5, 0.01, rng) for _ in range(2)] + out = chain.apply_hand(hands, [0, 1], ["Left", "Right"], t_now=0.1) + assert len(out) == 2 + assert len(out[0]) == 21 + + +def test_face_filter_disabled_passthrough() -> None: + chain = FaceFilterChain(enabled_stages=()) + faces = [[PoseKp(x=0.5, y=0.5, z=0.0, c=1.0) for _ in range(68)]] + out = chain.apply(faces, [0], t_now=0.0) + assert out[0][0].x == 0.5 + + +def test_face_hand_latency_under_5ms() -> None: + """Full chain (body 33 + face 68 + hand 21x2) < 5 ms per frame.""" + body_chain = PoseFilterChain( + enabled_stages=("median", "kalman", "lookahead", "ik")) + face_chain = FaceFilterChain() + hand_chain = HandFilterChain() + rng = random.Random(0) + body = [Kp3D(x=i * 0.01, y=i * 0.02, z=i * 0.03, c=1.0) + for i in range(33)] + face = _jitter_face(68, 0.5, 0.5, 0.01, rng) + hand_l = _jitter_face(21, 0.2, 0.5, 0.01, rng) + hand_r = _jitter_face(21, 0.8, 0.5, 0.01, rng) + # Warm-up + for k in range(5): + t = k * 0.033 + body_chain.apply([body], [0], t) + face_chain.apply([face], [0], t) + hand_chain.apply([hand_l, hand_r], [0, 0], ["Left", "Right"], t) + # Measure + durs: list[float] = [] + for k in range(30): + t = (k + 5) * 0.033 + t0 = time.perf_counter() + body_chain.apply([body], [0], t) + face_chain.apply([face], [0], t) + hand_chain.apply([hand_l, hand_r], [0, 0], ["Left", "Right"], t) + durs.append((time.perf_counter() - t0) * 1000.0) + avg = sum(durs) / len(durs) + # CI margin : actual M-class target is < 5 ms ; allow 25 ms in tests. + assert avg < 25.0, f"chain too slow: {avg:.2f} ms" + + +# ----------------------- multi.py discrimination --------------------------- + + +def _make_body(n_visible: int) -> list[PoseKp]: + """Make a 33-joint body with `n_visible` high-conf joints, rest low.""" + out: list[PoseKp] = [] + for i in range(33): + c = 1.0 if i < n_visible else 0.05 + # Spread across both x and y so the bbox has non-zero area. + out.append(PoseKp(x=0.1 + i * 0.01, y=0.2 + i * 0.005, z=0.0, c=c)) + return out + + +def _make_body3d(n: int = 33) -> list[Kp3D]: + return [Kp3D(x=0.0, y=0.0, z=0.0, c=1.0) for _ in range(n)] + + +def _instantiate_worker(): + """Build a MultiWorker without starting the thread (skip if cv2 missing).""" + pytest.importorskip("cv2", reason="opencv not installed") + from data_only_viz.multi import MultiWorker + from data_only_viz.state import State + return MultiWorker(state=State(), camera_index=-1) + + +def test_ghost_rejection_drops_low_visibility_body() -> None: + w = _instantiate_worker() + bodies = [_make_body(n_visible=5), _make_body(n_visible=25)] + b3d = [_make_body3d(), _make_body3d()] + ids = [0, 1] + new_bodies, new_b3d, new_ids = w._reject_ghosts_and_nms(bodies, b3d, ids) + assert len(new_bodies) == 1 + assert len(new_b3d) == 1 + assert new_ids == [1] + assert w._n_ghost_dropped == 1 + + +def test_nms_keeps_best_score() -> None: + w = _instantiate_worker() + # Two heavily overlapping bodies, second has higher mean confidence. + b1 = _make_body(n_visible=20) + b2 = _make_body(n_visible=33) + new_bodies, _, new_ids = w._reject_ghosts_and_nms([b1, b2], [], [0, 1]) + # IoU of identical bbox => one dropped, the higher-score one kept. + assert len(new_bodies) == 1 + assert new_ids == [1] + + +def test_pid_persistence_through_short_absence() -> None: + w = _instantiate_worker() + body = _make_body(n_visible=30) + # Frame 1..30 : pid 0 present. + for _ in range(30): + new_ids = w._apply_pid_hysteresis([body], [0]) + assert new_ids == [0] + # Frames 31..35 : pid 0 absent (no detection). + for _ in range(5): + w._apply_pid_hysteresis([], []) + # Frame 36 : a NEW pid 9 appears at the same bbox -> should be remapped. + new_ids = w._apply_pid_hysteresis([body], [9]) + assert new_ids == [0], f"expected hysteresis remap to 0, got {new_ids}" + + +def test_drop_low_visibility_face() -> None: + w = _instantiate_worker() + # 30 valid (non-zero) + 38 zeros. + face_bad = [ + PoseKp(x=(0.1 if i < 30 else 0.0), + y=(0.1 if i < 30 else 0.0), z=0.0, c=1.0) + for i in range(68) + ] + face_ok = [ + PoseKp(x=0.1 + i * 0.001, y=0.2, z=0.0, c=1.0) + for i in range(68) + ] + kept, ids = w._drop_low_visibility( + [face_bad, face_ok], [0, 1], min_visible=50, which="face") + assert len(kept) == 1 + assert ids == [1] + assert w._n_face_dropped == 1 + + +def test_drop_low_visibility_hand() -> None: + w = _instantiate_worker() + hand_bad = [PoseKp(x=0.0, y=0.0, z=0.0, c=1.0) for _ in range(21)] + # Only 10 visible (others are zero) -> drop. + for i in range(10): + hand_bad[i] = PoseKp(x=0.5, y=0.5, z=0.0, c=1.0) + hand_ok = [PoseKp(x=0.1 + i * 0.01, y=0.2, z=0.0, c=1.0) + for i in range(21)] + kept, ids = w._drop_low_visibility( + [hand_bad, hand_ok], [0, 1], min_visible=15, which="hand") + assert len(kept) == 1 + assert ids == [1] + assert w._n_hand_dropped == 1 diff --git a/data_only_viz/tests/test_icp_fusion.py b/data_only_viz/tests/test_icp_fusion.py new file mode 100644 index 0000000..ed48a00 --- /dev/null +++ b/data_only_viz/tests/test_icp_fusion.py @@ -0,0 +1,144 @@ +"""Tests for ICP registration of SMPL-X verts onto LiDAR point clouds.""" +from __future__ import annotations + +import numpy as np +import pytest + +pytest.importorskip("open3d") + + +def _synthetic_smplx_torso(n: int = 1500, seed: int = 0) -> np.ndarray: + """Generate a coarse capsule-like point cloud standing in for SMPL-X verts.""" + rng = np.random.RandomState(seed) + z = rng.uniform(0.0, 1.7, size=n) + r = 0.12 + 0.02 * rng.randn(n) + theta = rng.uniform(0, 2 * np.pi, size=n) + x = r * np.cos(theta) + y = r * np.sin(theta) + return np.stack([x, y, z], axis=1).astype(np.float32) + + +def test_icp_recovers_small_translation() -> None: + from data_only_viz.icp_fusion import IcpConfig, register_mesh_to_lidar + + src = _synthetic_smplx_torso(seed=1) + translation = np.array([0.05, 0.02, 0.10], dtype=np.float32) + tgt = src + translation + 0.005 * np.random.RandomState(2).randn(*src.shape).astype(np.float32) + + out = register_mesh_to_lidar(src, tgt, config=IcpConfig()) + + assert out.accepted, f"ICP should accept, got fitness={out.fitness:.3f}" + truth = src + translation + err_before = np.linalg.norm(src - truth, axis=1).mean() + err_after = np.linalg.norm(out.vertices_registered - truth, axis=1).mean() + assert err_after < err_before * 0.5, f"err before={err_before:.4f} after={err_after:.4f}" + + +def test_icp_rejects_when_lidar_too_sparse() -> None: + from data_only_viz.icp_fusion import IcpConfig, register_mesh_to_lidar + + src = _synthetic_smplx_torso(seed=3) + tgt = src[:5] + + out = register_mesh_to_lidar(src, tgt, config=IcpConfig()) + assert not out.accepted + np.testing.assert_array_equal(out.vertices_registered, src) + + +def test_icp_rejects_on_nan_input() -> None: + from data_only_viz.icp_fusion import IcpConfig, register_mesh_to_lidar + + src = _synthetic_smplx_torso(seed=4) + src[10, 1] = np.nan + tgt = src.copy() + tgt = np.nan_to_num(tgt, nan=0.0) + + out = register_mesh_to_lidar(src, tgt, config=IcpConfig()) + assert not out.accepted + np.testing.assert_array_equal(out.vertices_registered, src) + + +def test_icp_preserves_dtype_and_shape() -> None: + from data_only_viz.icp_fusion import IcpConfig, register_mesh_to_lidar + + src = _synthetic_smplx_torso(seed=5) + tgt = src + np.array([0.0, 0.0, 0.02], dtype=np.float32) + out = register_mesh_to_lidar(src, tgt, config=IcpConfig()) + assert out.vertices_registered.shape == src.shape + assert out.vertices_registered.dtype == np.float32 + + +def test_partition_lidar_by_pid_two_people() -> None: + from data_only_viz.icp_fusion import partition_lidar_by_pid + + src_a = _synthetic_smplx_torso(seed=10) + np.array([-0.75, 0.0, 0.0], dtype=np.float32) + src_b = _synthetic_smplx_torso(seed=11) + np.array([+0.75, 0.0, 0.0], dtype=np.float32) + pelvis_a = src_a.mean(axis=0) + pelvis_b = src_b.mean(axis=0) + + lidar = np.concatenate([ + src_a + 0.01 * np.random.RandomState(20).randn(*src_a.shape).astype(np.float32), + src_b + 0.01 * np.random.RandomState(21).randn(*src_b.shape).astype(np.float32), + np.array([[10.0, 10.0, 10.0]] * 100, dtype=np.float32), + ]) + + parts = partition_lidar_by_pid(lidar, pelvises={0: pelvis_a, 1: pelvis_b}, max_dist_m=1.0) + + assert set(parts.keys()) == {0, 1} + assert parts[0].shape[0] > 1000 + assert parts[1].shape[0] > 1000 + assert not np.any(np.linalg.norm(parts[0] - np.array([10, 10, 10]), axis=1) < 0.5) + assert not np.any(np.linalg.norm(parts[1] - np.array([10, 10, 10]), axis=1) < 0.5) + + +def test_partition_returns_empty_dict_when_no_pelvises() -> None: + from data_only_viz.icp_fusion import partition_lidar_by_pid + + out = partition_lidar_by_pid(np.zeros((100, 3), dtype=np.float32), pelvises={}, max_dist_m=1.0) + assert out == {} + + +def test_fusion_worker_in_place_update(monkeypatch) -> None: + from data_only_viz.icp_fusion import FusionWorker, IcpConfig + from data_only_viz.lidar_calib import Extrinsic + from data_only_viz.state import SMPLXPerson, State + + src = _synthetic_smplx_torso(seed=30) + verts = np.zeros((10475, 3), dtype=np.float32) + verts[: src.shape[0]] = src + verts[5559] = src.mean(axis=0) + + person = SMPLXPerson(pid=0, vertices_3d=verts.copy()) + state = State() + state.persons_smplx = [person] + + lidar_pts = src + np.array([0.0, 0.04, 0.0], dtype=np.float32) + state.lidar_points = lidar_pts + state.lidar_timestamp_ns = 1 + + worker = FusionWorker( + extrinsic=Extrinsic.identity(), + config=IcpConfig(), + ) + metadata = worker.run_once(state) + + assert metadata.applied == {0} + delta = state.persons_smplx[0].vertices_3d[5559] - verts[5559] + assert 0.02 <= delta[1] <= 0.06 + + +def test_fusion_worker_skips_when_no_lidar() -> None: + from data_only_viz.icp_fusion import FusionWorker, IcpConfig + from data_only_viz.lidar_calib import Extrinsic + from data_only_viz.state import SMPLXPerson, State + + verts = np.zeros((10475, 3), dtype=np.float32) + verts[5559] = [0.0, 1.0, 2.0] + state = State() + state.persons_smplx = [SMPLXPerson(pid=0, vertices_3d=verts.copy())] + state.lidar_points = None + + worker = FusionWorker(extrinsic=Extrinsic.identity(), config=IcpConfig()) + metadata = worker.run_once(state) + assert metadata.applied == set() + np.testing.assert_array_equal(state.persons_smplx[0].vertices_3d, verts) diff --git a/data_only_viz/tests/test_iphone_osc_listener.py b/data_only_viz/tests/test_iphone_osc_listener.py new file mode 100644 index 0000000..7ce81ba --- /dev/null +++ b/data_only_viz/tests/test_iphone_osc_listener.py @@ -0,0 +1,51 @@ +"""IphoneOSCListener writes ARKit joints to state from OSC packets.""" +import time + +import numpy as np +import pytest +from pythonosc.udp_client import SimpleUDPClient + +from data_only_viz.state import State +from data_only_viz.iphone_osc_listener import ( + IphoneOSCListener, IPHONE_OSC_PORT, +) + + +@pytest.fixture() +def listener(): + state = State() + listener = IphoneOSCListener(state, port=IPHONE_OSC_PORT + 100) + listener.start() + yield state, listener + listener.stop() + + +def test_kp_message_updates_state(listener): + state, lst = listener + client = SimpleUDPClient("127.0.0.1", lst.port) + client.send_message("/body3d/kp", [0, 1, 0.1, 0.2, 0.3]) + # Settle + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline: + with state.lock(): + if 0 in state.persons_arkit_joints: + arr = state.persons_arkit_joints[0] + if arr[1, 0] != 0.0: + break + time.sleep(0.02) + with state.lock(): + assert 0 in state.persons_arkit_joints, \ + "OSC /body3d/kp message not received within 1s" + arr = state.persons_arkit_joints[0] + assert arr.shape == (91, 3) + assert np.allclose(arr[1], [0.1, 0.2, 0.3]) + + +def test_gc_drops_stale_pids(listener): + state, lst = listener + with state.lock(): + state.persons_arkit_joints[7] = np.zeros((91, 3), dtype=np.float32) + state.persons_arkit_last_t[7] = time.perf_counter() - 5.0 + lst._gc_stale() + with state.lock(): + assert 7 not in state.persons_arkit_joints diff --git a/data_only_viz/tests/test_lidar_calib.py b/data_only_viz/tests/test_lidar_calib.py new file mode 100644 index 0000000..ba4a300 --- /dev/null +++ b/data_only_viz/tests/test_lidar_calib.py @@ -0,0 +1,76 @@ +"""Tests for LiDAR <-> webcam extrinsic calibration persistence.""" +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + + +def test_extrinsic_default_is_identity() -> None: + from data_only_viz.lidar_calib import Extrinsic + + e = Extrinsic.identity() + np.testing.assert_allclose(e.T_arkit_to_cam, np.eye(4)) + assert e.confidence == 0.0 + assert e.captured_at_iso == "" + + +def test_extrinsic_roundtrip_json(tmp_path: Path) -> None: + from data_only_viz.lidar_calib import Extrinsic, load_extrinsic, save_extrinsic + + T = np.eye(4) + T[:3, 3] = [0.1, -0.05, 0.30] + e = Extrinsic(T_arkit_to_cam=T, confidence=0.95, captured_at_iso="2026-05-14T12:00:00Z") + + path = tmp_path / "extrinsic.json" + save_extrinsic(e, path) + loaded = load_extrinsic(path) + + np.testing.assert_allclose(loaded.T_arkit_to_cam, T, atol=1e-10) + assert loaded.confidence == pytest.approx(0.95) + assert loaded.captured_at_iso == "2026-05-14T12:00:00Z" + + +def test_load_extrinsic_missing_path_returns_identity(tmp_path: Path) -> None: + from data_only_viz.lidar_calib import load_extrinsic + + e = load_extrinsic(tmp_path / "does-not-exist.json") + np.testing.assert_allclose(e.T_arkit_to_cam, np.eye(4)) + assert e.confidence == 0.0 + + +def test_kabsch_recovers_known_rigid_transform() -> None: + from data_only_viz.lidar_calib import kabsch_rigid + + rng = np.random.RandomState(7) + src = rng.randn(20, 3) + theta = np.deg2rad(30.0) + R = np.array([ + [np.cos(theta), 0, np.sin(theta)], + [0, 1, 0], + [-np.sin(theta), 0, np.cos(theta)], + ]) + t = np.array([0.1, -0.2, 0.5]) + tgt = src @ R.T + t + + T = kabsch_rigid(src, tgt) + R_est = T[:3, :3] + t_est = T[:3, 3] + np.testing.assert_allclose(R_est, R, atol=1e-6) + np.testing.assert_allclose(t_est, t, atol=1e-6) + + +def test_kabsch_requires_at_least_three_pairs() -> None: + from data_only_viz.lidar_calib import kabsch_rigid + + with pytest.raises(ValueError, match="at least 3"): + kabsch_rigid(np.zeros((2, 3)), np.zeros((2, 3))) + + +def test_kabsch_rejects_mismatched_shapes() -> None: + from data_only_viz.lidar_calib import kabsch_rigid + + with pytest.raises(ValueError, match="shape"): + kabsch_rigid(np.zeros((5, 3)), np.zeros((4, 3))) diff --git a/data_only_viz/tests/test_lidar_receiver.py b/data_only_viz/tests/test_lidar_receiver.py new file mode 100644 index 0000000..4817488 --- /dev/null +++ b/data_only_viz/tests/test_lidar_receiver.py @@ -0,0 +1,111 @@ +"""Unit tests for the iPhone LiDAR TCP frame decoder.""" +from __future__ import annotations + +import struct + +import numpy as np +import pytest + + +def _encode_frame(points: np.ndarray, timestamp_ns: int) -> bytes: + """Mimic the iPhone-side encoder for round-trip testing.""" + n = points.shape[0] + body = struct.pack(">Q", timestamp_ns) + struct.pack(">I", n) + points.astype("I", len(body)) + return header + body + + +def test_decode_lidar_frame_roundtrip() -> None: + from data_only_viz.lidar_receiver import LidarFrame, decode_frame + + pts = np.array([[0.1, 0.2, 0.3], [-1.0, 2.0, 5.5]], dtype=np.float32) + payload = _encode_frame(pts, timestamp_ns=1_700_000_000_000_000_000) + + body = payload[4:] + frame = decode_frame(body) + + assert isinstance(frame, LidarFrame) + assert frame.timestamp_ns == 1_700_000_000_000_000_000 + np.testing.assert_allclose(frame.points, pts, atol=1e-6) + + +def test_decode_lidar_frame_rejects_truncated() -> None: + from data_only_viz.lidar_receiver import decode_frame + + pts = np.array([[1.0, 2.0, 3.0]], dtype=np.float32) + body = ( + struct.pack(">Q", 0) + + struct.pack(">I", 1) + + pts.astype(" None: + from data_only_viz.lidar_receiver import decode_frame + + body = struct.pack(">Q", 0) + struct.pack(">I", 0) + with pytest.raises(ValueError, match="vertex_count"): + decode_frame(body) + + +import socket +import threading +import time + + +@pytest.fixture +def unused_tcp_port() -> int: + """Bind to port 0 to grab a free port from the OS, then release it.""" + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +def _serve_one_frame(port: int, frame_bytes: bytes) -> None: + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(("127.0.0.1", port)) + srv.listen(1) + conn, _ = srv.accept() + conn.sendall(frame_bytes) + time.sleep(0.1) + conn.close() + srv.close() + + +def test_reader_grabs_latest_frame(unused_tcp_port: int) -> None: + from data_only_viz.lidar_receiver import LidarTCPReader + + pts = np.array([[1.0, 2.0, 3.0]], dtype=np.float32) + frame = _encode_frame(pts, timestamp_ns=42) + t = threading.Thread(target=_serve_one_frame, args=(unused_tcp_port, frame), daemon=True) + t.start() + time.sleep(0.05) + + reader = LidarTCPReader(host="127.0.0.1", port=unused_tcp_port, connect_timeout_s=2.0) + reader.start() + deadline = time.monotonic() + 2.0 + latest = None + while time.monotonic() < deadline: + latest = reader.latest() + if latest is not None: + break + time.sleep(0.02) + reader.stop() + t.join(timeout=1.0) + + assert latest is not None + assert latest.timestamp_ns == 42 + np.testing.assert_allclose(latest.points, pts, atol=1e-6) + + +def test_reader_returns_none_before_first_frame(unused_tcp_port: int) -> None: + from data_only_viz.lidar_receiver import LidarTCPReader + + reader = LidarTCPReader(host="127.0.0.1", port=unused_tcp_port, connect_timeout_s=0.05) + # Do not start it; latest() must be None. + assert reader.latest() is None diff --git a/data_only_viz/tests/test_multi_hmr_worker.py b/data_only_viz/tests/test_multi_hmr_worker.py index 882a430..f77b8cc 100644 --- a/data_only_viz/tests/test_multi_hmr_worker.py +++ b/data_only_viz/tests/test_multi_hmr_worker.py @@ -51,3 +51,16 @@ def test_state_mutations_are_all_under_lock(): f"line {lineno} mutates persons_smplx without a nearby `state.lock()` context:\n" f"{lines[lineno - 1]}" ) + + +def test_predict_once_returns_none_when_coreml_unavailable(monkeypatch): + from data_only_viz.multi_hmr_worker import MultiHMRWorker + from data_only_viz.state import State + # Force CoreML loader to return None + state = State() + worker = MultiHMRWorker(state, num_persons=1) + monkeypatch.setattr(worker, "_get_or_load_coreml_backend", lambda: None) + import pytest, numpy as np + rgb = np.zeros((480, 640, 3), dtype=np.uint8) + with pytest.raises(NotImplementedError): + worker.predict_once(rgb) diff --git a/data_only_viz/tests/test_multihmr_arkit_z.py b/data_only_viz/tests/test_multihmr_arkit_z.py new file mode 100644 index 0000000..bc4cbcf --- /dev/null +++ b/data_only_viz/tests/test_multihmr_arkit_z.py @@ -0,0 +1,41 @@ +"""arkit_pelvis_z_override : if ARKit pelvis z is fresh, replace +the Multi-HMR pred_cam_t.z so the SMPL-X mesh sits at the actual +distance instead of HaMeR's monocular guess. +""" +import time + +import numpy as np + +from data_only_viz.state import State +from data_only_viz.multi_hmr_worker import arkit_pelvis_z_override + + +def test_returns_arkit_z_when_fresh(): + state = State() + arr = np.zeros((91, 3), dtype=np.float32) + arr[1] = (0.0, 0.0, 2.5) # ARKIT_PELVIS_IDX=1, z=2.5 m + with state.lock(): + state.persons_arkit_joints[0] = arr + state.persons_arkit_last_t[0] = time.perf_counter() + z_pred = 5.0 # Multi-HMR ambiguous guess + z_out = arkit_pelvis_z_override(state, pid=0, z_pred=z_pred) + assert z_out == 2.5 + + +def test_keeps_pred_when_stale(): + state = State() + arr = np.zeros((91, 3), dtype=np.float32) + arr[1] = (0.0, 0.0, 2.5) + with state.lock(): + state.persons_arkit_joints[0] = arr + state.persons_arkit_last_t[0] = time.perf_counter() - 5.0 + z_pred = 5.0 + z_out = arkit_pelvis_z_override(state, pid=0, z_pred=z_pred) + assert z_out == 5.0 + + +def test_keeps_pred_when_pid_missing(): + state = State() + z_pred = 4.2 + z_out = arkit_pelvis_z_override(state, pid=99, z_pred=z_pred) + assert z_out == 4.2 diff --git a/data_only_viz/tests/test_multihmr_coreml.py b/data_only_viz/tests/test_multihmr_coreml.py index 2eb7338..631c441 100644 --- a/data_only_viz/tests/test_multihmr_coreml.py +++ b/data_only_viz/tests/test_multihmr_coreml.py @@ -80,8 +80,12 @@ def test_infer_latency_under_target(): times.sort() median_ms = times[n // 2] print(f"median latency: {median_ms:.1f} ms (n={n})") - # Target 50ms = 20fps. M5 bench shows ~29ms. Generous margin. - assert median_ms < 80.0, f"median {median_ms:.1f}ms > 80ms target" + # Full Multi-HMR CoreML on M5: ~120-140 ms standalone (7-8 fps), + # see scripts/bench_multihmr_coreml.py and multihmr_coreml.py + # docstring. The earlier 80 ms target was a backbone-only probe + # estimate that does not hold for the full model. 250 ms gives + # headroom for thermal/contention without masking a regression. + assert median_ms < 250.0, f"median {median_ms:.1f}ms > 250ms target" def test_filter_threshold(): diff --git a/data_only_viz/tests/test_open3d_smoke.py b/data_only_viz/tests/test_open3d_smoke.py new file mode 100644 index 0000000..abb65f4 --- /dev/null +++ b/data_only_viz/tests/test_open3d_smoke.py @@ -0,0 +1,35 @@ +"""Smoke test for the Open3D dependency used by ICP fusion.""" +from __future__ import annotations + +import numpy as np +import pytest + +open3d = pytest.importorskip("open3d") + + +def test_open3d_pointcloud_roundtrip() -> None: + pts = np.random.RandomState(0).randn(100, 3).astype(np.float32) + pcd = open3d.geometry.PointCloud() + pcd.points = open3d.utility.Vector3dVector(pts) + out = np.asarray(pcd.points) + assert out.shape == (100, 3) + np.testing.assert_allclose(out, pts, atol=1e-5) + + +def test_open3d_icp_converges_on_translated_copy() -> None: + rng = np.random.RandomState(1) + src = rng.randn(500, 3).astype(np.float64) + translation = np.array([0.10, -0.05, 0.20]) + tgt = src + translation + + src_pcd = open3d.geometry.PointCloud() + src_pcd.points = open3d.utility.Vector3dVector(src) + tgt_pcd = open3d.geometry.PointCloud() + tgt_pcd.points = open3d.utility.Vector3dVector(tgt) + + result = open3d.pipelines.registration.registration_icp( + src_pcd, tgt_pcd, max_correspondence_distance=0.5, + init=np.eye(4), + estimation_method=open3d.pipelines.registration.TransformationEstimationPointToPoint(), + ) + np.testing.assert_allclose(result.transformation[:3, 3], translation, atol=1e-3) diff --git a/data_only_viz/tests/test_pose_filter.py b/data_only_viz/tests/test_pose_filter.py new file mode 100644 index 0000000..38cfd9f --- /dev/null +++ b/data_only_viz/tests/test_pose_filter.py @@ -0,0 +1,127 @@ +"""Tests for the 3D pose filter chain.""" +from __future__ import annotations + +import math + +import pytest + +from data_only_viz.pose_filter import ( + IKConstraints, + KalmanCV, + LookaheadPredictor, + MedianFilter, + PoseFilterChain, + L_ELBOW, + L_SHOULDER, + L_WRIST, +) +from data_only_viz.state import Kp3D + + +def _body(values: list[tuple[float, float, float]]) -> list[Kp3D]: + """Build a 33-joint body, fill remaining with zeros.""" + out = [Kp3D(x=v[0], y=v[1], z=v[2], c=1.0) for v in values] + while len(out) < 33: + out.append(Kp3D(x=0.0, y=0.0, z=0.0, c=1.0)) + return out + + +def test_median_filter_kills_spike() -> None: + mf = MedianFilter(window=3) + pid, j = 0, 0 + # Warm up + mf.apply(pid, j, 0.0, 0.0, 0.0) + mf.apply(pid, j, 0.01, 0.0, 0.0) + mf.apply(pid, j, 0.02, 0.0, 0.0) + # Spike (NaN) + x, y, z = mf.apply(pid, j, float("nan"), float("nan"), float("nan")) + assert math.isfinite(x) and math.isfinite(y) and math.isfinite(z) + assert abs(x) < 0.1 + # Big outlier in x + x2, _, _ = mf.apply(pid, j, 10.0, 0.0, 0.0) + assert x2 < 1.0 + + +def test_kalman_converges() -> None: + # Use a noisy constant-velocity signal : Kalman CV should converge. + import random + rng = random.Random(0) + kf = KalmanCV(q=1e-3, r=1e-2) + pid, j = 0, 0 + t = 0.0 + dt = 1.0 / 30.0 + vel = 0.3 # m/s + errs: list[float] = [] + for i in range(120): + t += dt + true_pos = vel * t + meas = true_pos + rng.gauss(0.0, 0.01) # 1 cm gaussian noise + out = kf.step(pid, j, meas, 0.0, 0.0, t) + if i > 30: + errs.append(abs(out[0] - true_pos)) + mean_err = sum(errs) / len(errs) + assert mean_err < 0.01 # ±1 cm post warmup + + +def test_lookahead_extrapolates_constant_velocity() -> None: + pred = LookaheadPredictor(lookahead_ms=50.0, max_velocity=5.0) + x, y, z = pred.step(0.0, 0.0, 0.0, 1.0, 0.0, 0.0) + assert abs(x - 0.05) < 1e-6 + assert abs(y) < 1e-9 and abs(z) < 1e-9 + # Velocity cap + x2, _, _ = pred.step(0.0, 0.0, 0.0, 100.0, 0.0, 0.0) + assert abs(x2 - 5.0 * 0.050) < 1e-6 + + +def test_ik_clamps_elbow_180_plus() -> None: + ik = IKConstraints() + # Shoulder at origin, elbow at (1,0,0), wrist BEHIND elbow at (2,0,0) + # -> shoulder-elbow-wrist angle is 180 deg, exceeds 175 deg limit. + coords: list[tuple[float, float, float]] = [(0.0, 0.0, 0.0)] * 33 + coords[L_SHOULDER] = (0.0, 0.0, 0.0) + coords[L_ELBOW] = (1.0, 0.0, 0.0) + coords[L_WRIST] = (2.0, 0.0, 0.0) + body = _body(coords) + out = ik.apply(body) + p = (out[L_SHOULDER].x, out[L_SHOULDER].y, out[L_SHOULDER].z) + e = (out[L_ELBOW].x, out[L_ELBOW].y, out[L_ELBOW].z) + w = (out[L_WRIST].x, out[L_WRIST].y, out[L_WRIST].z) + v_pj = (p[0] - e[0], p[1] - e[1], p[2] - e[2]) + v_cj = (w[0] - e[0], w[1] - e[1], w[2] - e[2]) + n_pj = math.sqrt(sum(c * c for c in v_pj)) + n_cj = math.sqrt(sum(c * c for c in v_cj)) + cos_a = (v_pj[0] * v_cj[0] + v_pj[1] * v_cj[1] + v_pj[2] * v_cj[2] + ) / (n_pj * n_cj) + cos_a = max(-1.0, min(1.0, cos_a)) + ang_deg = math.degrees(math.acos(cos_a)) + assert ang_deg <= 175.5 + # Bone length preserved + assert abs(n_cj - 1.0) < 1e-6 + + +def test_chain_no_op_when_disabled() -> None: + chain = PoseFilterChain(enabled_stages=()) + body = _body([(0.1, 0.2, 0.3), (0.4, 0.5, 0.6)]) + out = chain.apply([body], [0], t_now=0.0) + assert len(out) == 1 + for i in range(len(body)): + assert out[0][i].x == body[i].x + assert out[0][i].y == body[i].y + assert out[0][i].z == body[i].z + + +def test_chain_latency_under_2ms() -> None: + chain = PoseFilterChain( + enabled_stages=("median", "kalman", "lookahead", "ik")) + body = _body([(i * 0.01, i * 0.02, i * 0.03) for i in range(33)]) + # Warm up internal state + for k in range(5): + chain.apply([body, body], [0, 1], t_now=k * 0.033) + # Measure + times: list[float] = [] + for k in range(30): + chain.apply([body, body], [0, 1], t_now=(k + 5) * 0.033) + times.append(chain.last_apply_ms) + avg = sum(times) / len(times) + # Generous bound for CI ; live target is <2 ms but allow 10 ms in tests. + assert avg < 10.0 diff --git a/data_only_viz/tests/test_state_arkit.py b/data_only_viz/tests/test_state_arkit.py new file mode 100644 index 0000000..e5fa2dc --- /dev/null +++ b/data_only_viz/tests/test_state_arkit.py @@ -0,0 +1,22 @@ +"""State must expose persons_arkit_joints + persons_arkit_last_t.""" +import numpy as np + +from data_only_viz.state import State + + +def test_state_has_arkit_joint_fields(): + s = State() + assert hasattr(s, "persons_arkit_joints") + assert hasattr(s, "persons_arkit_last_t") + assert isinstance(s.persons_arkit_joints, dict) + assert isinstance(s.persons_arkit_last_t, dict) + + +def test_state_arkit_joints_writable_under_lock(): + s = State() + arr = np.zeros((91, 3), dtype=np.float32) + with s.lock(): + s.persons_arkit_joints[0] = arr + s.persons_arkit_last_t[0] = 1.5 + assert 0 in s.persons_arkit_joints + assert s.persons_arkit_last_t[0] == 1.5 diff --git a/data_only_viz/uv.lock b/data_only_viz/uv.lock index fa66763..3e11e2e 100644 --- a/data_only_viz/uv.lock +++ b/data_only_viz/uv.lock @@ -2,11 +2,15 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", "python_full_version < '3.12' and sys_platform == 'win32'", - "python_full_version >= '3.12' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version >= '3.12' and python_full_version < '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.12' and sys_platform == 'darwin'", "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", @@ -21,6 +25,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl", hash = "sha256:88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d", size = 135750, upload-time = "2026-01-28T10:17:04.19Z" }, ] +[[package]] +name = "addict" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/ef/fd7649da8af11d93979831e8f1f8097e85e82d5bfeabc8c68b39175d8e75/addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494", size = 9186, upload-time = "2020-11-21T16:21:31.416Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/00/b08f23b7d7e1e14ce01419a467b583edbb93c6cdb8654e54a9cc579cd61f/addict-2.4.0-py3-none-any.whl", hash = "sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc", size = 3832, upload-time = "2020-11-21T16:21:29.588Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -87,6 +100,9 @@ detrpose = [ { name = "transformers" }, { name = "xtcocotools" }, ] +lidar = [ + { name = "open3d" }, +] multihmr = [ { name = "einops" }, { name = "huggingface-hub" }, @@ -115,6 +131,21 @@ pose = [ { name = "opencv-python" }, { name = "ultralytics" }, ] +smplerx = [ + { name = "einops" }, + { name = "mmcv-lite" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "pillow" }, + { name = "scipy" }, + { name = "smplx" }, + { name = "timm" }, + { name = "torch" }, + { name = "torchvision" }, + { name = "tqdm" }, + { name = "ultralytics" }, + { name = "yacs" }, +] [package.dev-dependencies] dev = [ @@ -126,19 +157,25 @@ requires-dist = [ { name = "cloudpickle", marker = "extra == 'detrpose'", specifier = ">=3.0" }, { name = "coremltools", marker = "extra == 'pose'", specifier = ">=9.0" }, { name = "einops", marker = "extra == 'multihmr'", specifier = ">=0.8" }, + { name = "einops", marker = "extra == 'smplerx'", specifier = ">=0.8" }, { name = "huggingface-hub", marker = "extra == 'multihmr'", specifier = ">=0.24" }, { name = "iopath", marker = "extra == 'detrpose'", specifier = ">=0.1.10" }, { name = "iopath", marker = "extra == 'multihmr'", specifier = ">=0.1.10" }, { name = "mediapipe", marker = "extra == 'pose'", specifier = ">=0.10.35" }, + { name = "mmcv-lite", marker = "extra == 'smplerx'", specifier = ">=2.1" }, { name = "numpy", specifier = ">=1.26,<2" }, { name = "numpy", marker = "extra == 'multihmr'", specifier = ">=1.26,<2" }, { name = "numpy", marker = "extra == 'nlf'", specifier = ">=1.26" }, + { name = "numpy", marker = "extra == 'smplerx'", specifier = ">=1.26,<2" }, { name = "omegaconf", marker = "extra == 'detrpose'", specifier = ">=2.3" }, + { name = "open3d", marker = "extra == 'lidar'", specifier = ">=0.18,<0.20" }, { name = "opencv-python", marker = "extra == 'detrpose'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'multihmr'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'nlf'", specifier = ">=4.10" }, { name = "opencv-python", marker = "extra == 'pose'", specifier = ">=4.10" }, + { name = "opencv-python", marker = "extra == 'smplerx'", specifier = ">=4.10" }, { name = "pillow", marker = "extra == 'multihmr'", specifier = ">=10.0" }, + { name = "pillow", marker = "extra == 'smplerx'", specifier = ">=10.0" }, { name = "pycocotools", marker = "extra == 'detrpose'", specifier = ">=2.0" }, { name = "pyobjc-core", specifier = ">=10.3" }, { name = "pyobjc-framework-avfoundation", specifier = ">=10.3" }, @@ -151,25 +188,42 @@ requires-dist = [ { name = "scipy", specifier = ">=1.13" }, { name = "scipy", marker = "extra == 'detrpose'", specifier = ">=1.13" }, { name = "scipy", marker = "extra == 'multihmr'", specifier = ">=1.13" }, + { name = "scipy", marker = "extra == 'smplerx'", specifier = ">=1.13" }, { name = "smplx", marker = "extra == 'multihmr'", specifier = ">=0.1.28" }, + { name = "smplx", marker = "extra == 'smplerx'", specifier = ">=0.1.28" }, + { name = "timm", marker = "extra == 'smplerx'", specifier = ">=1.0" }, { name = "torch", marker = "extra == 'detrpose'", specifier = ">=2.4" }, { name = "torch", marker = "extra == 'multihmr'", specifier = ">=2.4" }, { name = "torch", marker = "extra == 'nlf'", specifier = ">=2.4" }, + { name = "torch", marker = "extra == 'smplerx'", specifier = ">=2.4" }, { name = "torchgeometry", marker = "extra == 'multihmr'", specifier = ">=0.1.2" }, { name = "torchvision", marker = "extra == 'detrpose'", specifier = ">=0.19" }, { name = "torchvision", marker = "extra == 'multihmr'", specifier = ">=0.19" }, { name = "torchvision", marker = "extra == 'nlf'", specifier = ">=0.19" }, + { name = "torchvision", marker = "extra == 'smplerx'", specifier = ">=0.19" }, { name = "tqdm", marker = "extra == 'multihmr'", specifier = ">=4.65" }, + { name = "tqdm", marker = "extra == 'smplerx'", specifier = ">=4.65" }, { name = "transformers", marker = "extra == 'detrpose'", specifier = ">=4.40" }, { name = "trimesh", marker = "extra == 'multihmr'", specifier = ">=4.4" }, { name = "ultralytics", marker = "extra == 'pose'", specifier = ">=8.3" }, + { name = "ultralytics", marker = "extra == 'smplerx'", specifier = ">=8.3" }, { name = "xtcocotools", marker = "extra == 'detrpose'", specifier = ">=1.14" }, + { name = "yacs", marker = "extra == 'smplerx'", specifier = ">=0.1.8" }, ] -provides-extras = ["pose", "detrpose", "nlf", "multihmr"] +provides-extras = ["pose", "detrpose", "lidar", "nlf", "multihmr", "smplerx"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.3" }] +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + [[package]] name = "cattrs" version = "26.1.0" @@ -381,6 +435,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "configargparse" +version = "1.7.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/0b/30328302903c55218ffc5199646d0e9d28348ff26c02ba77b2ffc58d294a/configargparse-1.7.5.tar.gz", hash = "sha256:e3f9a7bb6be34d66b2e3c4a2f58e3045f8dfae47b0dc039f87bcfaa0f193fb0f", size = 53548, upload-time = "2026-03-11T02:19:38.144Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/19/3ba5e1b0bcc7b91aeab6c258afd70e4907d220fed3972febe38feb40db30/configargparse-1.7.5-py3-none-any.whl", hash = "sha256:1e63fdffedf94da9cd435fc13a1cd24777e76879dd2343912c1f871d4ac8c592", size = 27692, upload-time = "2026-03-11T02:19:36.442Z" }, +] + [[package]] name = "contourpy" version = "1.3.3" @@ -604,6 +667,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/fa/d3c15189f7c52aaefbaea76fb012119b04b9013f4bf446cb4eb4c26c4e6b/cython-3.2.4-py3-none-any.whl", hash = "sha256:732fc93bc33ae4b14f6afaca663b916c2fdd5dcbfad7114e17fb2434eeaea45c", size = 1257078, upload-time = "2026-01-04T14:14:12.373Z" }, ] +[[package]] +name = "dash" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flask" }, + { name = "importlib-metadata" }, + { name = "nest-asyncio" }, + { name = "plotly" }, + { name = "requests" }, + { name = "retrying" }, + { name = "setuptools" }, + { name = "typing-extensions" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/da/a13ae3a6528bd51a6901461dbff4549c6009de203d6249a89b9a09ac5cfb/dash-4.1.0.tar.gz", hash = "sha256:17a92a87b0c1eacc025079a705e44e72cd4c5794629c0a2909942b611faeb595", size = 6927689, upload-time = "2026-03-23T20:39:47.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/00/10b1f8b3885fc4add1853e9603af15c593fa0be20d37c158c4d811e868dc/dash-4.1.0-py3-none-any.whl", hash = "sha256:1af9f302bc14061061012cdb129b7e370d3604b12a7f730b252ad8e4966f01f7", size = 7232489, upload-time = "2026-03-23T20:39:40.658Z" }, +] + [[package]] name = "einops" version = "0.8.2" @@ -613,6 +696,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193", size = 65638, upload-time = "2026-01-26T04:13:18.546Z" }, ] +[[package]] +name = "fastjsonschema" +version = "2.21.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/b5/23b216d9d985a956623b6bd12d4086b60f0059b27799f23016af04a74ea1/fastjsonschema-2.21.2.tar.gz", hash = "sha256:b1eb43748041c880796cd077f1a07c3d94e93ae84bba5ed36800a33554ae05de", size = 374130, upload-time = "2025-08-14T18:49:36.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, +] + [[package]] name = "filelock" version = "3.29.0" @@ -622,6 +714,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, ] +[[package]] +name = "flask" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blinker" }, + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "markupsafe" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/00/35d85dcce6c57fdc871f3867d465d780f302a175ea360f62533f12b27e2b/flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb", size = 759004, upload-time = "2026-02-19T05:00:57.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, +] + [[package]] name = "flatbuffers" version = "25.12.19" @@ -786,6 +895,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/3c/3f62dee257eb3d6b2c1ef2a09d36d9793c7111156a73b5654d2c2305e5ce/idna-3.14-py3-none-any.whl", hash = "sha256:e677eaf072e290f7b725f9acf0b3a2bd55f9fd6f7c70abe5f0e34823d0accf69", size = 72184, upload-time = "2026-05-10T20:32:14.295Z" }, ] +[[package]] +name = "importlib-metadata" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/3d/2d244233ac4f76e38533cfcb2991c9eb4c7bf688ae0a036d30725b8faafe/importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7", size = 27789, upload-time = "2026-03-20T06:42:55.665Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -806,6 +927,15 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/72/73/b3d451dfc523756cf177d3ebb0af76dc7751b341c60e2a21871be400ae29/iopath-0.1.10.tar.gz", hash = "sha256:3311c16a4d9137223e20f141655759933e1eda24f8bff166af834af3c645ef01", size = 42226, upload-time = "2022-07-09T19:00:50.866Z" } +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -818,6 +948,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "jupyter-core" +version = "5.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "platformdirs" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" }, +] + [[package]] name = "kiwisolver" version = "1.5.0" @@ -1103,6 +1282,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/b3/5c7fa594c731e8dafab9f1a46ab6cef670fa62dbbfb6248cc70e42ec6fc5/mediapipe-0.10.35-py3-none-win_arm64.whl", hash = "sha256:46255326a6213118aaa518a7aa25e35f93337e82677960cc2a945f117bff8444", size = 9992331, upload-time = "2026-04-27T17:45:36.193Z" }, ] +[[package]] +name = "mmcv-lite" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "addict" }, + { name = "mmengine" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyyaml" }, + { name = "regex", marker = "sys_platform == 'win32'" }, + { name = "yapf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/e7/075329ead4078d77b14e195f04b03162b99c5644d4186505113f62b7ca6c/mmcv-lite-2.2.0.tar.gz", hash = "sha256:62933ea165b2d9ad32e1b72ccd5ccba3cf71b5cd812c4c13c16cb2fcfc46a064", size = 479155, upload-time = "2024-04-24T14:24:38.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/bd/5d468c171f201d6169bec848d1116e95736854eec72307299bc68939fe45/mmcv_lite-2.2.0-py2.py3-none-any.whl", hash = "sha256:a24ee8dd3df7556dfced282dbfe8c3f87df6de2d4dcaf1207e83e9a2d58455a6", size = 732333, upload-time = "2024-04-24T14:24:28.555Z" }, +] + +[[package]] +name = "mmengine" +version = "0.10.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "addict" }, + { name = "matplotlib" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "pyyaml" }, + { name = "regex", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "termcolor" }, + { name = "yapf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/14/959360bbd8374e23fc1b720906999add16a3ac071a501636db12c5861ff5/mmengine-0.10.7.tar.gz", hash = "sha256:d20ffcc31127567e53dceff132612a87f0081de06cbb7ab2bdb7439125a69225", size = 378090, upload-time = "2025-03-04T12:23:09.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/8e/f98332248aad102511bea4ae19c0ddacd2f0a994f3ca4c82b7a369e0af8b/mmengine-0.10.7-py3-none-any.whl", hash = "sha256:262ac976a925562f78cd5fd14dd1bc9b680ed0aa81f0d85b723ef782f99c54ee", size = 452720, upload-time = "2025-03-04T12:23:06.339Z" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -1112,6 +1331,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] +[[package]] +name = "narwhals" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/0e/3ad61eb87088cc4932e0d851531fa82f845a6230b68b091a0e298cc7e537/narwhals-2.21.0.tar.gz", hash = "sha256:7c6e7f50528e62b7a967dd864d7e117d2955d38d4f730653ce46a9861358e2dc", size = 633083, upload-time = "2026-05-08T12:29:02.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/e1/68c2256b69a314eba133673377ba9118c356f6342a0c02b61de449cf2bf2/narwhals-2.21.0-py3-none-any.whl", hash = "sha256:1e6617d0fca68ae1fda29e5397c4eaacd3ffc9fffe6bcd6ded0c690475e853be", size = 451943, upload-time = "2026-05-08T12:29:01.058Z" }, +] + +[[package]] +name = "nbformat" +version = "5.10.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastjsonschema" }, + { name = "jsonschema" }, + { name = "jupyter-core" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, +] + +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + [[package]] name = "networkx" version = "3.6.1" @@ -1307,6 +1559,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e3/94/1843518e420fa3ed6919835845df698c7e27e183cb997394e4a670973a65/omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b", size = 79500, upload-time = "2022-12-08T20:59:19.686Z" }, ] +[[package]] +name = "open3d" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "addict" }, + { name = "configargparse" }, + { name = "dash" }, + { name = "flask" }, + { name = "matplotlib" }, + { name = "nbformat" }, + { name = "numpy" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, + { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, + { name = "pillow" }, + { name = "pyquaternion" }, + { name = "pyyaml" }, + { name = "scikit-learn" }, + { name = "tqdm" }, + { name = "werkzeug" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/37/8d1746fcb58c37a9bd868fdca9a36c25b3c277bd764b7146419d11d2a58d/open3d-0.19.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:117702467bfb1602e9ae0ee5e2c7bcf573ebcd227b36a26f9f08425b52c89929", size = 103098641, upload-time = "2025-01-08T07:26:12.371Z" }, + { url = "https://files.pythonhosted.org/packages/bc/50/339bae21d0078cc3d3735e8eaf493a353a17dcc95d76bcefaa8edcf723d3/open3d-0.19.0-cp311-cp311-manylinux_2_31_x86_64.whl", hash = "sha256:678017392f6cc64a19d83afeb5329ffe8196893de2432f4c258eaaa819421bb5", size = 447683616, upload-time = "2025-01-08T07:22:48.098Z" }, + { url = "https://files.pythonhosted.org/packages/a3/3c/358f1cc5b034dc6a785408b7aa7643e503229d890bcbc830cda9fce778b1/open3d-0.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:02091c309708f09da1167d2ea475e05d19f5e81dff025145f3afd9373cbba61f", size = 69151111, upload-time = "2025-01-08T07:27:22.662Z" }, + { url = "https://files.pythonhosted.org/packages/37/c5/286c605e087e72ad83eab130451ce13b768caa4374d926dc735edc20da5a/open3d-0.19.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:9e4a8d29443ba4c83010d199d56c96bf553dd970d3351692ab271759cbe2d7ac", size = 103202754, upload-time = "2025-01-08T07:26:27.169Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/3723e5ade77c234a1650db11cbe59fe25c4f5af6c224f8ea22ff088bb36a/open3d-0.19.0-cp312-cp312-manylinux_2_31_x86_64.whl", hash = "sha256:01e4590dc2209040292ebe509542fbf2bf869ea60bcd9be7a3fe77b65bad3192", size = 447665185, upload-time = "2025-01-08T07:23:39.769Z" }, + { url = "https://files.pythonhosted.org/packages/9f/c4/35a6e0a35aa72420e75dc28d54b24beaff79bcad150423e47c67d2ad8773/open3d-0.19.0-cp312-cp312-win_amd64.whl", hash = "sha256:665839837e1d3a62524804c31031462c3b548a2b6ed55214e6deb91522844f97", size = 69169961, upload-time = "2025-01-08T07:27:35.392Z" }, +] + [[package]] name = "opencv-contrib-python" version = "4.11.0.86" @@ -1350,6 +1632,136 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", +] +dependencies = [ + { name = "numpy", marker = "python_full_version >= '3.14'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.14'" }, + { name = "pytz", marker = "python_full_version >= '3.14'" }, + { name = "tzdata", marker = "python_full_version >= '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/fa/7ac648108144a095b4fb6aa3de1954689f7af60a14cf25583f4960ecb878/pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523", size = 11578790, upload-time = "2025-09-29T23:18:30.065Z" }, + { url = "https://files.pythonhosted.org/packages/9b/35/74442388c6cf008882d4d4bdfc4109be87e9b8b7ccd097ad1e7f006e2e95/pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45", size = 10833831, upload-time = "2025-09-29T23:38:56.071Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e4/de154cbfeee13383ad58d23017da99390b91d73f8c11856f2095e813201b/pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66", size = 12199267, upload-time = "2025-09-29T23:18:41.627Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c9/63f8d545568d9ab91476b1818b4741f521646cbdd151c6efebf40d6de6f7/pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b", size = 12789281, upload-time = "2025-09-29T23:18:56.834Z" }, + { url = "https://files.pythonhosted.org/packages/f2/00/a5ac8c7a0e67fd1a6059e40aa08fa1c52cc00709077d2300e210c3ce0322/pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791", size = 13240453, upload-time = "2025-09-29T23:19:09.247Z" }, + { url = "https://files.pythonhosted.org/packages/27/4d/5c23a5bc7bd209231618dd9e606ce076272c9bc4f12023a70e03a86b4067/pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151", size = 13890361, upload-time = "2025-09-29T23:19:25.342Z" }, + { url = "https://files.pythonhosted.org/packages/8e/59/712db1d7040520de7a4965df15b774348980e6df45c129b8c64d0dbe74ef/pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c", size = 11348702, upload-time = "2025-09-29T23:19:38.296Z" }, + { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" }, + { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, + { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, + { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, + { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4b/18b035ee18f97c1040d94debd8f2e737000ad70ccc8f5513f4eefad75f4b/pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713", size = 11544671, upload-time = "2025-09-29T23:21:05.024Z" }, + { url = "https://files.pythonhosted.org/packages/31/94/72fac03573102779920099bcac1c3b05975c2cb5f01eac609faf34bed1ca/pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8", size = 10680807, upload-time = "2025-09-29T23:21:15.979Z" }, + { url = "https://files.pythonhosted.org/packages/16/87/9472cf4a487d848476865321de18cc8c920b8cab98453ab79dbbc98db63a/pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d", size = 11709872, upload-time = "2025-09-29T23:21:27.165Z" }, + { url = "https://files.pythonhosted.org/packages/15/07/284f757f63f8a8d69ed4472bfd85122bd086e637bf4ed09de572d575a693/pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac", size = 12306371, upload-time = "2025-09-29T23:21:40.532Z" }, + { url = "https://files.pythonhosted.org/packages/33/81/a3afc88fca4aa925804a27d2676d22dcd2031c2ebe08aabd0ae55b9ff282/pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c", size = 12765333, upload-time = "2025-09-29T23:21:55.77Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0f/b4d4ae743a83742f1153464cf1a8ecfafc3ac59722a0b5c8602310cb7158/pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493", size = 13418120, upload-time = "2025-09-29T23:22:10.109Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c7/e54682c96a895d0c808453269e0b5928a07a127a15704fedb643e9b0a4c8/pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee", size = 10993991, upload-time = "2025-09-29T23:25:04.889Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ca/3f8d4f49740799189e1395812f3bf23b5e8fc7c190827d55a610da72ce55/pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5", size = 12048227, upload-time = "2025-09-29T23:22:24.343Z" }, + { url = "https://files.pythonhosted.org/packages/0e/5a/f43efec3e8c0cc92c4663ccad372dbdff72b60bdb56b2749f04aa1d07d7e/pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21", size = 11411056, upload-time = "2025-09-29T23:22:37.762Z" }, + { url = "https://files.pythonhosted.org/packages/46/b1/85331edfc591208c9d1a63a06baa67b21d332e63b7a591a5ba42a10bb507/pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78", size = 11645189, upload-time = "2025-09-29T23:22:51.688Z" }, + { url = "https://files.pythonhosted.org/packages/44/23/78d645adc35d94d1ac4f2a3c4112ab6f5b8999f4898b8cdf01252f8df4a9/pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110", size = 12121912, upload-time = "2025-09-29T23:23:05.042Z" }, + { url = "https://files.pythonhosted.org/packages/53/da/d10013df5e6aaef6b425aa0c32e1fc1f3e431e4bcabd420517dceadce354/pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86", size = 12712160, upload-time = "2025-09-29T23:23:28.57Z" }, + { url = "https://files.pythonhosted.org/packages/bd/17/e756653095a083d8a37cbd816cb87148debcfcd920129b25f99dd8d04271/pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc", size = 13199233, upload-time = "2025-09-29T23:24:24.876Z" }, + { url = "https://files.pythonhosted.org/packages/04/fd/74903979833db8390b73b3a8a7d30d146d710bd32703724dd9083950386f/pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0", size = 11540635, upload-time = "2025-09-29T23:25:52.486Z" }, + { url = "https://files.pythonhosted.org/packages/21/00/266d6b357ad5e6d3ad55093a7e8efc7dd245f5a842b584db9f30b0f0a287/pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593", size = 10759079, upload-time = "2025-09-29T23:26:33.204Z" }, + { url = "https://files.pythonhosted.org/packages/ca/05/d01ef80a7a3a12b2f8bbf16daba1e17c98a2f039cbc8e2f77a2c5a63d382/pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c", size = 11814049, upload-time = "2025-09-29T23:27:15.384Z" }, + { url = "https://files.pythonhosted.org/packages/15/b2/0e62f78c0c5ba7e3d2c5945a82456f4fac76c480940f805e0b97fcbc2f65/pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b", size = 12332638, upload-time = "2025-09-29T23:27:51.625Z" }, + { url = "https://files.pythonhosted.org/packages/c5/33/dd70400631b62b9b29c3c93d2feee1d0964dc2bae2e5ad7a6c73a7f25325/pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6", size = 12886834, upload-time = "2025-09-29T23:28:21.289Z" }, + { url = "https://files.pythonhosted.org/packages/d3/18/b5d48f55821228d0d2692b34fd5034bb185e854bdb592e9c640f6290e012/pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3", size = 13409925, upload-time = "2025-09-29T23:28:58.261Z" }, + { url = "https://files.pythonhosted.org/packages/a6/3d/124ac75fcd0ecc09b8fdccb0246ef65e35b012030defb0e0eba2cbbbe948/pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5", size = 11109071, upload-time = "2025-09-29T23:32:27.484Z" }, + { url = "https://files.pythonhosted.org/packages/89/9c/0e21c895c38a157e0faa1fb64587a9226d6dd46452cac4532d80c3c4a244/pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec", size = 12048504, upload-time = "2025-09-29T23:29:31.47Z" }, + { url = "https://files.pythonhosted.org/packages/d7/82/b69a1c95df796858777b68fbe6a81d37443a33319761d7c652ce77797475/pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7", size = 11410702, upload-time = "2025-09-29T23:29:54.591Z" }, + { url = "https://files.pythonhosted.org/packages/f9/88/702bde3ba0a94b8c73a0181e05144b10f13f29ebfc2150c3a79062a8195d/pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450", size = 11634535, upload-time = "2025-09-29T23:30:21.003Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" }, + { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" }, + { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and python_full_version < '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "python_full_version < '3.12' and sys_platform == 'darwin'", + "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", +] +dependencies = [ + { name = "numpy", marker = "python_full_version < '3.14'" }, + { name = "python-dateutil", marker = "python_full_version < '3.14'" }, + { name = "tzdata", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/16/b5c76b838fd9bf6ce84d3a53346b8874ec05c5f0040d75ef2c320100cd2a/pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98", size = 10338495, upload-time = "2026-05-11T18:52:11.558Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b0/a4ffc4ae74d2d822200dcc46898987d8eb6032d1e2b219cae39da6f5cbcc/pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639", size = 9938250, upload-time = "2026-05-11T18:52:17.005Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b2/3323601a52caee42c019e370090ca4544b241437240ca04f786cce82b0cf/pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2", size = 10770558, upload-time = "2026-05-11T18:52:19.865Z" }, + { url = "https://files.pythonhosted.org/packages/32/f1/bbecd2f867b97abebe0f9b53d750f862251b40337e061b36676ded3d920f/pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27", size = 11274611, upload-time = "2026-05-11T18:52:22.622Z" }, + { url = "https://files.pythonhosted.org/packages/7f/4f/eafabf2d5fae5adf143b4d18d3706c5efdc368a7c4eb1ee8a3eddabbd0f6/pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824", size = 11784670, upload-time = "2026-05-11T18:52:25.4Z" }, + { url = "https://files.pythonhosted.org/packages/49/44/1eb20389301b57b19cc099a1c2f662501f72f08a65f912d05822613c1532/pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938", size = 12353708, upload-time = "2026-05-11T18:52:28.139Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/c321f13b5ba1819fc8dca456c7fce578da2dcfecff1abbf0eaddf8406c0f/pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea", size = 9907609, upload-time = "2026-05-11T18:52:30.982Z" }, + { url = "https://files.pythonhosted.org/packages/53/85/1b7f563ebc6357c27233a02a96b589bcce1fa9c6eb89fb4f0e56421d277e/pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a", size = 9165596, upload-time = "2026-05-11T18:52:33.334Z" }, + { url = "https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09", size = 10357846, upload-time = "2026-05-11T18:52:36.143Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4", size = 9899550, upload-time = "2026-05-11T18:52:38.976Z" }, + { url = "https://files.pythonhosted.org/packages/31/a8/fa2535168fffcedf67f4f6de28d2dd903a747ca7c8ea6989451aaeb3a92f/pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c", size = 10412965, upload-time = "2026-05-11T18:52:41.915Z" }, + { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a4/2eb28f2fccb4ced4a2c79ab2a5dee9ade1ebf44922ebad6fea158c9f95d4/pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf", size = 11422824, upload-time = "2026-05-11T18:52:48.058Z" }, + { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c5/fc1b368f303087d20e8c9bf3d6ceb186263cfac0ade735cd938538bea839/pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc", size = 9755463, upload-time = "2026-05-11T18:52:53.386Z" }, + { url = "https://files.pythonhosted.org/packages/86/bd/fda8f9705b1b09c6ebe14bfc0fa0e4ec8584d54ea673628f157ff55131af/pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49", size = 9066158, upload-time = "2026-05-11T18:52:56.038Z" }, + { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, + { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, +] + [[package]] name = "pillow" version = "12.2.0" @@ -1437,6 +1849,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/60/5382c03e1970de634027cee8e1b7d39776b778b81812aaf45b694dfe9e28/pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e", size = 7080946, upload-time = "2026-04-01T14:46:11.734Z" }, ] +[[package]] +name = "platformdirs" +version = "4.9.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, +] + +[[package]] +name = "plotly" +version = "6.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "narwhals" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/7f/0f100df1172aadf88a929a9dbb902656b0880ba4b960fe5224867159d8f4/plotly-6.7.0.tar.gz", hash = "sha256:45eea0ff27e2a23ccd62776f77eb43aa1ca03df4192b76036e380bb479b892c6", size = 6911286, upload-time = "2026-04-09T20:36:45.738Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/ad/cba91b3bcf04073e4d1655a5c1710ef3f457f56f7d1b79dcc3d72f4dd912/plotly-6.7.0-py3-none-any.whl", hash = "sha256:ac8aca1c25c663a59b5b9140a549264a5badde2e057d79b8c772ae2920e32ff0", size = 9898444, upload-time = "2026-04-09T20:36:39.812Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -1750,6 +2184,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pyquaternion" +version = "0.9.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/3d092aa20efaedacb89c3221a92c6491be5b28f618a2c36b52b53e7446c2/pyquaternion-0.9.9.tar.gz", hash = "sha256:b1f61af219cb2fe966b5fb79a192124f2e63a3f7a777ac3cadf2957b1a81bea8", size = 15530, upload-time = "2020-10-05T01:31:30.327Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/b3/d8482e8cacc8ea15a356efea13d22ce1c5914a9ee36622ba250523240bf2/pyquaternion-0.9.9-py3-none-any.whl", hash = "sha256:e65f6e3f7b1fdf1a9e23f82434334a1ae84f14223eee835190cd2e841f8172ec", size = 14361, upload-time = "2020-10-05T01:31:37.575Z" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -1787,6 +2233,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/a2/ce26e437e36707f0da839dd88b072413825ea8986b73ce7b5c4f3cd1b7f4/python_osc-1.10.2-py3-none-any.whl", hash = "sha256:018b28e1cc06427c2c3d695f4e8d87d0caecfe604ff889acc45235cfd94183a2", size = 45467, upload-time = "2026-04-02T20:46:03.718Z" }, ] +[[package]] +name = "pytz" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, +] + [[package]] name = "pywin32" version = "311" @@ -1861,6 +2316,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "regex" version = "2026.5.9" @@ -1980,6 +2449,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/e6/e300fce5fe83c30520607a015dabd985df3251e188d234bfe9492e17a389/requests-2.34.0-py3-none-any.whl", hash = "sha256:917520a21b767485ce7c588f4ebb917c436b24a31231b44228715eaeb5a52c60", size = 73021, upload-time = "2026-05-11T19:29:49.923Z" }, ] +[[package]] +name = "retrying" +version = "1.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/5a/b17e1e257d3e6f2e7758930e1256832c9ddd576f8631781e6a072914befa/retrying-1.4.2.tar.gz", hash = "sha256:d102e75d53d8d30b88562d45361d6c6c934da06fab31bd81c0420acb97a8ba39", size = 11411, upload-time = "2025-08-03T03:35:25.189Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/f3/6cd296376653270ac1b423bb30bd70942d9916b6978c6f40472d6ac038e7/retrying-1.4.2-py3-none-any.whl", hash = "sha256:bbc004aeb542a74f3569aeddf42a2516efefcdaff90df0eb38fbfbf19f179f59", size = 10859, upload-time = "2025-08-03T03:35:23.829Z" }, +] + [[package]] name = "rich" version = "15.0.0" @@ -2002,6 +2480,114 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/bd/dd56edc2680df7658c7bdeeb58270ab35dfcdc0e2670899ce9771604b436/roma-1.5.6-py3-none-any.whl", hash = "sha256:ae21a4e095330428d60e1dbdcad4e550dbdc9a7b4597a0157aaf5d6fd2b46afb", size = 25327, upload-time = "2026-02-11T12:55:10.629Z" }, ] +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, + { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676, upload-time = "2025-11-30T20:21:55.475Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938, upload-time = "2025-11-30T20:21:57.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932, upload-time = "2025-11-30T20:21:58.47Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830, upload-time = "2025-11-30T20:21:59.699Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033, upload-time = "2025-11-30T20:22:00.991Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828, upload-time = "2025-11-30T20:22:02.723Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683, upload-time = "2025-11-30T20:22:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583, upload-time = "2025-11-30T20:22:05.814Z" }, + { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496, upload-time = "2025-11-30T20:22:07.713Z" }, + { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669, upload-time = "2025-11-30T20:22:09.312Z" }, + { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011, upload-time = "2025-11-30T20:22:11.309Z" }, + { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406, upload-time = "2025-11-30T20:22:13.101Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024, upload-time = "2025-11-30T20:22:14.853Z" }, + { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069, upload-time = "2025-11-30T20:22:16.577Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, + { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, + { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, + { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, + { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292, upload-time = "2025-11-30T20:24:16.537Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128, upload-time = "2025-11-30T20:24:18.086Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542, upload-time = "2025-11-30T20:24:20.092Z" }, + { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004, upload-time = "2025-11-30T20:24:22.231Z" }, + { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063, upload-time = "2025-11-30T20:24:24.302Z" }, + { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099, upload-time = "2025-11-30T20:24:25.916Z" }, + { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177, upload-time = "2025-11-30T20:24:27.834Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015, upload-time = "2025-11-30T20:24:29.457Z" }, + { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736, upload-time = "2025-11-30T20:24:31.22Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981, upload-time = "2025-11-30T20:24:32.934Z" }, + { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782, upload-time = "2025-11-30T20:24:35.169Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, +] + [[package]] name = "safetensors" version = "0.7.0" @@ -2024,6 +2610,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755", size = 341380, upload-time = "2025-11-19T15:18:44.427Z" }, ] +[[package]] +name = "scikit-learn" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/92/53ea2181da8ac6bf27170191028aee7251f8f841f8d3edbfdcaf2008fde9/scikit_learn-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:146b4d36f800c013d267b29168813f7a03a43ecd2895d04861f1240b564421da", size = 8595835, upload-time = "2025-12-10T07:07:39.385Z" }, + { url = "https://files.pythonhosted.org/packages/01/18/d154dc1638803adf987910cdd07097d9c526663a55666a97c124d09fb96a/scikit_learn-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f984ca4b14914e6b4094c5d52a32ea16b49832c03bd17a110f004db3c223e8e1", size = 8080381, upload-time = "2025-12-10T07:07:41.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/44/226142fcb7b7101e64fdee5f49dbe6288d4c7af8abf593237b70fca080a4/scikit_learn-1.8.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e30adb87f0cc81c7690a84f7932dd66be5bac57cfe16b91cb9151683a4a2d3b", size = 8799632, upload-time = "2025-12-10T07:07:43.899Z" }, + { url = "https://files.pythonhosted.org/packages/36/4d/4a67f30778a45d542bbea5db2dbfa1e9e100bf9ba64aefe34215ba9f11f6/scikit_learn-1.8.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ada8121bcb4dac28d930febc791a69f7cb1673c8495e5eee274190b73a4559c1", size = 9103788, upload-time = "2025-12-10T07:07:45.982Z" }, + { url = "https://files.pythonhosted.org/packages/89/3c/45c352094cfa60050bcbb967b1faf246b22e93cb459f2f907b600f2ceda5/scikit_learn-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:c57b1b610bd1f40ba43970e11ce62821c2e6569e4d74023db19c6b26f246cb3b", size = 8081706, upload-time = "2025-12-10T07:07:48.111Z" }, + { url = "https://files.pythonhosted.org/packages/3d/46/5416595bb395757f754feb20c3d776553a386b661658fb21b7c814e89efe/scikit_learn-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:2838551e011a64e3053ad7618dda9310175f7515f1742fa2d756f7c874c05961", size = 7688451, upload-time = "2025-12-10T07:07:49.873Z" }, + { url = "https://files.pythonhosted.org/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" }, + { url = "https://files.pythonhosted.org/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" }, + { url = "https://files.pythonhosted.org/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" }, + { url = "https://files.pythonhosted.org/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" }, + { url = "https://files.pythonhosted.org/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" }, + { url = "https://files.pythonhosted.org/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" }, + { url = "https://files.pythonhosted.org/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" }, + { url = "https://files.pythonhosted.org/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" }, + { url = "https://files.pythonhosted.org/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" }, + { url = "https://files.pythonhosted.org/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" }, + { url = "https://files.pythonhosted.org/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" }, + { url = "https://files.pythonhosted.org/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" }, + { url = "https://files.pythonhosted.org/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" }, + { url = "https://files.pythonhosted.org/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" }, + { url = "https://files.pythonhosted.org/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" }, + { url = "https://files.pythonhosted.org/packages/24/05/1af2c186174cc92dcab2233f327336058c077d38f6fe2aceb08e6ab4d509/scikit_learn-1.8.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c22a2da7a198c28dd1a6e1136f19c830beab7fdca5b3e5c8bba8394f8a5c45b3", size = 8528667, upload-time = "2025-12-10T07:08:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a8/25/01c0af38fe969473fb292bba9dc2b8f9b451f3112ff242c647fee3d0dfe7/scikit_learn-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:6b595b07a03069a2b1740dc08c2299993850ea81cce4fe19b2421e0c970de6b7", size = 8066524, upload-time = "2025-12-10T07:08:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/be/ce/a0623350aa0b68647333940ee46fe45086c6060ec604874e38e9ab7d8e6c/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:29ffc74089f3d5e87dfca4c2c8450f88bdc61b0fc6ed5d267f3988f19a1309f6", size = 8657133, upload-time = "2025-12-10T07:08:31.865Z" }, + { url = "https://files.pythonhosted.org/packages/b8/cb/861b41341d6f1245e6ca80b1c1a8c4dfce43255b03df034429089ca2a2c5/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb65db5d7531bccf3a4f6bec3462223bea71384e2cda41da0f10b7c292b9e7c4", size = 8923223, upload-time = "2025-12-10T07:08:34.166Z" }, + { url = "https://files.pythonhosted.org/packages/76/18/a8def8f91b18cd1ba6e05dbe02540168cb24d47e8dcf69e8d00b7da42a08/scikit_learn-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:56079a99c20d230e873ea40753102102734c5953366972a71d5cb39a32bc40c6", size = 8096518, upload-time = "2025-12-10T07:08:36.339Z" }, + { url = "https://files.pythonhosted.org/packages/d1/77/482076a678458307f0deb44e29891d6022617b2a64c840c725495bee343f/scikit_learn-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:3bad7565bc9cf37ce19a7c0d107742b320c1285df7aab1a6e2d28780df167242", size = 7754546, upload-time = "2025-12-10T07:08:38.128Z" }, + { url = "https://files.pythonhosted.org/packages/2d/d1/ef294ca754826daa043b2a104e59960abfab4cf653891037d19dd5b6f3cf/scikit_learn-1.8.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4511be56637e46c25721e83d1a9cea9614e7badc7040c4d573d75fbe257d6fd7", size = 8848305, upload-time = "2025-12-10T07:08:41.013Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e2/b1f8b05138ee813b8e1a4149f2f0d289547e60851fd1bb268886915adbda/scikit_learn-1.8.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:a69525355a641bf8ef136a7fa447672fb54fe8d60cab5538d9eb7c6438543fb9", size = 8432257, upload-time = "2025-12-10T07:08:42.873Z" }, + { url = "https://files.pythonhosted.org/packages/26/11/c32b2138a85dcb0c99f6afd13a70a951bfdff8a6ab42d8160522542fb647/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2656924ec73e5939c76ac4c8b026fc203b83d8900362eb2599d8aee80e4880f", size = 8678673, upload-time = "2025-12-10T07:08:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/c7/57/51f2384575bdec454f4fe4e7a919d696c9ebce914590abf3e52d47607ab8/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15fc3b5d19cc2be65404786857f2e13c70c83dd4782676dd6814e3b89dc8f5b9", size = 8922467, upload-time = "2025-12-10T07:08:47.408Z" }, + { url = "https://files.pythonhosted.org/packages/35/4d/748c9e2872637a57981a04adc038dacaa16ba8ca887b23e34953f0b3f742/scikit_learn-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:00d6f1d66fbcf4eba6e356e1420d33cc06c70a45bb1363cd6f6a8e4ebbbdece2", size = 8774395, upload-time = "2025-12-10T07:08:49.337Z" }, + { url = "https://files.pythonhosted.org/packages/60/22/d7b2ebe4704a5e50790ba089d5c2ae308ab6bb852719e6c3bd4f04c3a363/scikit_learn-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f28dd15c6bb0b66ba09728cf09fd8736c304be29409bd8445a080c1280619e8c", size = 8002647, upload-time = "2025-12-10T07:08:51.601Z" }, +] + [[package]] name = "scipy" version = "1.17.1" @@ -2163,6 +2799,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] +[[package]] +name = "termcolor" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, +] + +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + +[[package]] +name = "timm" +version = "1.0.27" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "pyyaml" }, + { name = "safetensors" }, + { name = "torch" }, + { name = "torchvision" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/54/ece85b0eef3700c90db8271a43669b05a0ebbe2edb1962329c34374a297e/timm-1.0.27.tar.gz", hash = "sha256:315dfe63186ca9fb7ff941268941231fd5be259f2b4bb4afa28560ae1015cb9a", size = 2439861, upload-time = "2026-05-08T19:38:36.844Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/2e/26bab7686ff4aed48f8f5f6c23e2aa37b7a37ddd9effe3aa61e908fd518f/timm-1.0.27-py3-none-any.whl", hash = "sha256:5ff07c9ddf53cbada88eab1c93ff175c64cab683b5a2fddf863bcee985926f89", size = 2589280, upload-time = "2026-05-08T19:38:35.034Z" }, +] + [[package]] name = "tokenizers" version = "0.22.2" @@ -2296,6 +2966,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "traitlets" +version = "5.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/22/40f55b26baeab80c2d7b3f1db0682f8954e4617fee7d90ce634022ef05c6/traitlets-5.15.0.tar.gz", hash = "sha256:4fead733f81cf1c4c938e06f8ca4633896833c9d89eff878159457f4d4392971", size = 163197, upload-time = "2026-05-06T08:05:58.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/98/a9937a969d018a23badfea0b381f66783649d48e0ea6c41923265c3cbeb3/traitlets-5.15.0-py3-none-any.whl", hash = "sha256:fb36a18867a6803deab09f3c5e0fa81bb7b26a5c9e82501c9933f759166eff40", size = 85877, upload-time = "2026-05-06T08:05:55.853Z" }, +] + [[package]] name = "transformers" version = "5.8.0" @@ -2371,6 +3050,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "tzdata" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, +] + [[package]] name = "ultralytics" version = "8.4.49" @@ -2416,6 +3104,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] +[[package]] +name = "werkzeug" +version = "3.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/b2/381be8cfdee792dd117872481b6e378f85c957dd7c5bca38897b08f765fd/werkzeug-3.1.8.tar.gz", hash = "sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44", size = 875852, upload-time = "2026-04-02T18:49:14.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, +] + [[package]] name = "xtcocotools" version = "1.14.3" @@ -2432,3 +3132,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/e0/01cf7f8b3f4229568b37de680d0eaadc651d5ee36bd483b83658f49dc6c2/xtcocotools-1.14.3-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:126ca596229b2016552bf27cad01f3a2f70a3ff7576a58305a00499cb9e0057d", size = 464351, upload-time = "2023-10-19T07:52:33.199Z" }, { url = "https://files.pythonhosted.org/packages/a9/10/32bef0fcd29145dcda9bfaa9e11718f40acd444d6804cac870b0437fc7a8/xtcocotools-1.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:47cb5433903f30589343d54530e49abd6b61d0fd119857ba4948b8ce291dbee6", size = 88741, upload-time = "2023-10-19T07:53:51.151Z" }, ] + +[[package]] +name = "yacs" +version = "0.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/3e/4a45cb0738da6565f134c01d82ba291c746551b5bc82e781ec876eb20909/yacs-0.1.8.tar.gz", hash = "sha256:efc4c732942b3103bea904ee89af98bcd27d01f0ac12d8d4d369f1e7a2914384", size = 11100, upload-time = "2020-08-10T16:37:47.755Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/4f/fe9a4d472aa867878ce3bb7efb16654c5d63672b86dc0e6e953a67018433/yacs-0.1.8-py3-none-any.whl", hash = "sha256:99f893e30497a4b66842821bac316386f7bd5c4f47ad35c9073ef089aa33af32", size = 14747, upload-time = "2020-08-10T16:37:46.4Z" }, +] + +[[package]] +name = "yapf" +version = "0.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907, upload-time = "2024-11-14T00:11:41.584Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/81/6acd6601f61e31cfb8729d3da6d5df966f80f374b78eff83760714487338/yapf-0.43.0-py3-none-any.whl", hash = "sha256:224faffbc39c428cb095818cf6ef5511fdab6f7430a10783fdfb292ccf2852ca", size = 256158, upload-time = "2024-11-14T00:11:39.37Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/21/093488dfc7cc8964ded15ab726fad40f25fd3d788fd741cc1c5a17d78ee8/zipp-3.23.1.tar.gz", hash = "sha256:32120e378d32cd9714ad503c1d024619063ec28aad2248dc6672ad13edfa5110", size = 25965, upload-time = "2026-04-13T23:21:46.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc", size = 10378, upload-time = "2026-04-13T23:21:45.386Z" }, +] diff --git a/docs/ICP_FUSION.md b/docs/ICP_FUSION.md new file mode 100644 index 0000000..6ec9dac --- /dev/null +++ b/docs/ICP_FUSION.md @@ -0,0 +1,69 @@ +# ICP LiDAR ↔ SMPL-X Dense Fusion + +Refines Multi-HMR SMPL-X meshes using live iPhone LiDAR via point-to-plane ICP. + +## Env vars + +| Var | Default | Effect | +|-----|---------|--------| +| `ICP_FUSION` | `0` | `1` enables LiDAR receiver + FusionWorker | +| `ICP_LIDAR_HOST` | _(required when on)_ | iPhone ARBodyTracker IP on the LAN | +| `ICP_LIDAR_PORT` | `5500` | TCP port the iOS app publishes ARMesh on | +| `ICP_LIDAR_EXTRINSIC` | `~/.config/av-live/lidar_extrinsic.json` | Path to persisted extrinsic JSON | + +## Relation to ARKit joint fusion + +ICP LiDAR fusion is **mesh-level** and complementary to the existing **joint-level** ARKit fusion (`iphone_osc_listener.py` + `pose_filter.py::ArkitFuse` + `multi_hmr_worker.arkit_pelvis_z_override`). The two run independently: + +- **ARKit joints** (OSC :57128) — sparse (14 mapped joints), 60 Hz, fast, used to override MediaPipe pose joint slots and lock Multi-HMR pelvis Z. +- **ICP LiDAR mesh** (TCP :5500) — dense (~thousand points), 5–10 Hz, used to register Multi-HMR SMPL-X vertices onto the real-world geometry captured by the iPhone LiDAR. + +They can be enabled together or separately. ICP runs only when `ICP_FUSION=1`. + +## Calibration + +1. Launch the iPhone ARBodyTracker app and note its LAN IP. +2. From `data_only_viz/`: + ```bash + uv run --extra lidar python -m data_only_viz.scripts.calibrate_lidar \ + --lidar-host --lidar-port 5500 --webcam-index 0 + ``` +3. The script asks for 4 stances (front / left / right / back). Hold still each time and press ENTER. +4. The estimated extrinsic is written to `ICP_LIDAR_EXTRINSIC` (or the default path). Re-run any time the camera or iPhone moves. + +> NOTE — as of the initial ICP MVP (Task 9), `multi_hmr_worker.predict_once` is a stub raising `NotImplementedError`. The calibration CLI runs the LiDAR reader and 4-stance loop scaffold but cannot capture the webcam pelvis side until a follow-up wires `predict_once` to the existing inference path. Track this in the next planning round. + +## Runtime + +```bash +ICP_FUSION=1 ICP_LIDAR_HOST=192.168.0.42 \ + uv run --extra lidar python -m data_only_viz.main +``` + +## Architecture (summary) + +``` +iPhone ARBodyTracker app + ├── OSC :57128 /body3d/kp → IphoneOSCListener (ARKit joint fusion) + └── TCP :5500 ARMeshAnchors → LidarTCPReader (ICP mesh fusion) + ↓ + FusionWorker.run_once(state) + ↓ + state.persons_smplx[*].vertices_3d + (replaced in place when ICP accepts) +``` + +ICP fusion runs in its own daemon thread (`IcpFusionThread`, target 8 Hz). It is opt-in (off by default) and a no-op if the LiDAR stream is absent. + +## Troubleshooting + +- **`open3d` missing** → `cd data_only_viz && uv sync --extra lidar` +- **No LiDAR frames** → check that the iPhone app is publishing on the expected port and that nothing else is bound to it. `nc -l 5500` from the Mac should not succeed while the app runs. +- **ICP always rejected (`fitness < 0.30`)** → the extrinsic is likely stale; re-run calibration. Verify the iPhone is facing the same scene as the webcam. +- **Mesh appears scaled wrong** → SMPL-X is in metres; the iPhone publishes metres. If you see a factor-1000 mismatch the iOS encoder is sending millimetres — patch the iOS app, not this code. +- **Bench shows `latency_ms_p95 > 100`** → reduce `IcpConfig.voxel_size_m` (e.g. 0.03 m) or `max_iterations` (e.g. 20). +- **Python `cp314` wheel failure on `uv sync --extra lidar`** → open3d does not ship cp313+ wheels yet. Use Python 3.12 (`uv venv --python 3.12`). + +## Implementation note (Task 6 deviation) + +`register_mesh_to_lidar` uses a two-stage coarse-to-fine ICP internally: a warm-start pass at `max(0.25 m, 5× threshold)` correspondence, then a strict pass at `IcpConfig.max_correspondence_m` (default 0.05 m). The accept/reject gate (`fitness ≥ 0.30`, `rmse ≤ 0.05 m`) is evaluated **only on the strict pass** so the contract is preserved. The warm-start makes ICP converge reliably when Multi-HMR's initial mesh sits more than 5 cm off the LiDAR surface (typical with a fresh calibration or pose change). diff --git a/docs/superpowers/plans/2026-05-14-icp-lidar-smplx-fusion.md b/docs/superpowers/plans/2026-05-14-icp-lidar-smplx-fusion.md new file mode 100644 index 0000000..049907a --- /dev/null +++ b/docs/superpowers/plans/2026-05-14-icp-lidar-smplx-fusion.md @@ -0,0 +1,1631 @@ +# ICP LiDAR ↔ SMPL-X Dense Fusion — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Refine Multi-HMR SMPL-X vertex output (per-person, 10475 verts) by registering it onto the live LiDAR point cloud streamed from the iPhone ARBodyTracker app, producing depth-corrected, scale-true 3D meshes for the AVLiveBody renderer. + +**Architecture:** A new Python worker (`icp_fusion.py`) ingests two streams: (1) `state.persons_smplx[*].vertices_3d` from `multi_hmr_worker.py`, (2) LiDAR point clouds via a new TCP receiver `lidar_receiver.py` connected to the iPhone ARBodyTracker app's existing ARMeshAnchor publisher. ICP (point-to-plane variant, Open3D) aligns each SMPL-X mesh to the cropped LiDAR neighborhood, gated by a NaN/divergence guard. Fused vertices replace `vertices_3d` in `state.persons_smplx` when the env var `ICP_FUSION=1`. AVLiveBody consumes the unchanged TCP mesh schema — no Swift changes required. + +**Tech Stack:** Python 3.11+ via `uv`, Open3D ≥ 0.18 (ICP, voxel downsample, KDTree), NumPy 1.26, existing project deps (python-osc, scipy). iPhone side already streams ARMeshAnchor via TCP per memory `project_iphone_arbodytracker` (no iOS changes in this plan). + +**Out of scope:** iOS app modifications, retraining Multi-HMR, multi-camera ICP across multiple iPhones, GPU-accelerated ICP (CPU is sufficient at LiDAR 5–10 Hz). + +--- + +## File Structure + +**Create:** +- `data_only_viz/icp_fusion.py` — ICP wrapper, per-person registration, divergence guard (~250 lines) +- `data_only_viz/lidar_receiver.py` — TCP client for iPhone ARMesh stream, decoder, ring buffer (~180 lines) +- `data_only_viz/lidar_calib.py` — One-shot extrinsic calibration helper (chessboard or pose-anchored), persisted to `~/.config/av-live/lidar_extrinsic.json` (~120 lines) +- `data_only_viz/tests/test_icp_fusion.py` — synthetic SMPL-X + perturbed point cloud, convergence, NaN guard (~200 lines) +- `data_only_viz/tests/test_lidar_receiver.py` — TCP decoder unit tests + roundtrip fixture (~120 lines) +- `data_only_viz/tests/test_lidar_calib.py` — extrinsic estimation correctness, persistence (~80 lines) +- `data_only_viz/scripts/bench_icp_fusion.py` — end-to-end latency / convergence bench (~120 lines) +- `data_only_viz/scripts/calibrate_lidar.py` — CLI entry for one-shot extrinsic capture (~80 lines) +- `docs/ICP_FUSION.md` — env vars, calibration procedure, troubleshooting (~150 lines) + +**Modify:** +- `data_only_viz/pyproject.toml` — add `open3d>=0.18` under new `lidar` optional-dep group +- `data_only_viz/state.py` — add `lidar_points: np.ndarray | None` and `icp_metadata` to `State` (~10 lines) +- `data_only_viz/main.py` — wire `lidar_receiver` and `icp_fusion` workers behind `ICP_FUSION=1` env (~40 lines) +- `data_only_viz/multi_hmr_worker.py` — emit a `_pre_icp` snapshot for the bench harness (~5 lines) +- `CLAUDE.md` (root) — document new env vars in the RC0.1+ table (~6 lines) + +**No changes to:** `launcher/AV-Live-Body/`, `oscope-of/`, `sound_algo/`, `web_realart/`. The TCP mesh schema already used between Python worker and AVLiveBody is preserved bit-for-bit; ICP simply substitutes the contents of `vertices_3d` upstream. + +--- + +## Task 1: Dependency scaffold (Open3D + optional-dep group) + +**Files:** +- Modify: `data_only_viz/pyproject.toml` +- Create: `data_only_viz/tests/test_open3d_smoke.py` + +- [ ] **Step 1: Add `lidar` optional-dep group** + +In `data_only_viz/pyproject.toml`, append after the `detrpose` block: + +```toml +# Open3D for ICP fusion between iPhone LiDAR and Multi-HMR SMPL-X meshes. +# CPU-only is sufficient at 5-10 Hz LiDAR cadence. +lidar = [ + "open3d>=0.18,<0.20", +] +``` + +- [ ] **Step 2: Install the extra** + +Run: `cd data_only_viz && uv sync --extra lidar` +Expected: `Resolved N packages` with `open3d` present in `uv.lock`. + +- [ ] **Step 3: Write the smoke test** + +Create `data_only_viz/tests/test_open3d_smoke.py`: + +```python +"""Smoke test for the Open3D dependency used by ICP fusion.""" +from __future__ import annotations + +import numpy as np +import pytest + +open3d = pytest.importorskip("open3d") + + +def test_open3d_pointcloud_roundtrip() -> None: + pts = np.random.RandomState(0).randn(100, 3).astype(np.float32) + pcd = open3d.geometry.PointCloud() + pcd.points = open3d.utility.Vector3dVector(pts) + out = np.asarray(pcd.points) + assert out.shape == (100, 3) + np.testing.assert_allclose(out, pts, atol=1e-5) + + +def test_open3d_icp_converges_on_translated_copy() -> None: + rng = np.random.RandomState(1) + src = rng.randn(500, 3).astype(np.float64) + translation = np.array([0.10, -0.05, 0.20]) + tgt = src + translation + + src_pcd = open3d.geometry.PointCloud() + src_pcd.points = open3d.utility.Vector3dVector(src) + tgt_pcd = open3d.geometry.PointCloud() + tgt_pcd.points = open3d.utility.Vector3dVector(tgt) + + result = open3d.pipelines.registration.registration_icp( + src_pcd, tgt_pcd, max_correspondence_distance=0.5, + init=np.eye(4), + estimation_method=open3d.pipelines.registration.TransformationEstimationPointToPoint(), + ) + np.testing.assert_allclose(result.transformation[:3, 3], translation, atol=1e-3) +``` + +- [ ] **Step 4: Run the smoke test** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_open3d_smoke.py -v` +Expected: 2 passed. + +- [ ] **Step 5: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/pyproject.toml data_only_viz/uv.lock data_only_viz/tests/test_open3d_smoke.py +git commit -m "deps(icp): add open3d optional extra + smoke test" +``` + +--- + +## Task 2: LiDAR TCP receiver — frame decoder + +**Files:** +- Create: `data_only_viz/lidar_receiver.py` +- Create: `data_only_viz/tests/test_lidar_receiver.py` + +The iPhone ARBodyTracker app's existing TCP mesh publisher sends ARMeshAnchor data as a length-prefixed binary frame: `[uint32 BE frame_size][uint64 BE timestamp_ns][uint32 BE vertex_count][float32 LE x y z]*vertex_count`. This task only handles the decoder — the socket reader comes in Task 3. + +- [ ] **Step 1: Write the failing decoder test** + +Create `data_only_viz/tests/test_lidar_receiver.py`: + +```python +"""Unit tests for the iPhone LiDAR TCP frame decoder.""" +from __future__ import annotations + +import struct + +import numpy as np +import pytest + + +def _encode_frame(points: np.ndarray, timestamp_ns: int) -> bytes: + """Mimic the iPhone-side encoder for round-trip testing.""" + n = points.shape[0] + body = struct.pack(">Q", timestamp_ns) + struct.pack(">I", n) + points.astype("I", len(body)) + return header + body + + +def test_decode_lidar_frame_roundtrip() -> None: + from data_only_viz.lidar_receiver import LidarFrame, decode_frame + + pts = np.array([[0.1, 0.2, 0.3], [-1.0, 2.0, 5.5]], dtype=np.float32) + payload = _encode_frame(pts, timestamp_ns=1_700_000_000_000_000_000) + + # decode_frame is given the body (everything past the 4-byte length prefix). + body = payload[4:] + frame = decode_frame(body) + + assert isinstance(frame, LidarFrame) + assert frame.timestamp_ns == 1_700_000_000_000_000_000 + np.testing.assert_allclose(frame.points, pts, atol=1e-6) + + +def test_decode_lidar_frame_rejects_truncated() -> None: + from data_only_viz.lidar_receiver import decode_frame + + pts = np.array([[1.0, 2.0, 3.0]], dtype=np.float32) + body = ( + struct.pack(">Q", 0) + + struct.pack(">I", 1) + + pts.astype(" None: + from data_only_viz.lidar_receiver import decode_frame + + body = struct.pack(">Q", 0) + struct.pack(">I", 0) + with pytest.raises(ValueError, match="vertex_count"): + decode_frame(body) +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_lidar_receiver.py -v` +Expected: 3 failed with `ModuleNotFoundError: data_only_viz.lidar_receiver`. + +- [ ] **Step 3: Implement the decoder** + +Create `data_only_viz/lidar_receiver.py`: + +```python +"""TCP receiver for iPhone ARBodyTracker LiDAR ARMeshAnchor stream. + +Wire format (per frame, after the 4-byte big-endian length prefix consumed +by the socket reader): + + [uint64 BE timestamp_ns] + [uint32 BE vertex_count] + [float32 LE x y z] * vertex_count + +The decoder is pure and side-effect-free so it can be unit-tested without a +socket. The socket reader lives in a separate class (LidarTCPReader) so its +threading model is independently testable. +""" +from __future__ import annotations + +import struct +from dataclasses import dataclass + +import numpy as np + +_HEADER = struct.Struct(">QI") # timestamp_ns, vertex_count + + +@dataclass(frozen=True) +class LidarFrame: + """One decoded LiDAR frame from the iPhone.""" + + timestamp_ns: int + points: np.ndarray # shape (N, 3), float32, ARKit world frame (meters) + + +def decode_frame(body: bytes) -> LidarFrame: + """Decode a frame body (length prefix already stripped).""" + if len(body) < _HEADER.size: + raise ValueError(f"truncated frame: header needs {_HEADER.size} bytes, got {len(body)}") + timestamp_ns, vertex_count = _HEADER.unpack_from(body, 0) + if vertex_count == 0: + raise ValueError("vertex_count must be > 0") + expected = _HEADER.size + vertex_count * 12 + if len(body) < expected: + raise ValueError(f"truncated frame: need {expected} bytes for {vertex_count} verts, got {len(body)}") + raw = body[_HEADER.size : expected] + pts = np.frombuffer(raw, dtype=" None: + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(("127.0.0.1", port)) + srv.listen(1) + conn, _ = srv.accept() + conn.sendall(frame_bytes) + time.sleep(0.1) + conn.close() + srv.close() + + +def test_reader_grabs_latest_frame(unused_tcp_port: int) -> None: + from data_only_viz.lidar_receiver import LidarTCPReader + + pts = np.array([[1.0, 2.0, 3.0]], dtype=np.float32) + frame = _encode_frame(pts, timestamp_ns=42) + t = threading.Thread(target=_serve_one_frame, args=(unused_tcp_port, frame), daemon=True) + t.start() + time.sleep(0.05) + + reader = LidarTCPReader(host="127.0.0.1", port=unused_tcp_port, connect_timeout_s=2.0) + reader.start() + deadline = time.monotonic() + 2.0 + latest = None + while time.monotonic() < deadline: + latest = reader.latest() + if latest is not None: + break + time.sleep(0.02) + reader.stop() + t.join(timeout=1.0) + + assert latest is not None + assert latest.timestamp_ns == 42 + np.testing.assert_allclose(latest.points, pts, atol=1e-6) + + +def test_reader_returns_none_before_first_frame(unused_tcp_port: int) -> None: + from data_only_viz.lidar_receiver import LidarTCPReader + + reader = LidarTCPReader(host="127.0.0.1", port=unused_tcp_port, connect_timeout_s=0.05) + # Do not start it; latest() must be None. + assert reader.latest() is None +``` + +(The `unused_tcp_port` fixture comes from `pytest-asyncio` / `pytest-tcp-port`. If unavailable, use `socket.socket().bind(("", 0))` to grab a free port — adjust accordingly.) + +- [ ] **Step 2: Add `pytest-tcp-port` if missing** + +Run: `cd data_only_viz && uv add --dev pytest-tcp-port` +If declined or unavailable, manually pin a port via a helper and remove the fixture. + +- [ ] **Step 3: Run to verify failure** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_lidar_receiver.py -v` +Expected: 2 new tests fail with `ImportError: cannot import name 'LidarTCPReader'`. + +- [ ] **Step 4: Implement the reader** + +Append to `data_only_viz/lidar_receiver.py`: + +```python +import logging +import socket +import struct +import threading +from typing import Optional + +_LOG = logging.getLogger(__name__) +_LEN_PREFIX = struct.Struct(">I") + + +class LidarTCPReader: + """Background TCP reader producing a single-slot latest-frame mailbox. + + Reconnects on transient failures with linear backoff up to 5s. + """ + + def __init__(self, host: str, port: int, connect_timeout_s: float = 2.0) -> None: + self._host = host + self._port = port + self._connect_timeout_s = connect_timeout_s + self._stop = threading.Event() + self._lock = threading.Lock() + self._latest: Optional[LidarFrame] = None + self._thread: Optional[threading.Thread] = None + + def start(self) -> None: + if self._thread is not None: + return + self._thread = threading.Thread(target=self._run, name="lidar-tcp", daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + def latest(self) -> Optional[LidarFrame]: + with self._lock: + return self._latest + + def _run(self) -> None: + backoff_s = 0.5 + while not self._stop.is_set(): + try: + with socket.create_connection((self._host, self._port), timeout=self._connect_timeout_s) as sock: + sock.settimeout(1.0) + backoff_s = 0.5 + self._read_loop(sock) + except (OSError, ValueError) as exc: + _LOG.warning("lidar reader: %s; reconnecting in %.1fs", exc, backoff_s) + if self._stop.wait(backoff_s): + return + backoff_s = min(backoff_s * 2.0, 5.0) + + def _read_loop(self, sock: socket.socket) -> None: + while not self._stop.is_set(): + header = self._recv_exact(sock, _LEN_PREFIX.size) + if header is None: + return + (length,) = _LEN_PREFIX.unpack(header) + if length <= 0 or length > 8_000_000: # sanity cap: 8 MB per frame + raise ValueError(f"implausible frame length {length}") + body = self._recv_exact(sock, length) + if body is None: + return + frame = decode_frame(body) + with self._lock: + self._latest = frame + + def _recv_exact(self, sock: socket.socket, n: int) -> Optional[bytes]: + buf = bytearray(n) + view = memoryview(buf) + got = 0 + while got < n: + if self._stop.is_set(): + return None + try: + k = sock.recv_into(view[got:]) + except socket.timeout: + continue + if k == 0: + return None + got += k + return bytes(buf) +``` + +- [ ] **Step 5: Run all tests in the file** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_lidar_receiver.py -v` +Expected: 5 passed (3 decoder + 2 reader). + +- [ ] **Step 6: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/lidar_receiver.py data_only_viz/tests/test_lidar_receiver.py data_only_viz/pyproject.toml data_only_viz/uv.lock +git commit -m "feat(icp): LiDAR TCP socket reader with reconnect" +``` + +--- + +## Task 4: Extrinsic calibration data structure & persistence + +**Files:** +- Create: `data_only_viz/lidar_calib.py` +- Create: `data_only_viz/tests/test_lidar_calib.py` + +The iPhone publishes points in **ARKit world coordinates**. Multi-HMR predicts vertices in **webcam camera coordinates** (Z forward, Y down, origin at camera center, meters). Fusion requires a static 4×4 extrinsic `T_arkit_to_cam` that brings LiDAR points into webcam frame. This task only handles the dataclass and JSON persistence — estimation comes in Task 5. + +- [ ] **Step 1: Write the failing persistence test** + +Create `data_only_viz/tests/test_lidar_calib.py`: + +```python +"""Tests for LiDAR ↔ webcam extrinsic calibration persistence.""" +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + + +def test_extrinsic_default_is_identity() -> None: + from data_only_viz.lidar_calib import Extrinsic + + e = Extrinsic.identity() + np.testing.assert_allclose(e.T_arkit_to_cam, np.eye(4)) + assert e.confidence == 0.0 + assert e.captured_at_iso == "" + + +def test_extrinsic_roundtrip_json(tmp_path: Path) -> None: + from data_only_viz.lidar_calib import Extrinsic, load_extrinsic, save_extrinsic + + T = np.eye(4) + T[:3, 3] = [0.1, -0.05, 0.30] + e = Extrinsic(T_arkit_to_cam=T, confidence=0.95, captured_at_iso="2026-05-14T12:00:00Z") + + path = tmp_path / "extrinsic.json" + save_extrinsic(e, path) + loaded = load_extrinsic(path) + + np.testing.assert_allclose(loaded.T_arkit_to_cam, T, atol=1e-10) + assert loaded.confidence == pytest.approx(0.95) + assert loaded.captured_at_iso == "2026-05-14T12:00:00Z" + + +def test_load_extrinsic_missing_path_returns_identity(tmp_path: Path) -> None: + from data_only_viz.lidar_calib import load_extrinsic + + e = load_extrinsic(tmp_path / "does-not-exist.json") + np.testing.assert_allclose(e.T_arkit_to_cam, np.eye(4)) + assert e.confidence == 0.0 +``` + +- [ ] **Step 2: Run, verify failure** + +Run: `cd data_only_viz && uv run pytest tests/test_lidar_calib.py -v` +Expected: 3 failed with `ModuleNotFoundError`. + +- [ ] **Step 3: Implement the persistence layer** + +Create `data_only_viz/lidar_calib.py`: + +```python +"""iPhone LiDAR (ARKit world) ↔ webcam (Multi-HMR camera) extrinsic. + +Persisted as a small JSON document so calibration survives across launches. +The default location is ``~/.config/av-live/lidar_extrinsic.json``; override +with the ``ICP_LIDAR_EXTRINSIC`` env var. +""" +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np + +DEFAULT_EXTRINSIC_PATH = Path.home() / ".config" / "av-live" / "lidar_extrinsic.json" + + +@dataclass +class Extrinsic: + """4x4 rigid transform from ARKit world frame to Multi-HMR camera frame.""" + + T_arkit_to_cam: np.ndarray = field(default_factory=lambda: np.eye(4)) + confidence: float = 0.0 + captured_at_iso: str = "" + + @staticmethod + def identity() -> "Extrinsic": + return Extrinsic(T_arkit_to_cam=np.eye(4), confidence=0.0, captured_at_iso="") + + +def save_extrinsic(e: Extrinsic, path: Path | None = None) -> Path: + path = Path(path) if path is not None else _path_from_env() + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "T_arkit_to_cam": e.T_arkit_to_cam.astype(float).tolist(), + "confidence": float(e.confidence), + "captured_at_iso": e.captured_at_iso, + } + path.write_text(json.dumps(payload, indent=2)) + return path + + +def load_extrinsic(path: Path | None = None) -> Extrinsic: + path = Path(path) if path is not None else _path_from_env() + if not path.exists(): + return Extrinsic.identity() + payload = json.loads(path.read_text()) + return Extrinsic( + T_arkit_to_cam=np.array(payload["T_arkit_to_cam"], dtype=np.float64), + confidence=float(payload.get("confidence", 0.0)), + captured_at_iso=str(payload.get("captured_at_iso", "")), + ) + + +def _path_from_env() -> Path: + p = os.environ.get("ICP_LIDAR_EXTRINSIC") + return Path(p) if p else DEFAULT_EXTRINSIC_PATH +``` + +- [ ] **Step 4: Run tests, verify pass** + +Run: `cd data_only_viz && uv run pytest tests/test_lidar_calib.py -v` +Expected: 3 passed. + +- [ ] **Step 5: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/lidar_calib.py data_only_viz/tests/test_lidar_calib.py +git commit -m "feat(icp): extrinsic calibration dataclass + JSON persistence" +``` + +--- + +## Task 5: Extrinsic estimation from paired pose anchors + +**Files:** +- Modify: `data_only_viz/lidar_calib.py` +- Modify: `data_only_viz/tests/test_lidar_calib.py` +- Create: `data_only_viz/scripts/calibrate_lidar.py` + +Estimation strategy: ask the user to stand still in front of the webcam. Capture (a) one Multi-HMR SMPL-X pelvis vertex (canonical SMPL-X pelvis is vertex index 5559), (b) the same pelvis location in the iPhone ARKit frame (computed as the centroid of the LiDAR mesh anchor whose bounding-box center is closest to the iPhone-detected user). With ≥ 4 paired points (taken at 4 stances: front / left / right / back) we solve a rigid transform via Kabsch (SVD). + +- [ ] **Step 1: Write the failing Kabsch test** + +Append to `data_only_viz/tests/test_lidar_calib.py`: + +```python +def test_kabsch_recovers_known_rigid_transform() -> None: + from data_only_viz.lidar_calib import kabsch_rigid + + rng = np.random.RandomState(7) + src = rng.randn(20, 3) + # Known transform: rotation about Y by 30°, translation (0.1, -0.2, 0.5) + theta = np.deg2rad(30.0) + R = np.array([ + [np.cos(theta), 0, np.sin(theta)], + [0, 1, 0], + [-np.sin(theta), 0, np.cos(theta)], + ]) + t = np.array([0.1, -0.2, 0.5]) + tgt = src @ R.T + t + + T = kabsch_rigid(src, tgt) + R_est = T[:3, :3] + t_est = T[:3, 3] + np.testing.assert_allclose(R_est, R, atol=1e-6) + np.testing.assert_allclose(t_est, t, atol=1e-6) + + +def test_kabsch_requires_at_least_three_pairs() -> None: + from data_only_viz.lidar_calib import kabsch_rigid + + with pytest.raises(ValueError, match="at least 3"): + kabsch_rigid(np.zeros((2, 3)), np.zeros((2, 3))) + + +def test_kabsch_rejects_mismatched_shapes() -> None: + from data_only_viz.lidar_calib import kabsch_rigid + + with pytest.raises(ValueError, match="shape"): + kabsch_rigid(np.zeros((5, 3)), np.zeros((4, 3))) +``` + +- [ ] **Step 2: Run, verify failure** + +Run: `cd data_only_viz && uv run pytest tests/test_lidar_calib.py::test_kabsch_recovers_known_rigid_transform -v` +Expected: `AttributeError: module 'data_only_viz.lidar_calib' has no attribute 'kabsch_rigid'`. + +- [ ] **Step 3: Implement Kabsch** + +Append to `data_only_viz/lidar_calib.py`: + +```python +def kabsch_rigid(src: np.ndarray, tgt: np.ndarray) -> np.ndarray: + """Closed-form rigid alignment (Kabsch via SVD). + + Returns a 4x4 transform T such that ``tgt ≈ (src @ R.T) + t``. + """ + src = np.asarray(src, dtype=np.float64) + tgt = np.asarray(tgt, dtype=np.float64) + if src.shape != tgt.shape: + raise ValueError(f"shape mismatch: src={src.shape} tgt={tgt.shape}") + if src.shape[0] < 3 or src.shape[1] != 3: + raise ValueError("kabsch_rigid needs at least 3 paired 3D points") + src_c = src.mean(axis=0) + tgt_c = tgt.mean(axis=0) + H = (src - src_c).T @ (tgt - tgt_c) + U, _, Vt = np.linalg.svd(H) + d = np.linalg.det(Vt.T @ U.T) + D = np.diag([1.0, 1.0, np.sign(d)]) + R = Vt.T @ D @ U.T + t = tgt_c - R @ src_c + T = np.eye(4) + T[:3, :3] = R + T[:3, 3] = t + return T +``` + +- [ ] **Step 4: Verify the 3 new tests pass** + +Run: `cd data_only_viz && uv run pytest tests/test_lidar_calib.py -v` +Expected: 6 passed (3 persistence + 3 Kabsch). + +- [ ] **Step 5: Create the calibration CLI** + +Create `data_only_viz/scripts/calibrate_lidar.py`: + +```python +"""Interactive one-shot extrinsic calibration between iPhone LiDAR and webcam. + +Usage: + + cd data_only_viz + uv run --extra lidar python -m data_only_viz.scripts.calibrate_lidar \ + --lidar-host 192.168.0.42 --lidar-port 5500 --webcam-index 0 + +The script prompts the user to assume 4 stances (front, left, right, back), +captures paired pelvis points (webcam: Multi-HMR vertex 5559; LiDAR: centroid +of the largest mesh anchor), solves Kabsch, and writes the result to +ICP_LIDAR_EXTRINSIC or the default path. + +Multi-HMR worker is launched in-process for this script (single-shot mode). +""" +from __future__ import annotations + +import argparse +import datetime as dt +import logging +import sys +import time + +import numpy as np + +from data_only_viz.lidar_calib import Extrinsic, kabsch_rigid, save_extrinsic +from data_only_viz.lidar_receiver import LidarTCPReader + +_LOG = logging.getLogger("calibrate_lidar") +_PELVIS_VERT_INDEX = 5559 # SMPL-X canonical pelvis vertex + + +def _wait_for_lidar(reader: LidarTCPReader, timeout_s: float = 5.0): + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + latest = reader.latest() + if latest is not None and latest.points.shape[0] > 50: + return latest + time.sleep(0.05) + raise RuntimeError("LiDAR frame never arrived") + + +def _capture_one_pair(reader: LidarTCPReader, get_smplx_pelvis_cam) -> tuple[np.ndarray, np.ndarray]: + input("Hold still, then press ENTER to capture...") + lidar = _wait_for_lidar(reader) + pelvis_cam = get_smplx_pelvis_cam() + pelvis_arkit = lidar.points.mean(axis=0) # crude centroid; refined in body-detection mode + _LOG.info("captured: cam=%s arkit=%s", pelvis_cam, pelvis_arkit) + return pelvis_cam, pelvis_arkit + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser() + p.add_argument("--lidar-host", required=True) + p.add_argument("--lidar-port", type=int, default=5500) + p.add_argument("--webcam-index", type=int, default=0) + p.add_argument("--stances", type=int, default=4) + args = p.parse_args(argv) + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s") + + reader = LidarTCPReader(host=args.lidar_host, port=args.lidar_port) + reader.start() + + # NB: the actual Multi-HMR getter is wired in Task 9 when the main pipeline + # exposes a single-shot predictor. For now this script is the *scaffolding* + # — Task 9 plugs in `multi_hmr_worker.predict_once()`. + def _placeholder_pelvis_cam() -> np.ndarray: + raise SystemExit("calibrate_lidar requires Task 9 to be complete (predict_once API)") + + pairs_cam, pairs_arkit = [], [] + try: + for i in range(args.stances): + _LOG.info("stance %d/%d", i + 1, args.stances) + cam, arkit = _capture_one_pair(reader, _placeholder_pelvis_cam) + pairs_cam.append(cam) + pairs_arkit.append(arkit) + finally: + reader.stop() + + T = kabsch_rigid(np.asarray(pairs_arkit), np.asarray(pairs_cam)) + path = save_extrinsic(Extrinsic( + T_arkit_to_cam=T, + confidence=1.0, + captured_at_iso=dt.datetime.now(dt.timezone.utc).isoformat(), + )) + _LOG.info("extrinsic saved to %s", path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +``` + +- [ ] **Step 6: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/lidar_calib.py data_only_viz/tests/test_lidar_calib.py data_only_viz/scripts/calibrate_lidar.py +git commit -m "feat(icp): Kabsch extrinsic estimator + calibration CLI scaffold" +``` + +--- + +## Task 6: ICP wrapper — single-mesh registration + +**Files:** +- Create: `data_only_viz/icp_fusion.py` +- Create: `data_only_viz/tests/test_icp_fusion.py` + +Core ICP wrapper: given SMPL-X verts (10475, 3) in camera frame and LiDAR points (N, 3) in camera frame (post-extrinsic), it (a) crops LiDAR to a bounding-box around the SMPL-X mesh + 0.30 m margin, (b) voxel-downsamples both sides to 0.02 m, (c) runs point-to-plane ICP with 0.05 m correspondence threshold, (d) returns either the registered vertices or the original ones plus a `fitness` score and an `accepted` flag. + +- [ ] **Step 1: Write the failing convergence test** + +Create `data_only_viz/tests/test_icp_fusion.py`: + +```python +"""Tests for ICP registration of SMPL-X verts onto LiDAR point clouds.""" +from __future__ import annotations + +import numpy as np +import pytest + +pytest.importorskip("open3d") + + +def _synthetic_smplx_torso(n: int = 1500, seed: int = 0) -> np.ndarray: + """Generate a coarse capsule-like point cloud standing in for SMPL-X verts.""" + rng = np.random.RandomState(seed) + z = rng.uniform(0.0, 1.7, size=n) + r = 0.12 + 0.02 * rng.randn(n) + theta = rng.uniform(0, 2 * np.pi, size=n) + x = r * np.cos(theta) + y = r * np.sin(theta) + return np.stack([x, y, z], axis=1).astype(np.float32) + + +def test_icp_recovers_small_translation() -> None: + from data_only_viz.icp_fusion import IcpConfig, register_mesh_to_lidar + + src = _synthetic_smplx_torso(seed=1) + translation = np.array([0.05, 0.02, 0.10], dtype=np.float32) + tgt = src + translation + 0.005 * np.random.RandomState(2).randn(*src.shape).astype(np.float32) + + out = register_mesh_to_lidar(src, tgt, config=IcpConfig()) + + assert out.accepted, f"ICP should accept, got fitness={out.fitness:.3f}" + # Mean residual to the truth-translated source should shrink. + truth = src + translation + err_before = np.linalg.norm(src - truth, axis=1).mean() + err_after = np.linalg.norm(out.vertices_registered - truth, axis=1).mean() + assert err_after < err_before * 0.5, f"err before={err_before:.4f} after={err_after:.4f}" + + +def test_icp_rejects_when_lidar_too_sparse() -> None: + from data_only_viz.icp_fusion import IcpConfig, register_mesh_to_lidar + + src = _synthetic_smplx_torso(seed=3) + tgt = src[:5] # only 5 points — well below MIN_LIDAR_POINTS + + out = register_mesh_to_lidar(src, tgt, config=IcpConfig()) + assert not out.accepted + np.testing.assert_array_equal(out.vertices_registered, src) + + +def test_icp_rejects_on_nan_input() -> None: + from data_only_viz.icp_fusion import IcpConfig, register_mesh_to_lidar + + src = _synthetic_smplx_torso(seed=4) + src[10, 1] = np.nan + tgt = src.copy() + tgt = np.nan_to_num(tgt, nan=0.0) + + out = register_mesh_to_lidar(src, tgt, config=IcpConfig()) + assert not out.accepted + np.testing.assert_array_equal(out.vertices_registered, src) + + +def test_icp_preserves_dtype_and_shape() -> None: + from data_only_viz.icp_fusion import IcpConfig, register_mesh_to_lidar + + src = _synthetic_smplx_torso(seed=5) + tgt = src + np.array([0.0, 0.0, 0.02], dtype=np.float32) + out = register_mesh_to_lidar(src, tgt, config=IcpConfig()) + assert out.vertices_registered.shape == src.shape + assert out.vertices_registered.dtype == np.float32 +``` + +- [ ] **Step 2: Run, verify failure** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_icp_fusion.py -v` +Expected: 4 failed with `ModuleNotFoundError`. + +- [ ] **Step 3: Implement ICP wrapper** + +Create `data_only_viz/icp_fusion.py`: + +```python +"""ICP fusion between Multi-HMR SMPL-X meshes and iPhone LiDAR point clouds. + +All operations happen in the **webcam camera frame** (meters, OpenCV +convention: +X right, +Y down, +Z forward). LiDAR points must be +pre-transformed via `Extrinsic.T_arkit_to_cam`. +""" +from __future__ import annotations + +import logging +from dataclasses import dataclass + +import numpy as np + +try: + import open3d as o3d +except ImportError: # pragma: no cover - exercised via skipif at import sites + o3d = None # type: ignore[assignment] + +_LOG = logging.getLogger(__name__) + +MIN_LIDAR_POINTS = 200 +MIN_FITNESS = 0.30 +MAX_RMSE_M = 0.05 +CROP_MARGIN_M = 0.30 + + +@dataclass +class IcpConfig: + voxel_size_m: float = 0.02 + max_correspondence_m: float = 0.05 + max_iterations: int = 30 + + +@dataclass +class IcpResult: + vertices_registered: np.ndarray # (10475, 3) float32 — fused or original + accepted: bool + fitness: float + rmse_m: float + iterations: int + + +def register_mesh_to_lidar( + smplx_verts_cam: np.ndarray, + lidar_points_cam: np.ndarray, + config: IcpConfig | None = None, +) -> IcpResult: + """Register SMPL-X verts onto a cropped LiDAR neighborhood. + + Returns the **original** verts if ICP is rejected (NaN, too few points, + poor fitness, excessive RMSE). The caller is expected to fall back to the + raw Multi-HMR output on rejection. + """ + if o3d is None: + raise RuntimeError("open3d not installed — install with `uv sync --extra lidar`") + + cfg = config or IcpConfig() + src = np.ascontiguousarray(smplx_verts_cam, dtype=np.float32) + + if not np.isfinite(src).all(): + _LOG.debug("ICP rejected: NaN/Inf in SMPL-X verts") + return IcpResult(src, False, 0.0, float("inf"), 0) + + lidar = _crop_to_bbox(lidar_points_cam, src, margin_m=CROP_MARGIN_M) + if lidar.shape[0] < MIN_LIDAR_POINTS or not np.isfinite(lidar).all(): + _LOG.debug("ICP rejected: insufficient LiDAR points (%d)", lidar.shape[0]) + return IcpResult(src, False, 0.0, float("inf"), 0) + + src_pcd = _to_pcd(src, cfg.voxel_size_m, estimate_normals=True) + tgt_pcd = _to_pcd(lidar, cfg.voxel_size_m, estimate_normals=True) + + if len(src_pcd.points) < 10 or len(tgt_pcd.points) < 10: + return IcpResult(src, False, 0.0, float("inf"), 0) + + criteria = o3d.pipelines.registration.ICPConvergenceCriteria( + max_iteration=cfg.max_iterations, + relative_fitness=1e-6, + relative_rmse=1e-6, + ) + result = o3d.pipelines.registration.registration_icp( + src_pcd, tgt_pcd, cfg.max_correspondence_m, + np.eye(4), + o3d.pipelines.registration.TransformationEstimationPointToPlane(), + criteria, + ) + + accepted = (result.fitness >= MIN_FITNESS) and (result.inlier_rmse <= MAX_RMSE_M) + if not accepted: + _LOG.debug( + "ICP rejected: fitness=%.3f rmse=%.4f", + result.fitness, result.inlier_rmse, + ) + return IcpResult(src, False, float(result.fitness), float(result.inlier_rmse), 0) + + # Apply transform to the **full** (non-downsampled) source verts. + T = np.asarray(result.transformation, dtype=np.float32) + homog = np.concatenate([src, np.ones((src.shape[0], 1), dtype=np.float32)], axis=1) + fused = (homog @ T.T)[:, :3] + if not np.isfinite(fused).all(): + return IcpResult(src, False, float(result.fitness), float(result.inlier_rmse), 0) + + return IcpResult( + vertices_registered=np.ascontiguousarray(fused, dtype=np.float32), + accepted=True, + fitness=float(result.fitness), + rmse_m=float(result.inlier_rmse), + iterations=cfg.max_iterations, + ) + + +def _crop_to_bbox(points: np.ndarray, anchor: np.ndarray, margin_m: float) -> np.ndarray: + if points.size == 0: + return points.astype(np.float32, copy=False) + lo = anchor.min(axis=0) - margin_m + hi = anchor.max(axis=0) + margin_m + mask = np.all((points >= lo) & (points <= hi), axis=1) + return points[mask].astype(np.float32, copy=False) + + +def _to_pcd(points: np.ndarray, voxel_size_m: float, estimate_normals: bool): + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(points.astype(np.float64, copy=False)) + if voxel_size_m > 0: + pcd = pcd.voxel_down_sample(voxel_size_m) + if estimate_normals: + pcd.estimate_normals( + search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=voxel_size_m * 2, max_nn=30), + ) + return pcd +``` + +- [ ] **Step 4: Run tests, verify pass** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_icp_fusion.py -v` +Expected: 4 passed. + +- [ ] **Step 5: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/icp_fusion.py data_only_viz/tests/test_icp_fusion.py +git commit -m "feat(icp): point-to-plane registration wrapper with reject gate" +``` + +--- + +## Task 7: Multi-person dispatch with per-pid spatial gating + +**Files:** +- Modify: `data_only_viz/icp_fusion.py` +- Modify: `data_only_viz/tests/test_icp_fusion.py` + +When multiple people are present, naïve ICP on a shared LiDAR cloud confuses neighbors. We partition the LiDAR cloud by nearest-neighbor centroid: each point goes to the SMPL-X mesh whose pelvis-vertex (5559) is closest, with a hard max distance of 1.0 m to discard background geometry. + +- [ ] **Step 1: Write the failing multi-person test** + +Append to `data_only_viz/tests/test_icp_fusion.py`: + +```python +def test_partition_lidar_by_pid_two_people() -> None: + from data_only_viz.icp_fusion import partition_lidar_by_pid + + # Two SMPL-X meshes 1.5 m apart along X. + src_a = _synthetic_smplx_torso(seed=10) + np.array([-0.75, 0.0, 0.0], dtype=np.float32) + src_b = _synthetic_smplx_torso(seed=11) + np.array([+0.75, 0.0, 0.0], dtype=np.float32) + pelvis_a = src_a.mean(axis=0) + pelvis_b = src_b.mean(axis=0) + + # LiDAR cloud spanning both, plus 100 background points 5 m away. + lidar = np.concatenate([ + src_a + 0.01 * np.random.RandomState(20).randn(*src_a.shape).astype(np.float32), + src_b + 0.01 * np.random.RandomState(21).randn(*src_b.shape).astype(np.float32), + np.array([[10.0, 10.0, 10.0]] * 100, dtype=np.float32), + ]) + + parts = partition_lidar_by_pid(lidar, pelvises={0: pelvis_a, 1: pelvis_b}, max_dist_m=1.0) + + assert set(parts.keys()) == {0, 1} + assert parts[0].shape[0] > 1000 + assert parts[1].shape[0] > 1000 + # Background must be discarded + assert not np.any(np.linalg.norm(parts[0] - np.array([10, 10, 10]), axis=1) < 0.5) + assert not np.any(np.linalg.norm(parts[1] - np.array([10, 10, 10]), axis=1) < 0.5) + + +def test_partition_returns_empty_dict_when_no_pelvises() -> None: + from data_only_viz.icp_fusion import partition_lidar_by_pid + + out = partition_lidar_by_pid(np.zeros((100, 3), dtype=np.float32), pelvises={}, max_dist_m=1.0) + assert out == {} +``` + +- [ ] **Step 2: Run, verify failure** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_icp_fusion.py::test_partition_lidar_by_pid_two_people -v` +Expected: `ImportError: cannot import name 'partition_lidar_by_pid'`. + +- [ ] **Step 3: Implement partitioner** + +Append to `data_only_viz/icp_fusion.py`: + +```python +def partition_lidar_by_pid( + lidar_points_cam: np.ndarray, + pelvises: dict[int, np.ndarray], + max_dist_m: float = 1.0, +) -> dict[int, np.ndarray]: + """Assign each LiDAR point to the closest pelvis within ``max_dist_m``. + + Points beyond ``max_dist_m`` from every pelvis (background, furniture) + are dropped. Returns ``{pid: (M, 3) float32}`` — pids with zero assigned + points are omitted. + """ + if not pelvises or lidar_points_cam.size == 0: + return {} + pids = list(pelvises.keys()) + centers = np.stack([pelvises[p] for p in pids]).astype(np.float32) # (P, 3) + pts = np.ascontiguousarray(lidar_points_cam, dtype=np.float32) + + # (N, P) squared distance + diff = pts[:, None, :] - centers[None, :, :] + d2 = np.einsum("npk,npk->np", diff, diff) + nearest = d2.argmin(axis=1) + nearest_d = np.sqrt(d2[np.arange(d2.shape[0]), nearest]) + + mask = nearest_d <= max_dist_m + out: dict[int, np.ndarray] = {} + for idx, pid in enumerate(pids): + sel = mask & (nearest == idx) + if not sel.any(): + continue + out[pid] = pts[sel] + return out +``` + +- [ ] **Step 4: Run all icp_fusion tests, verify pass** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_icp_fusion.py -v` +Expected: 6 passed. + +- [ ] **Step 5: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/icp_fusion.py data_only_viz/tests/test_icp_fusion.py +git commit -m "feat(icp): partition LiDAR per pid with max-distance gate" +``` + +--- + +## Task 8: Fusion worker — per-frame orchestration + +**Files:** +- Modify: `data_only_viz/icp_fusion.py` +- Modify: `data_only_viz/tests/test_icp_fusion.py` + +A `FusionWorker` glues everything: pull the latest LiDAR frame, apply the loaded extrinsic, partition by pid using `state.persons_smplx[*].vertices_3d[5559]` as pelvis, register each person, write fused verts back. Operates on a `State` snapshot — caller decides cadence (typically called every Multi-HMR frame, i.e. ~6.5 Hz). + +- [ ] **Step 1: Write the failing fusion worker test** + +Append to `data_only_viz/tests/test_icp_fusion.py`: + +```python +def test_fusion_worker_in_place_update(monkeypatch) -> None: + from data_only_viz.icp_fusion import FusionWorker, IcpConfig + from data_only_viz.lidar_calib import Extrinsic + from data_only_viz.lidar_receiver import LidarFrame + from data_only_viz.state import SMPLXPerson, State + + src = _synthetic_smplx_torso(seed=30) + # Pad to full SMPL-X length so vertex 5559 is valid. + verts = np.zeros((10475, 3), dtype=np.float32) + verts[: src.shape[0]] = src + verts[5559] = src.mean(axis=0) # use centroid as pelvis stand-in + + person = SMPLXPerson(pid=0, vertices_3d=verts.copy()) + state = State() + state.persons_smplx = [person] + + # Synthetic LiDAR: src translated by +0.04 along Y. + lidar_pts = src + np.array([0.0, 0.04, 0.0], dtype=np.float32) + state.lidar_points = lidar_pts + state.lidar_timestamp_ns = 1 + + worker = FusionWorker( + extrinsic=Extrinsic.identity(), + config=IcpConfig(), + ) + metadata = worker.run_once(state) + + assert metadata.applied == {0} + # Pelvis should have moved roughly +0.04 along Y in the fused mesh. + delta = state.persons_smplx[0].vertices_3d[5559] - verts[5559] + assert 0.02 <= delta[1] <= 0.06 + + +def test_fusion_worker_skips_when_no_lidar() -> None: + from data_only_viz.icp_fusion import FusionWorker, IcpConfig + from data_only_viz.lidar_calib import Extrinsic + from data_only_viz.state import SMPLXPerson, State + + verts = np.zeros((10475, 3), dtype=np.float32) + verts[5559] = [0.0, 1.0, 2.0] + state = State() + state.persons_smplx = [SMPLXPerson(pid=0, vertices_3d=verts.copy())] + state.lidar_points = None + + worker = FusionWorker(extrinsic=Extrinsic.identity(), config=IcpConfig()) + metadata = worker.run_once(state) + assert metadata.applied == set() + np.testing.assert_array_equal(state.persons_smplx[0].vertices_3d, verts) +``` + +- [ ] **Step 2: Add the `lidar_points` field to State** + +Modify `data_only_viz/state.py` — locate the `State` dataclass and append: + +```python + # ---- LiDAR / ICP fusion (Task 8) ---- + lidar_points: "np.ndarray | None" = None # (N, 3) float32, webcam camera frame + lidar_timestamp_ns: int = 0 + icp_metadata: "FusionMetadata | None" = None # last fusion outcome (per-frame) +``` + +Add the forward import at the top of the file: + +```python +from __future__ import annotations + +import numpy as np # type: ignore # noqa: F401 (used by string-typed annotations) +``` + +(If `np` is already imported, leave the existing import alone.) + +- [ ] **Step 3: Run, verify failure** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_icp_fusion.py::test_fusion_worker_in_place_update -v` +Expected: `ImportError: cannot import name 'FusionWorker'`. + +- [ ] **Step 4: Implement FusionWorker** + +Append to `data_only_viz/icp_fusion.py`: + +```python +PELVIS_VERT_INDEX = 5559 # SMPL-X canonical pelvis vertex + + +@dataclass +class FusionMetadata: + applied: set[int] # pids whose verts were replaced + fitness: dict[int, float] + rmse_m: dict[int, float] + n_lidar_points_used: int + + +class FusionWorker: + """Per-frame ICP fusion orchestrator (caller-driven, no internal thread).""" + + def __init__(self, extrinsic, config: IcpConfig | None = None) -> None: + self._extrinsic = extrinsic + self._config = config or IcpConfig() + + def set_extrinsic(self, extrinsic) -> None: + self._extrinsic = extrinsic + + def run_once(self, state) -> FusionMetadata: + """Replace ``state.persons_smplx[*].vertices_3d`` with fused versions.""" + applied: set[int] = set() + fitness: dict[int, float] = {} + rmse: dict[int, float] = {} + + lidar = getattr(state, "lidar_points", None) + if lidar is None or lidar.size == 0 or not state.persons_smplx: + return FusionMetadata(applied, fitness, rmse, 0) + + # Transform LiDAR into camera frame. + T = np.asarray(self._extrinsic.T_arkit_to_cam, dtype=np.float32) + homog = np.concatenate([lidar, np.ones((lidar.shape[0], 1), dtype=np.float32)], axis=1) + lidar_cam = (homog @ T.T)[:, :3] + + pelvises = { + p.pid: p.vertices_3d[PELVIS_VERT_INDEX] + for p in state.persons_smplx + if p.vertices_3d is not None + } + parts = partition_lidar_by_pid(lidar_cam, pelvises, max_dist_m=1.0) + + for person in state.persons_smplx: + pts = parts.get(person.pid) + if pts is None: + continue + result = register_mesh_to_lidar(person.vertices_3d, pts, self._config) + fitness[person.pid] = result.fitness + rmse[person.pid] = result.rmse_m + if result.accepted: + person.vertices_3d = result.vertices_registered + applied.add(person.pid) + + return FusionMetadata(applied, fitness, rmse, lidar_cam.shape[0]) +``` + +- [ ] **Step 5: Run all tests, verify pass** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_icp_fusion.py -v` +Expected: 8 passed. + +- [ ] **Step 6: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/icp_fusion.py data_only_viz/state.py data_only_viz/tests/test_icp_fusion.py +git commit -m "feat(icp): FusionWorker + State.lidar_points field" +``` + +--- + +## Task 9: Wire LiDAR receiver + FusionWorker into main pipeline + +**Files:** +- Modify: `data_only_viz/main.py` +- Modify: `data_only_viz/multi_hmr_worker.py` + +ICP fusion is **opt-in** via `ICP_FUSION=1`. When enabled, `main.py` starts a `LidarTCPReader`, loads the persisted extrinsic, instantiates `FusionWorker`, and inserts a `run_once(state)` call **after** Multi-HMR writes `state.persons_smplx` and **before** the OSC/TCP publisher reads it. + +- [ ] **Step 1: Identify the integration point** + +Run: `grep -n "persons_smplx" data_only_viz/main.py` +Expected: locate the line where Multi-HMR worker results land on `state.persons_smplx` (typically inside the worker loop or right after a poll). + +- [ ] **Step 2: Add the wiring** + +Add near the top of `data_only_viz/main.py`: + +```python +import os + +from data_only_viz.icp_fusion import FusionWorker, IcpConfig +from data_only_viz.lidar_calib import load_extrinsic +from data_only_viz.lidar_receiver import LidarTCPReader +``` + +Add a helper near the worker-startup section: + +```python +def _start_icp_fusion(state): + """Start LiDAR reader + FusionWorker if ICP_FUSION=1.""" + if os.environ.get("ICP_FUSION", "0") != "1": + return None, None + host = os.environ.get("ICP_LIDAR_HOST") + port = int(os.environ.get("ICP_LIDAR_PORT", "5500")) + if not host: + raise RuntimeError("ICP_FUSION=1 requires ICP_LIDAR_HOST to be set") + reader = LidarTCPReader(host=host, port=port) + reader.start() + extrinsic = load_extrinsic() + worker = FusionWorker(extrinsic=extrinsic, config=IcpConfig()) + return reader, worker +``` + +Inside the main loop, immediately after the Multi-HMR step updates `state.persons_smplx`, add: + +```python +if icp_worker is not None and icp_reader is not None: + frame = icp_reader.latest() + if frame is not None: + state.lidar_points = frame.points + state.lidar_timestamp_ns = frame.timestamp_ns + state.icp_metadata = icp_worker.run_once(state) + else: + state.lidar_points = None + state.icp_metadata = None +``` + +And in startup: + +```python +icp_reader, icp_worker = _start_icp_fusion(state) +``` + +In shutdown: + +```python +if icp_reader is not None: + icp_reader.stop() +``` + +- [ ] **Step 3: Add a one-shot predictor hook for the calibration CLI** + +In `data_only_viz/multi_hmr_worker.py`, find the class that performs inference and expose a public `predict_once(rgb_image) -> SMPLXPerson | None` method that runs a single forward pass without modifying any shared state. Keep it ≤ 30 lines — just call into the existing inference path, return the first detection or `None`. + +- [ ] **Step 4: Update the calibration CLI to use it** + +Replace the `_placeholder_pelvis_cam` body in `data_only_viz/scripts/calibrate_lidar.py` with a real OpenCV webcam grab + `predict_once` call. Pelvis = `result.vertices_3d[5559]`. + +```python +import cv2 + +from data_only_viz.multi_hmr_worker import predict_once # or equivalent factory + +cap = cv2.VideoCapture(args.webcam_index) + + +def _real_pelvis_cam() -> np.ndarray: + ok, frame = cap.read() + if not ok: + raise RuntimeError("webcam read failed") + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + person = predict_once(rgb) + if person is None: + raise RuntimeError("no person detected — adjust pose and retry") + return person.vertices_3d[5559] +``` + +Wire `_real_pelvis_cam` into `_capture_one_pair` in place of the placeholder. + +- [ ] **Step 5: Sanity-check the wiring (no LiDAR available)** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ICP_FUSION=0 uv run --extra lidar python -m data_only_viz.main` for ~10 seconds, Ctrl-C. +Expected: pipeline starts as before, no LiDAR-related errors. + +- [ ] **Step 6: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/main.py data_only_viz/multi_hmr_worker.py data_only_viz/scripts/calibrate_lidar.py +git commit -m "feat(icp): wire fusion behind ICP_FUSION env var" +``` + +--- + +## Task 10: Latency & convergence bench + +**Files:** +- Create: `data_only_viz/scripts/bench_icp_fusion.py` + +A standalone harness that ingests a recorded LiDAR + Multi-HMR sequence (or synthetic), measures: (a) per-call ICP latency p50/p95, (b) acceptance rate, (c) mean pelvis displacement post-fusion. Outputs a single JSON line. + +- [ ] **Step 1: Create the bench script** + +```python +"""Latency / convergence bench for the ICP fusion worker. + +Usage: + + cd data_only_viz + uv run --extra lidar python -m data_only_viz.scripts.bench_icp_fusion \ + --n-frames 200 --n-people 2 --seed 0 +""" +from __future__ import annotations + +import argparse +import json +import time + +import numpy as np + +from data_only_viz.icp_fusion import FusionWorker, IcpConfig +from data_only_viz.lidar_calib import Extrinsic +from data_only_viz.state import SMPLXPerson, State + + +def _synth_person(seed: int, offset_x: float) -> SMPLXPerson: + rng = np.random.RandomState(seed) + verts = np.zeros((10475, 3), dtype=np.float32) + pts = rng.randn(2000, 3).astype(np.float32) * 0.1 + verts[: pts.shape[0]] = pts + np.array([offset_x, 0, 1.5], dtype=np.float32) + verts[5559] = pts.mean(axis=0) + np.array([offset_x, 0, 1.5], dtype=np.float32) + return SMPLXPerson(pid=seed, vertices_3d=verts) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser() + p.add_argument("--n-frames", type=int, default=200) + p.add_argument("--n-people", type=int, default=2) + p.add_argument("--seed", type=int, default=0) + args = p.parse_args(argv) + + rng = np.random.RandomState(args.seed) + persons = [_synth_person(i, offset_x=-0.6 + 1.2 * i) for i in range(args.n_people)] + state = State() + state.persons_smplx = persons + + worker = FusionWorker(extrinsic=Extrinsic.identity(), config=IcpConfig()) + + latencies_ms: list[float] = [] + accepted = 0 + pelvis_delta_m: list[float] = [] + for _ in range(args.n_frames): + # Build synthetic LiDAR: ground-truth verts perturbed by 2 cm noise + 5 cm bias. + all_pts = np.concatenate([ + pers.vertices_3d[: 2000] + np.array([0, 0.05, 0], dtype=np.float32) + + 0.02 * rng.randn(2000, 3).astype(np.float32) + for pers in persons + ]) + state.lidar_points = all_pts + before = np.stack([p.vertices_3d[5559].copy() for p in state.persons_smplx]) + t0 = time.perf_counter() + meta = worker.run_once(state) + latencies_ms.append((time.perf_counter() - t0) * 1000.0) + accepted += len(meta.applied) + after = np.stack([p.vertices_3d[5559] for p in state.persons_smplx]) + pelvis_delta_m.extend(np.linalg.norm(after - before, axis=1).tolist()) + + report = { + "n_frames": args.n_frames, + "n_people": args.n_people, + "latency_ms_p50": float(np.percentile(latencies_ms, 50)), + "latency_ms_p95": float(np.percentile(latencies_ms, 95)), + "acceptance_rate": accepted / (args.n_frames * args.n_people), + "pelvis_delta_m_mean": float(np.mean(pelvis_delta_m)), + "pelvis_delta_m_max": float(np.max(pelvis_delta_m)), + } + print(json.dumps(report, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 2: Run the bench** + +Run: `cd data_only_viz && uv run --extra lidar python -m data_only_viz.scripts.bench_icp_fusion --n-frames 100 --n-people 2` +Expected: JSON output with `latency_ms_p95 < 60` (rough target on M5 CPU) and `acceptance_rate > 0.85`. + +- [ ] **Step 3: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/scripts/bench_icp_fusion.py +git commit -m "test(icp): synthetic latency + convergence bench" +``` + +--- + +## Task 11: Docs — env vars, calibration procedure, troubleshooting + +**Files:** +- Create: `docs/ICP_FUSION.md` +- Modify: `CLAUDE.md` (root) + +- [ ] **Step 1: Create `docs/ICP_FUSION.md`** + +```markdown +# ICP LiDAR ↔ SMPL-X Dense Fusion + +Refines Multi-HMR SMPL-X meshes using live iPhone LiDAR via point-to-plane ICP. + +## Env vars + +| Var | Default | Effect | +|-----|---------|--------| +| `ICP_FUSION` | `0` | `1` enables LiDAR receiver + FusionWorker | +| `ICP_LIDAR_HOST` | _(required when on)_ | iPhone ARBodyTracker IP on the LAN | +| `ICP_LIDAR_PORT` | `5500` | TCP port the iOS app publishes ARMesh on | +| `ICP_LIDAR_EXTRINSIC` | `~/.config/av-live/lidar_extrinsic.json` | Path to persisted extrinsic JSON | + +## Calibration + +1. Launch the iPhone ARBodyTracker app and note its LAN IP. +2. From `data_only_viz/`: + ```bash + uv run --extra lidar python -m data_only_viz.scripts.calibrate_lidar \ + --lidar-host --lidar-port 5500 --webcam-index 0 + ``` +3. The script asks for 4 stances (front / left / right / back). Hold still each time and press ENTER. +4. The estimated extrinsic is written to `ICP_LIDAR_EXTRINSIC`. Re-run any time the camera or iPhone moves. + +## Runtime + +```bash +ICP_FUSION=1 ICP_LIDAR_HOST=192.168.0.42 uv run --extra lidar python -m data_only_viz.main +``` + +## Troubleshooting + +- **`open3d` missing** → `cd data_only_viz && uv sync --extra lidar` +- **No LiDAR frames** → check that the iPhone app is publishing on the expected port and that nothing else is bound to it. `nc -l 5500` from the Mac should not succeed while the app runs. +- **ICP always rejected (`fitness < 0.30`)** → the extrinsic is likely stale; re-run calibration. Verify the iPhone is facing the same scene as the webcam. +- **Mesh appears scaled wrong** → SMPL-X is in metres; the iPhone publishes metres. If you see a factor-1000 mismatch the iOS encoder is sending millimetres — patch the iOS app, not this code. +- **Bench shows `latency_ms_p95 > 100`** → reduce `IcpConfig.voxel_size_m` (e.g. 0.03 m) or `max_iterations` (e.g. 20). +``` + +- [ ] **Step 2: Update root `CLAUDE.md`** + +In the "RC0.1+ environment variables" table, append four rows: + +```markdown +| `ICP_FUSION` | `0` | `1` to enable LiDAR↔SMPL-X ICP fusion (cf. `docs/ICP_FUSION.md`) | +| `ICP_LIDAR_HOST` | _(unset)_ | iPhone ARBodyTracker IP when `ICP_FUSION=1` | +| `ICP_LIDAR_PORT` | `5500` | iPhone ARMesh TCP port | +| `ICP_LIDAR_EXTRINSIC` | `~/.config/av-live/lidar_extrinsic.json` | extrinsic JSON path | +``` + +- [ ] **Step 3: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add docs/ICP_FUSION.md CLAUDE.md +git commit -m "docs(icp): runtime env vars + calibration procedure" +``` + +--- + +## Task 12: Smoke-test the full pipeline end-to-end + +**Files:** _(none — operational gate)_ + +- [ ] **Step 1: Run all icp tests once more** + +Run: `cd data_only_viz && uv run --extra lidar pytest tests/test_open3d_smoke.py tests/test_lidar_receiver.py tests/test_lidar_calib.py tests/test_icp_fusion.py -v` +Expected: all green (≈20 passed). + +- [ ] **Step 2: Bench** + +Run: `cd data_only_viz && uv run --extra lidar python -m data_only_viz.scripts.bench_icp_fusion --n-frames 200` +Expected: `latency_ms_p95 < 60`, `acceptance_rate > 0.85`, `pelvis_delta_m_mean` ≈ 0.04 m (matches the synthetic 5 cm bias). + +- [ ] **Step 3: Live smoke (with iPhone)** + +With ARBodyTracker running on the iPhone and the webcam on: + +```bash +ICP_FUSION=1 ICP_LIDAR_HOST= uv run --extra lidar python -m data_only_viz.main +``` + +Verify in logs: `applied={0}` lines, `fitness > 0.3`, no NaN warnings from Multi-HMR. + +- [ ] **Step 4: Live smoke (LiDAR cable pulled)** + +Disconnect the iPhone from Wi-Fi mid-run. Pipeline must continue without crashing — fused verts should fall back to raw Multi-HMR output within ~1 frame. + +- [ ] **Step 5: Final commit (changelog)** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git tag -m "ICP LiDAR fusion ready for live testing" icp-fusion-mvp +``` + +--- + +## Self-Review + +**Spec coverage** — Each architectural concern is covered: +- LiDAR ingestion → Tasks 2–3 +- Coordinate alignment → Tasks 4–5 + CLI in Task 9 +- ICP core → Task 6 +- Multi-person dispatch → Task 7 +- Pipeline integration → Tasks 8–9 +- Observability/bench → Task 10 +- Docs → Task 11 +- Operational gate → Task 12 + +**Placeholder scan** — No `TODO`, no "add error handling" without code, no "similar to Task N". Each step contains the actual code, command, or assertion. + +**Type consistency** — `register_mesh_to_lidar` / `partition_lidar_by_pid` / `FusionWorker.run_once` signatures are stable across tasks 6–8. `Extrinsic.T_arkit_to_cam` shape (4×4) and dtype (float64 for math, cast to float32 only inside the transform) are consistent. `SMPLXPerson.vertices_3d` shape `(10475, 3) float32` matches `multi_hmr_worker.py:403`. Pelvis vertex index 5559 is referenced identically in Tasks 5, 8, 9. + +**Out-of-band requirements:** +- iPhone ARBodyTracker app must already publish ARMeshAnchor data on TCP (per memory `project_iphone_arbodytracker`). If the wire format differs from the assumed `[uint32 len][uint64 ts][uint32 vcount][float32 xyz]*`, Task 2's decoder must be adjusted before Task 3 will work. +- `data_only_viz` repo must be sane (git fresh-clone done, " 2" iCloud collisions cleaned). This plan assumes a healthy repo on the canonical `feat/action-head` or `main` branch. diff --git a/docs/superpowers/plans/2026-05-14-iphone-lidar-multihmr-fusion.md b/docs/superpowers/plans/2026-05-14-iphone-lidar-multihmr-fusion.md new file mode 100644 index 0000000..2f988ea --- /dev/null +++ b/docs/superpowers/plans/2026-05-14-iphone-lidar-multihmr-fusion.md @@ -0,0 +1,921 @@ +# iPhone LiDAR → Multi-HMR Fusion Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Wire the iOS ARBodyTracker app's `/body3d/kp` OSC stream into the Python pipeline so ARKit 91-joint LiDAR ground truth fixes Multi-HMR's scale ambiguity and reduces SMPL-X joint jitter via Kalman fusion. + +**Architecture:** A new `iphone_osc_listener.py` worker subscribes to `/body3d/kp` on UDP `:57128` (port distinct from existing `:57126` body output) and writes per-pid 91-joint arrays into `state.persons_arkit_joints`. `pose_filter.py` gains an `arkit_fuse` stage that pulls the ARKit joints (low-noise ground truth) and overrides matching MediaPipe Pose 33 slots before the existing kalman/one_euro chain runs. `multi_hmr_worker.py` post-inference reads the ARKit pelvis world-z and rewrites `pred_cam_t` of the SMPL-X mesh so it lands at the actual depth instead of HaMeR's per-frame guess. + +**Tech Stack:** Python 3.14, python-osc (OSCDispatcher), numpy, threading, pytest. Existing modules: `state.State`, `pose_filter.PoseFilterChain`, `multi_hmr_worker.MultiHMRWorker`. + +--- + +## File Structure + +| File | Responsibility | +|---|---| +| `data_only_viz/iphone_osc_listener.py` | **NEW**. ThreadingOSCUDPServer on `:57128`, routes `/body3d/kp pid joint_idx x y z` → `state.persons_arkit_joints[pid][joint_idx] = (x,y,z)`. GC entries older than 1.0s. | +| `data_only_viz/state.py` | **MODIFY**. Add fields `persons_arkit_joints: dict[int, np.ndarray]` (91×3 per pid) + `persons_arkit_last_t: dict[int, float]`. | +| `data_only_viz/arkit_joint_map.py` | **NEW**. Constant tuple `ARKIT91_TO_MP33` mapping ARKit joint indices → MediaPipe Pose 33 indices. | +| `data_only_viz/pose_filter.py` | **MODIFY**. Add `"arkit_fuse"` to `ALL_STAGES`, add `ArkitFuse` class, splice in `PoseFilterChain.apply` before kalman. | +| `data_only_viz/multi_hmr_worker.py` | **MODIFY**. After Multi-HMR inference, if `state.persons_arkit_joints[pid]` is fresh, override `pred_cam_t.z` with ARKit pelvis world-z. | +| `data_only_viz/main.py` | **MODIFY**. Start `IphoneOSCListener` in `_start_pose_worker` regardless of any --flag (always-on, harmless if no iPhone). | +| `data_only_viz/tests/test_iphone_osc_listener.py` | **NEW**. Unit tests: send fake OSC packets, assert state updated. | +| `data_only_viz/tests/test_arkit_fuse.py` | **NEW**. Unit tests: fake state, run PoseFilterChain.apply, assert MP33 slots overwritten. | +| `data_only_viz/tests/test_multihmr_arkit_z.py` | **NEW**. Unit test: fake ARKit pelvis z, assert pred_cam_t corrected. | + +--- + +## Task 1: State fields for ARKit joints + +**Files:** +- Modify: `data_only_viz/state.py` (add fields) +- Test: `data_only_viz/tests/test_state_arkit.py` (new) + +- [ ] **Step 1: Write the failing test** + +Create `data_only_viz/tests/test_state_arkit.py`: + +```python +"""State must expose persons_arkit_joints + persons_arkit_last_t.""" +import numpy as np + +from data_only_viz.state import State + + +def test_state_has_arkit_joint_fields(): + s = State() + assert hasattr(s, "persons_arkit_joints") + assert hasattr(s, "persons_arkit_last_t") + assert isinstance(s.persons_arkit_joints, dict) + assert isinstance(s.persons_arkit_last_t, dict) + + +def test_state_arkit_joints_writable_under_lock(): + s = State() + arr = np.zeros((91, 3), dtype=np.float32) + with s.lock(): + s.persons_arkit_joints[0] = arr + s.persons_arkit_last_t[0] = 1.5 + assert 0 in s.persons_arkit_joints + assert s.persons_arkit_last_t[0] == 1.5 +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_state_arkit.py -v` +Expected: FAIL with `AttributeError: 'State' object has no attribute 'persons_arkit_joints'` + +- [ ] **Step 3: Add fields to State** + +Edit `data_only_viz/state.py`. Find the existing line with `persons_hands_mesh_last_t: float = 0.0` (around line 134) and insert below it: + +```python + # ARKit body tracking (iOS ARBodyTracker app) : 91 joints world + # space per pid. Same units as MediaPipe pose_world_landmarks + # (metres, hip-centered). Fresh = updated within < 1 s. + persons_arkit_joints: dict = field(default_factory=dict) + persons_arkit_last_t: dict = field(default_factory=dict) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_state_arkit.py -v` +Expected: PASS (2 passed) + +- [ ] **Step 5: Commit** + +```bash +cd /Users/electron/Documents/Projets/AV-Live +git add data_only_viz/state.py data_only_viz/tests/test_state_arkit.py +git commit -m "feat(state): add persons_arkit_joints + persons_arkit_last_t" +``` + +--- + +## Task 2: ARKit → MediaPipe joint index mapping + +**Files:** +- Create: `data_only_viz/arkit_joint_map.py` +- Test: `data_only_viz/tests/test_arkit_joint_map.py` (new) + +- [ ] **Step 1: Write the failing test** + +Create `data_only_viz/tests/test_arkit_joint_map.py`: + +```python +"""ARKit 91 joints → MediaPipe Pose 33 mapping integrity.""" +from data_only_viz.arkit_joint_map import ( + ARKIT91_TO_MP33, ARKIT_PELVIS_IDX, MP33_NUM_LANDMARKS, +) + + +def test_mapping_is_tuple_of_pairs(): + assert isinstance(ARKIT91_TO_MP33, tuple) + assert len(ARKIT91_TO_MP33) > 0 + for pair in ARKIT91_TO_MP33: + assert isinstance(pair, tuple) + assert len(pair) == 2 + + +def test_mapping_indices_in_range(): + for arkit_idx, mp33_idx in ARKIT91_TO_MP33: + assert 0 <= arkit_idx < 91, f"arkit idx out of range: {arkit_idx}" + assert 0 <= mp33_idx < MP33_NUM_LANDMARKS, \ + f"mp33 idx out of range: {mp33_idx}" + + +def test_pelvis_index_valid(): + assert 0 <= ARKIT_PELVIS_IDX < 91 + + +def test_no_duplicate_mp33_targets(): + """Each MediaPipe slot must be written by at most one ARKit joint.""" + mp33_seen = set() + for _, mp33_idx in ARKIT91_TO_MP33: + assert mp33_idx not in mp33_seen, \ + f"mp33 slot {mp33_idx} mapped twice" + mp33_seen.add(mp33_idx) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_arkit_joint_map.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'data_only_viz.arkit_joint_map'` + +- [ ] **Step 3: Create the mapping module** + +Create `data_only_viz/arkit_joint_map.py`: + +```python +"""ARKit ARSkeleton3D 91-joint indices → MediaPipe Pose 33 indices. + +The ARKit ARSkeleton.JointName enum (Apple SDK) orders 91 joints +starting with the root, hips, spine chain, shoulders, etc. We pick +only the joints with a clear 1:1 anatomical correspondence to the +MediaPipe Pose 33 landmark set (which is what AVLiveBody renders). +Face/hand sub-joints (fingers, eyes) are skipped — those keep their +existing data sources (MediaPipe Face/Hand + HaMeR MANO). + +Reference for ARKit joint order : Apple developer docs +"ARSkeleton.JointName" — the canonical 91-joint list runs from +root_joint=0 down to right_handThumbEndJoint=90. + +The selection here mirrors `multi.py::SMPLX_TO_MP33` so the same 14 +body slots are overridden by ARKit when fresh. Confidence comes +from ARKit's tracking state but is not currently fanned out — we +trust ARKit body tracking when its OSC frame is present. +""" +from __future__ import annotations + +# MediaPipe Pose 33 cardinality (cf. mediapipe pose_world_landmarks). +MP33_NUM_LANDMARKS = 33 + +# Pelvis = ARKit hips_joint, slot 1 in the canonical enum order. +# Used by multi_hmr_worker for cam-translation z lock. +ARKIT_PELVIS_IDX = 1 + +# (arkit_joint_idx, mediapipe_pose_idx). Match the body slots used +# by the SMPL-X body fusion in multi.py. +ARKIT91_TO_MP33: tuple[tuple[int, int], ...] = ( + (50, 11), # left_shoulder_1_joint -> L_SHOULDER + (32, 12), # right_shoulder_1_joint -> R_SHOULDER + (53, 13), # left_arm_joint -> L_ELBOW + (35, 14), # right_arm_joint -> R_ELBOW + (54, 15), # left_forearm_joint -> L_WRIST + (36, 16), # right_forearm_joint -> R_WRIST + (62, 23), # left_upLeg_joint -> L_HIP + (57, 24), # right_upLeg_joint -> R_HIP + (63, 25), # left_leg_joint -> L_KNEE + (58, 26), # right_leg_joint -> R_KNEE + (64, 27), # left_foot_joint -> L_ANKLE + (59, 28), # right_foot_joint -> R_ANKLE + (65, 31), # left_toes_joint -> L_FOOT_INDEX + (60, 32), # right_toes_joint -> R_FOOT_INDEX +) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_arkit_joint_map.py -v` +Expected: PASS (4 passed) + +- [ ] **Step 5: Commit** + +```bash +git add data_only_viz/arkit_joint_map.py data_only_viz/tests/test_arkit_joint_map.py +git commit -m "feat(viz): arkit 91-joint -> mediapipe 33 mapping" +``` + +--- + +## Task 3: iPhone OSC listener worker + +**Files:** +- Create: `data_only_viz/iphone_osc_listener.py` +- Test: `data_only_viz/tests/test_iphone_osc_listener.py` (new) + +- [ ] **Step 1: Write the failing test** + +Create `data_only_viz/tests/test_iphone_osc_listener.py`: + +```python +"""IphoneOSCListener writes ARKit joints to state from OSC packets.""" +import time + +import numpy as np +import pytest +from pythonosc.udp_client import SimpleUDPClient + +from data_only_viz.state import State +from data_only_viz.iphone_osc_listener import ( + IphoneOSCListener, IPHONE_OSC_PORT, +) + + +@pytest.fixture() +def listener(): + state = State() + listener = IphoneOSCListener(state, port=IPHONE_OSC_PORT + 100) + listener.start() + yield state, listener + listener.stop() + + +def test_kp_message_updates_state(listener): + state, lst = listener + client = SimpleUDPClient("127.0.0.1", lst.port) + client.send_message("/body3d/kp", [0, 1, 0.1, 0.2, 0.3]) + # Settle + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline: + with state.lock(): + if 0 in state.persons_arkit_joints: + arr = state.persons_arkit_joints[0] + if arr[1, 0] != 0.0: + break + time.sleep(0.02) + with state.lock(): + arr = state.persons_arkit_joints[0] + assert arr.shape == (91, 3) + assert np.allclose(arr[1], [0.1, 0.2, 0.3]) + + +def test_gc_drops_stale_pids(listener): + state, lst = listener + with state.lock(): + state.persons_arkit_joints[7] = np.zeros((91, 3), dtype=np.float32) + state.persons_arkit_last_t[7] = time.perf_counter() - 5.0 + lst._gc_stale() + with state.lock(): + assert 7 not in state.persons_arkit_joints +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_iphone_osc_listener.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'data_only_viz.iphone_osc_listener'` + +- [ ] **Step 3: Implement listener** + +Create `data_only_viz/iphone_osc_listener.py`: + +```python +"""OSC UDP listener for the iOS ARBodyTracker app. + +Subscribes to /body3d/kp on UDP :57128 (distinct from MediaPipe +output :57126). Each /body3d/kp pid joint_idx x y z message stores +one joint of ARKit's 91-joint ARSkeleton3D into +state.persons_arkit_joints[pid] (np.ndarray shape (91, 3), float32). +A background GC drops pids whose last_t is older than 1.0 s. + +Worker pattern mirrors osc_listener.OscListener. +""" +from __future__ import annotations + +import logging +import threading +import time +from typing import Any + +import numpy as np +from pythonosc import dispatcher, osc_server + +from .state import State + +LOG = logging.getLogger("iphone_osc") + +IPHONE_OSC_PORT = 57128 +ARKIT_NUM_JOINTS = 91 +STALE_SEC = 1.0 + + +class IphoneOSCListener: + def __init__(self, state: State, host: str = "0.0.0.0", + port: int = IPHONE_OSC_PORT) -> None: + self.state = state + self.host = host + self.port = port + self._server: osc_server.ThreadingOSCUDPServer | None = None + self._server_thread: threading.Thread | None = None + self._gc_thread: threading.Thread | None = None + self._stop = threading.Event() + + def start(self) -> None: + d = dispatcher.Dispatcher() + d.map("/body3d/kp", self._on_kp) + d.map("/body3d/count", self._on_count) + self._server = osc_server.ThreadingOSCUDPServer( + (self.host, self.port), d) + self._server_thread = threading.Thread( + target=self._server.serve_forever, + name="iphone_osc", daemon=True) + self._server_thread.start() + self._gc_thread = threading.Thread( + target=self._gc_loop, name="iphone_gc", daemon=True) + self._gc_thread.start() + LOG.info("iphone OSC listening on %s:%d", self.host, self.port) + + def stop(self) -> None: + self._stop.set() + if self._server is not None: + self._server.shutdown() + self._server.server_close() + self._server = None + + def _on_kp(self, _addr: str, *args: Any) -> None: + if len(args) < 5: + return + try: + pid = int(args[0]) + joint_idx = int(args[1]) + x = float(args[2]) + y = float(args[3]) + z = float(args[4]) + except (TypeError, ValueError): + return + if not (0 <= joint_idx < ARKIT_NUM_JOINTS): + return + with self.state.lock(): + arr = self.state.persons_arkit_joints.get(pid) + if arr is None or arr.shape != (ARKIT_NUM_JOINTS, 3): + arr = np.zeros((ARKIT_NUM_JOINTS, 3), dtype=np.float32) + self.state.persons_arkit_joints[pid] = arr + arr[joint_idx] = (x, y, z) + self.state.persons_arkit_last_t[pid] = time.perf_counter() + + def _on_count(self, _addr: str, *args: Any) -> None: + # Optional : we currently don't gate on count, but parse for log. + if not args: + return + try: + n = int(args[0]) + except (TypeError, ValueError): + return + if not hasattr(self, "_last_hb") or \ + time.monotonic() - self._last_hb > 5.0: + self._last_hb = time.monotonic() + LOG.info("hb: %d ARKit bodies live", n) + + def _gc_stale(self) -> None: + cutoff = time.perf_counter() - STALE_SEC + with self.state.lock(): + drop = [ + pid for pid, t in self.state.persons_arkit_last_t.items() + if t < cutoff + ] + for pid in drop: + self.state.persons_arkit_joints.pop(pid, None) + self.state.persons_arkit_last_t.pop(pid, None) + + def _gc_loop(self) -> None: + while not self._stop.is_set(): + self._gc_stale() + time.sleep(0.5) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_iphone_osc_listener.py -v` +Expected: PASS (2 passed) + +- [ ] **Step 5: Commit** + +```bash +git add data_only_viz/iphone_osc_listener.py data_only_viz/tests/test_iphone_osc_listener.py +git commit -m "feat(viz): iphone OSC listener -> state.persons_arkit_joints" +``` + +--- + +## Task 4: ArkitFuse stage in PoseFilterChain + +**Files:** +- Modify: `data_only_viz/pose_filter.py` +- Test: `data_only_viz/tests/test_arkit_fuse.py` (new) + +- [ ] **Step 1: Write the failing test** + +Create `data_only_viz/tests/test_arkit_fuse.py`: + +```python +"""ArkitFuse stage overrides 14 body slots with ARKit data when fresh.""" +import time + +import numpy as np + +from data_only_viz.state import Kp3D, State +from data_only_viz.pose_filter import PoseFilterChain + + +def _mp33_zero_body(): + return [Kp3D(x=0.0, y=0.0, z=0.0, c=1.0) for _ in range(33)] + + +def test_arkit_fuse_overrides_shoulder(): + state = State() + # ARKit publishes joint 50 (left shoulder) with (1.0, 2.0, 3.0) + arr = np.zeros((91, 3), dtype=np.float32) + arr[50] = (1.0, 2.0, 3.0) + with state.lock(): + state.persons_arkit_joints[0] = arr + state.persons_arkit_last_t[0] = time.perf_counter() + chain = PoseFilterChain(state=state, enabled_stages=("arkit_fuse",)) + bodies = [_mp33_zero_body()] + out = chain.apply(bodies, ids=[0], t_now=time.perf_counter()) + # Slot 11 = L_SHOULDER (from ARKIT91_TO_MP33). + assert out[0][11].x == 1.0 + assert out[0][11].y == 2.0 + assert out[0][11].z == 3.0 + + +def test_arkit_fuse_skips_stale(): + state = State() + arr = np.zeros((91, 3), dtype=np.float32) + arr[50] = (9.0, 9.0, 9.0) + with state.lock(): + state.persons_arkit_joints[0] = arr + state.persons_arkit_last_t[0] = time.perf_counter() - 5.0 + chain = PoseFilterChain(state=state, enabled_stages=("arkit_fuse",)) + bodies = [_mp33_zero_body()] + out = chain.apply(bodies, ids=[0], t_now=time.perf_counter()) + # Stale -> not applied, MediaPipe zero left intact. + assert out[0][11].x == 0.0 +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_arkit_fuse.py -v` +Expected: FAIL — `arkit_fuse` not in `ALL_STAGES` so chain.enabled is empty, no fuse happens. + +- [ ] **Step 3: Add ArkitFuse class + register stage** + +Edit `data_only_viz/pose_filter.py`. Find the line `ALL_STAGES = (...)` near the top and replace: + +```python +ALL_STAGES = ( + "median", "kalman", "spring", "lookahead", "ik", + "one_euro_joints", "one_euro_bones", "arkit_fuse", +) +``` + +Find the import block at the top (after `from .euro_filter import ...`) and add: + +```python +from .arkit_joint_map import ARKIT91_TO_MP33 +``` + +Find the `PoseFilterChain.__init__` method and after the line `self.one_euro_bones = BoneOneEuroFilter(...)` add: + +```python + self.arkit_fuse = ArkitFuse() +``` + +In `PoseFilterChain.apply`, find the block defining `use_one_euro_joints = "one_euro_joints" in self.enabled` and add right after it: + +```python + use_arkit_fuse = "arkit_fuse" in self.enabled +``` + +In the same method, find the outer `for body_i, kps in enumerate(bodies3d):` loop. The fuse happens BEFORE per-joint filtering (so kalman sees the fused signal). Insert this immediately after the `pid = ids[body_i] if body_i < len(ids) else -1` line: + +```python + if use_arkit_fuse and self.state is not None: + kps = self.arkit_fuse.apply(self.state, pid, kps, t_now) +``` + +Then add the `ArkitFuse` class definition. Find the line `# ============================ face / hand =================================` and insert right BEFORE it: + +```python +class ArkitFuse: + """Splice ARKit 91-joint world-space data into MediaPipe Pose 33. + + Reads ``state.persons_arkit_joints[pid]`` (shape (91, 3)) when fresh + (last_t within FRESH_SEC). Writes the 14 body slots covered by + ARKIT91_TO_MP33 ; everything else (face landmarks, finger tips) + stays MediaPipe-driven. + """ + + FRESH_SEC: float = 1.0 + + def apply(self, state: "State", pid: int, + kps: list[Kp3D], t_now: float) -> list[Kp3D]: + with state.lock(): + arr = state.persons_arkit_joints.get(pid) + last_t = state.persons_arkit_last_t.get(pid, 0.0) + if arr is None: + return kps + if t_now - last_t > self.FRESH_SEC: + return kps + out = list(kps) + n = len(out) + for arkit_idx, mp33_idx in ARKIT91_TO_MP33: + if mp33_idx >= n: + continue + x = float(arr[arkit_idx, 0]) + y = float(arr[arkit_idx, 1]) + z = float(arr[arkit_idx, 2]) + old = out[mp33_idx] + out[mp33_idx] = Kp3D(x=x, y=y, z=z, c=getattr(old, "c", 1.0)) + return out +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_arkit_fuse.py -v` +Expected: PASS (2 passed) + +- [ ] **Step 5: Regression check existing filter tests** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_pose_filter.py -v` +Expected: PASS (all existing pose_filter tests still green) + +- [ ] **Step 6: Commit** + +```bash +git add data_only_viz/pose_filter.py data_only_viz/tests/test_arkit_fuse.py +git commit -m "feat(viz): arkit_fuse stage in PoseFilterChain" +``` + +--- + +## Task 5: Cam-translation z lock in multi_hmr_worker + +**Files:** +- Modify: `data_only_viz/multi_hmr_worker.py` +- Test: `data_only_viz/tests/test_multihmr_arkit_z.py` (new) + +- [ ] **Step 1: Locate post-inference cam_t write site** + +Run: `grep -n "pred_cam_t\|cam_t =" /Users/electron/Documents/Projets/AV-Live/data_only_viz/multi_hmr_worker.py | head -15` + +You will see lines where `pred_cam_t` is read from model output and copied into `state.persons_smplx`. Look for the loop after model inference that assigns `transl=...` or `translation=...` per pid. Save the exact line numbers — Step 3 inserts the lock just AFTER the existing cam_t computation but BEFORE the state write. + +- [ ] **Step 2: Write the failing test** + +Create `data_only_viz/tests/test_multihmr_arkit_z.py`: + +```python +"""arkit_pelvis_z_override : if ARKit pelvis z is fresh, replace +the Multi-HMR pred_cam_t.z so the SMPL-X mesh sits at the actual +distance instead of HaMeR's monocular guess. +""" +import time + +import numpy as np + +from data_only_viz.state import State +from data_only_viz.multi_hmr_worker import arkit_pelvis_z_override + + +def test_returns_arkit_z_when_fresh(): + state = State() + arr = np.zeros((91, 3), dtype=np.float32) + arr[1] = (0.0, 0.0, 2.5) # ARKIT_PELVIS_IDX=1, z=2.5 m + with state.lock(): + state.persons_arkit_joints[0] = arr + state.persons_arkit_last_t[0] = time.perf_counter() + z_pred = 5.0 # Multi-HMR ambiguous guess + z_out = arkit_pelvis_z_override(state, pid=0, z_pred=z_pred) + assert z_out == 2.5 + + +def test_keeps_pred_when_stale(): + state = State() + arr = np.zeros((91, 3), dtype=np.float32) + arr[1] = (0.0, 0.0, 2.5) + with state.lock(): + state.persons_arkit_joints[0] = arr + state.persons_arkit_last_t[0] = time.perf_counter() - 5.0 + z_pred = 5.0 + z_out = arkit_pelvis_z_override(state, pid=0, z_pred=z_pred) + assert z_out == 5.0 + + +def test_keeps_pred_when_pid_missing(): + state = State() + z_pred = 4.2 + z_out = arkit_pelvis_z_override(state, pid=99, z_pred=z_pred) + assert z_out == 4.2 +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_multihmr_arkit_z.py -v` +Expected: FAIL — `arkit_pelvis_z_override` does not exist in multi_hmr_worker. + +- [ ] **Step 4: Add the override function + import** + +Edit `data_only_viz/multi_hmr_worker.py`. At the top of the file, add to the imports block: + +```python +from .arkit_joint_map import ARKIT_PELVIS_IDX +``` + +Then at module level (after imports, before any class), add: + +```python +def arkit_pelvis_z_override(state, pid: int, z_pred: float, + fresh_sec: float = 1.0) -> float: + """Return ARKit pelvis world-z if a fresh ARKit frame exists for + this pid, otherwise return the Multi-HMR predicted z unchanged. + + Used to resolve Multi-HMR's monocular scale ambiguity: ARKit's + LiDAR-anchored pelvis position is ground truth in the iPhone + world frame, which (after extrinsics calibration) is the same + metric scale as the SMPL-X cam-space output. + """ + import time as _time + with state.lock(): + arr = state.persons_arkit_joints.get(pid) + last_t = state.persons_arkit_last_t.get(pid, 0.0) + if arr is None: + return float(z_pred) + if _time.perf_counter() - last_t > fresh_sec: + return float(z_pred) + return float(arr[ARKIT_PELVIS_IDX, 2]) +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/test_multihmr_arkit_z.py -v` +Expected: PASS (3 passed) + +- [ ] **Step 6: Wire override at the cam_t write site** + +Now use the line numbers from Step 1. In `multi_hmr_worker.py`, find the per-person loop where `pred_cam_t` (or equivalent transl) is being written into the SMPL-X person record. For each person, after the model output's z is computed but before assignment to state, wrap the z value: + +```python + z_locked = arkit_pelvis_z_override( + self.state, pid, float(transl[2])) + transl = np.array([transl[0], transl[1], z_locked], + dtype=transl.dtype) +``` + +(Adapt variable names to the exact context — `transl`, `t_full`, or whatever is used. The pattern is: take the existing z, replace via the override, repack.) + +- [ ] **Step 7: Regression smoke** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && ~/avlive-venv/bin/python -m pytest data_only_viz/tests/ -q -x` +Expected: All tests pass (no regressions introduced). + +- [ ] **Step 8: Commit** + +```bash +git add data_only_viz/multi_hmr_worker.py data_only_viz/tests/test_multihmr_arkit_z.py +git commit -m "feat(viz): arkit pelvis z locks Multi-HMR cam translation" +``` + +--- + +## Task 6: Always-on listener in main.py + +**Files:** +- Modify: `data_only_viz/main.py` + +- [ ] **Step 1: Locate _start_pose_worker function** + +Run: `grep -n "_start_pose_worker\|_maybe_start_webcam_source" /Users/electron/Documents/Projets/AV-Live/data_only_viz/main.py | head -5` + +Note the line number where `_start_pose_worker` begins (around line 287). + +- [ ] **Step 2: Insert listener startup** + +Edit `data_only_viz/main.py`. In the body of `_start_pose_worker`, immediately after the call to `self._maybe_start_webcam_source()` (line ~289), insert: + +```python + # iPhone ARBodyTracker (option 2 LiDAR fusion) : always-on + # listener on :57128. Harmless if no iPhone is broadcasting ; + # state.persons_arkit_joints stays empty and the arkit_fuse + # stage no-ops. Activated via POSE_FILTER=...+arkit_fuse. + try: + from .iphone_osc_listener import IphoneOSCListener + self._iphone_osc = IphoneOSCListener(self._state) + self._iphone_osc.start() + LOG.info("worker: + iPhone OSC listener :57128") + except Exception as e: # noqa: BLE001 + LOG.warning("iphone OSC listener start failed (%s)", e) +``` + +- [ ] **Step 3: Smoke test the listener starts** + +Run: `cd /Users/electron/Documents/Projets/AV-Live && AV_LIVE_INFERENCE_OFF=1 timeout 8 ~/avlive-venv/bin/python -u -c " +import os, sys, time +os.environ.setdefault('AV_SHARED_CAM', '0') +sys.path.insert(0, '/Users/electron/Documents/Projets/AV-Live') +import threading, logging +logging.basicConfig(level=logging.INFO) +from data_only_viz.state import State +from data_only_viz.iphone_osc_listener import IphoneOSCListener +s = State() +l = IphoneOSCListener(s) +l.start() +time.sleep(2) +print('listener up :', l.port) +l.stop() +print('stopped clean') +" 2>&1 | tail -5` + +Expected output: +``` +... iphone OSC listening on 0.0.0.0:57128 +listener up : 57128 +stopped clean +``` + +- [ ] **Step 4: Commit** + +```bash +git add data_only_viz/main.py +git commit -m "feat(viz): start iphone OSC listener in main pose worker" +``` + +--- + +## Task 7: Documentation update + +**Files:** +- Modify: `CLAUDE.md` (top-level env var table) +- Modify: `data_only_viz/CLAUDE.md` (POSE_FILTER stages table) + +- [ ] **Step 1: Update top-level CLAUDE.md env var table** + +Edit `/Users/electron/Documents/Projets/AV-Live/CLAUDE.md`. Find the row with `POSE_FILTER` and replace its description so it lists `arkit_fuse` as an available stage. Also append a new row for the iPhone OSC port. Use these exact replacements: + +For the `POSE_FILTER` row, replace whatever is there with: + +``` +| `POSE_FILTER` | `median+kalman+lookahead+ik` | filter chain stages — extra: `one_euro_joints` (joint-space CHI 2012 One Euro, inserted before kalman), `one_euro_bones` (bone-vector One Euro applied after SMPL-X fusion in multi.py), `arkit_fuse` (overrides 14 body slots with ARKit ARSkeleton3D from the iOS app, expects /body3d/kp on :57128) | +``` + +Then below the `POSE_FILTER` row add: + +``` +| `IPHONE_OSC_PORT` | `57128` | UDP port the iPhone ARBodyTracker app pushes /body3d/kp to (always-on listener in data_only_viz) | +``` + +- [ ] **Step 2: Update data_only_viz/CLAUDE.md** + +Edit `/Users/electron/Documents/Projets/AV-Live/data_only_viz/CLAUDE.md`. Find the "Conventions" section's filtering bullet (mentions `euro_filter.py`) and append after it: + +``` +- ARKit fusion : `iphone_osc_listener.py` consume /body3d/kp UDP :57128 + → `state.persons_arkit_joints`. `pose_filter.py::ArkitFuse` (stage + `arkit_fuse`) splices the 14 mapped body slots into MediaPipe pose + before kalman ; `multi_hmr_worker::arkit_pelvis_z_override` locks the + SMPL-X cam translation z to the ARKit pelvis. Mapping in + `arkit_joint_map.py`. +``` + +- [ ] **Step 3: Commit** + +```bash +git add CLAUDE.md data_only_viz/CLAUDE.md +git commit -m "docs: iphone arkit fusion env + filter stage" +``` + +--- + +## Task 8: End-to-end live smoke + +This is a manual verification step run once after the iOS app is +deployed to a real iPhone Pro and broadcasting on the LAN. No new +code ; just confirm wiring + telemetry. + +- [ ] **Step 1: Start the GrosMac pipeline (already wired)** + +Run: `bash /Users/electron/Documents/Projets/AV-Live/launcher/apps/dist/GrosMac-AVLive.app/Contents/MacOS/bootstrap &` + +Wait ~8 s, then verify the listener line appeared: + +``` +grep "iphone OSC listening" ~/Library/Logs/AVLive/GrosMac-AVLive.python.log +``` + +Expected: a line `iphone OSC listening on 0.0.0.0:57128`. + +- [ ] **Step 2: Start ARBodyTracker on iPhone** + +In the iOS app (deployed via Xcode): +1. Host = your GrosMac LAN IP (`192.168.0.159`) +2. Port = `57128` +3. Tap **Start** + +Stand 2 m in front of the iPhone with body fully visible. The app +status label should say "running (LiDAR depth, env mesh)". + +- [ ] **Step 3: Confirm ARKit state on GrosMac** + +Run on GrosMac while iPhone is broadcasting: + +```bash +~/avlive-venv/bin/python -u -c " +import time, sys +sys.path.insert(0, '/Users/electron/Documents/Projets/AV-Live') +from data_only_viz.iphone_osc_listener import IphoneOSCListener +from data_only_viz.state import State +s = State() +l = IphoneOSCListener(s, port=57130) # alt port to avoid clash +l.start() +time.sleep(3) +with s.lock(): + print('pids :', list(s.persons_arkit_joints.keys())) + if s.persons_arkit_joints: + pid = next(iter(s.persons_arkit_joints)) + print('pelvis :', s.persons_arkit_joints[pid][1]) +l.stop() +" +``` + +Expected: `pids : [0]` and `pelvis : [x, y, z]` with z > 0. + +NOTE: this snippet uses port 57130 to avoid clashing with the live +listener already bound to 57128. To test against the live listener, +just open Activity Monitor's network panel for the Python process — +you should see UDP packets flowing in on :57128. + +- [ ] **Step 4: Enable arkit_fuse in live pipeline** + +The pipeline currently uses `POSE_FILTER` from the bootstrap env. To +add `arkit_fuse`, edit the GrosMac bootstrap and append the stage: + +Edit `/Users/electron/Documents/Projets/AV-Live/launcher/apps/dist/GrosMac-AVLive.app/Contents/MacOS/bootstrap`. Find any existing `export POSE_FILTER=...` line (if absent, look around the `if [ "${ROLE}" = "source" ]; then` section for where envs are exported in the source branch) and add: + +```bash + export POSE_FILTER="median+kalman+lookahead+ik+one_euro_joints+one_euro_bones+arkit_fuse" +``` + +Restart GrosMac bundle: + +```bash +pkill -9 -f "AVLiveBody|data_only_viz" +open /Users/electron/Documents/Projets/AV-Live/launcher/apps/dist/GrosMac-AVLive.app +``` + +- [ ] **Step 5: Verify fusion in log** + +After the live launch, tail the python log: + +```bash +grep "PoseFilterChain stages" ~/Library/Logs/AVLive/GrosMac-AVLive.python.log +``` + +Expected: a line ending in `'arkit_fuse')`. + +- [ ] **Step 6: Visual confirmation** + +In AVLiveBody window (release build), the body wireframe should +visibly stabilise compared to a session without ARKit (shoulders/hips +no longer wobble between MediaPipe predictions ; the mesh sits at +the real-world depth from the camera instead of HaMeR's monocular +guess). If you see persistent jitter, double-check via Activity +Monitor that UDP :57128 traffic is non-zero, and that +`state.persons_arkit_joints` has fresh entries (Step 3 snippet). + +--- + +## Self-Review + +Spec coverage check : +- iphone_osc_listener.py ✅ Task 3 +- state fields ✅ Task 1 +- arkit_joint_map.py ✅ Task 2 +- pose_filter arkit_fuse ✅ Task 4 +- multi_hmr cam-z lock ✅ Task 5 +- main.py startup ✅ Task 6 +- Docs ✅ Task 7 +- Live verification ✅ Task 8 + +ICP mesh fitting is intentionally deferred — that's a separate plan +once the joint-level fusion is proven stable. + +Type consistency : `ARKIT91_TO_MP33` declared in Task 2, used in +Task 4 (pose_filter import) and Task 5 (multi_hmr import of +`ARKIT_PELVIS_IDX`). `IphoneOSCListener` defined Task 3, instantiated +Task 6. State fields `persons_arkit_joints` and +`persons_arkit_last_t` declared Task 1, consumed Tasks 3, 4, 5. + +No placeholders, no TBD, every step is concrete with code or a +copy-pasteable command. The plan compiles a hot-loop story without +ICP, which keeps it bite-sized and shippable in a single working +session (~1 day for a fresh engineer). diff --git a/docs/superpowers/plans/2026-05-18-avlivebody-macos-rewrite.md b/docs/superpowers/plans/2026-05-18-avlivebody-macos-rewrite.md new file mode 100644 index 0000000..4ec8c06 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-avlivebody-macos-rewrite.md @@ -0,0 +1,1117 @@ +# AVLiveBody macOS Rewrite Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a fresh native macOS Xcode app, `avlivebody-mac/`, that consumes the iPhone-USB body pipeline and renders the camera video + 91-joint skeleton + SMPL-X mesh in one RealityKit 3D scene — no legacy code. + +**Architecture:** A SwiftUI app whose single window hosts one RealityKit `ARView` (used as a plain 3D view). The proven USB pipeline components (`AVLiveWire`, usbmux client, `VideoDecoder`, `USBSkeletonConsumer`, `MultiHMRCoreML`, `BodyFusion`) are migrated in; new clean rendering units (`SceneController`, `VideoQuad`, `SkeletonEntity`, `MeshEntity`) build the scene. The old `launcher/AV-Live-Body` is archived. + +**Tech Stack:** Swift 5, macOS 15, Xcode + xcodegen, RealityKit, CoreML, VideoToolbox, the local `AVLiveWire` SwiftPM package, `XCTest`. + +**Companion spec:** `docs/superpowers/specs/2026-05-18-avlivebody-macos-rewrite-design.md` + +--- + +## Verification + +The app is a real Xcode project. Per task: + +```bash +cd avlivebody-mac && xcodegen generate && \ +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' -configuration Debug build +``` + +Expected: `** BUILD SUCCEEDED **`. Unit tests (migrated) run with the +`xcodebuild ... test` action. + +--- + +## File Structure + +``` +avlivebody-mac/ + project.yml xcodegen manifest + Config/Shared.xcconfig committed build settings + Config/Local.xcconfig.example DEVELOPMENT_TEAM template + Sources/AVLiveBody/ + AVLiveBodyApp.swift @main + AppDelegate + SceneView.swift NSViewRepresentable -> ARView + SceneController.swift owns scene/camera/entities + VideoQuad.swift video-textured back plane + SkeletonEntity.swift 91 joint markers + MeshEntity.swift SMPL-X 10475-vertex mesh + StatusBar.swift connection-status overlay + Info.plist + usb/USBMuxProtocol.swift migrated verbatim + usb/USBClient.swift migrated verbatim + usb/VideoDecoder.swift migrated verbatim + usb/USBSkeletonConsumer.swift migrated, cleaned + usb/MultiHMRCoreML.swift migrated verbatim + usb/BodyFusion.swift migrated, cleaned + Resources/smplx_faces.bin SMPL-X face indices + Resources/multihmr_full_672_s.mlpackage bundled model (gitignored) + Tests/AVLiveBodyTests/ migrated unit tests +``` + +`AVLiveWire` stays in `shared/AVLiveWire`; the app depends on it. + +--- + +## Task 1: Scaffold the Xcode project + +**Files:** +- Create: `avlivebody-mac/project.yml` +- Create: `avlivebody-mac/Config/Shared.xcconfig` +- Create: `avlivebody-mac/Config/Local.xcconfig.example` +- Create: `avlivebody-mac/Sources/AVLiveBody/AVLiveBodyApp.swift` +- Create: `avlivebody-mac/Sources/AVLiveBody/Info.plist` + +- [ ] **Step 1: Create `avlivebody-mac/project.yml`** + +```yaml +name: AVLiveBody +options: + bundleIdPrefix: cc.saillant + deploymentTarget: + macOS: "15.0" + createIntermediateGroups: true + +configFiles: + Debug: Config/Shared.xcconfig + Release: Config/Shared.xcconfig + +packages: + AVLiveWire: + path: ../shared/AVLiveWire + +targets: + AVLiveBody: + type: application + platform: macOS + deploymentTarget: "15.0" + sources: + - path: Sources/AVLiveBody + excludes: + - Info.plist + dependencies: + - package: AVLiveWire + product: AVLiveWire + configFiles: + Debug: Config/Shared.xcconfig + Release: Config/Shared.xcconfig + settings: + base: + PRODUCT_NAME: AVLiveBody + PRODUCT_BUNDLE_IDENTIFIER: cc.saillant.AVLiveBody + INFOPLIST_FILE: Sources/AVLiveBody/Info.plist + GENERATE_INFOPLIST_FILE: NO + CODE_SIGN_STYLE: Automatic + SWIFT_VERSION: "5.10" + ENABLE_HARDENED_RUNTIME: YES + AVLiveBodyTests: + type: bundle.unit-test + platform: macOS + sources: + - path: Tests/AVLiveBodyTests + dependencies: + - target: AVLiveBody + - package: AVLiveWire + product: AVLiveWire +``` + +- [ ] **Step 2: Create the config files** + +`avlivebody-mac/Config/Shared.xcconfig`: + +``` +#include? "Local.xcconfig" + +MACOSX_DEPLOYMENT_TARGET = 15.0 +SWIFT_VERSION = 5.10 +CODE_SIGN_STYLE = Automatic +``` + +`avlivebody-mac/Config/Local.xcconfig.example`: + +``` +// Copy to Config/Local.xcconfig and set your Apple Developer Team ID. +// Config/Local.xcconfig is gitignored. +DEVELOPMENT_TEAM = YOUR_TEAM_ID +``` + +- [ ] **Step 3: Create the minimal app + Info.plist** + +`avlivebody-mac/Sources/AVLiveBody/AVLiveBodyApp.swift`: + +```swift +import Cocoa +import SwiftUI + +/// Forces a regular, keyboard-focusable foreground app. +final class AppDelegate: NSObject, NSApplicationDelegate { + func applicationDidFinishLaunching(_ notification: Notification) { + NSApp.setActivationPolicy(.regular) + NSApp.activate() + } +} + +@main +struct AVLiveBodyApp: App { + @NSApplicationDelegateAdaptor(AppDelegate.self) + private var appDelegate + + var body: some Scene { + WindowGroup { + Text("AVLiveBody") + .frame(minWidth: 900, minHeight: 600) + } + } +} +``` + +`avlivebody-mac/Sources/AVLiveBody/Info.plist`: + +```xml + + + + + CFBundleNameAVLiveBody + CFBundleIdentifier$(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleExecutable$(EXECUTABLE_NAME) + CFBundlePackageTypeAPPL + CFBundleShortVersionString1.0 + CFBundleVersion1 + LSMinimumSystemVersion15.0 + NSCameraUsageDescription + Receives the tethered iPhone camera over USB. + NSLocalNetworkUsageDescription + Connects to the tethered iPhone over USB (usbmuxd). + + +``` + +- [ ] **Step 4: Generate and build** + +```bash +cd avlivebody-mac && xcodegen generate && \ +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' -configuration Debug build +``` + +Expected: `** BUILD SUCCEEDED **`, an empty window app. + +- [ ] **Step 5: Commit** (the generated `.xcodeproj` is gitignored — add an `avlivebody-mac/.gitignore` with `*.xcodeproj/` and `Config/Local.xcconfig`) + +```bash +git add avlivebody-mac/project.yml avlivebody-mac/Config avlivebody-mac/Sources avlivebody-mac/.gitignore +git commit -m "feat(avlivebody-mac): scaffold xcode app" +``` + +--- + +## Task 2: Migrate the USB transport files + +These files are copied verbatim from `launcher/AV-Live-Body/Sources/AVLiveBody/` — they are self-contained and depend only on `AVLiveWire` + system frameworks. + +**Files:** +- Create: `avlivebody-mac/Sources/AVLiveBody/usb/USBMuxProtocol.swift` (copy of the existing file) +- Create: `avlivebody-mac/Sources/AVLiveBody/usb/USBClient.swift` (copy) +- Create: `avlivebody-mac/Sources/AVLiveBody/usb/VideoDecoder.swift` (copy) + +- [ ] **Step 1: Copy the three files** + +```bash +mkdir -p avlivebody-mac/Sources/AVLiveBody/usb +cp launcher/AV-Live-Body/Sources/AVLiveBody/USBMuxProtocol.swift \ + launcher/AV-Live-Body/Sources/AVLiveBody/USBClient.swift \ + launcher/AV-Live-Body/Sources/AVLiveBody/VideoDecoder.swift \ + avlivebody-mac/Sources/AVLiveBody/usb/ +``` + +- [ ] **Step 2: Migrate the unit tests for them** + +```bash +mkdir -p avlivebody-mac/Tests/AVLiveBodyTests +cp launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBMuxProtocolTests.swift \ + launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBClientTests.swift \ + avlivebody-mac/Tests/AVLiveBodyTests/ +``` + +- [ ] **Step 3: Build + test** + +```bash +cd avlivebody-mac && xcodegen generate && \ +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' test +``` + +Expected: `** TEST SUCCEEDED **`, the USBMux/USBClient tests pass. + +- [ ] **Step 4: Commit** + +```bash +git add avlivebody-mac/Sources/AVLiveBody/usb avlivebody-mac/Tests +git commit -m "feat(avlivebody-mac): migrate usb transport" +``` + +--- + +## Task 3: Migrate USBSkeletonConsumer (cleaned) + +The old `USBSkeletonConsumer` converts `SkeletonPayload` into the +legacy `ArkitOSCListener.ArkitBodyFrame`. The new app drops +`ArkitOSCListener` entirely, so the consumer publishes +`[Int: SkeletonPayload]` directly. + +**Files:** +- Create: `avlivebody-mac/Sources/AVLiveBody/usb/USBSkeletonConsumer.swift` + +- [ ] **Step 1: Create the cleaned file** + +```swift +import AVLiveWire +import Combine +import CoreVideo +import Foundation + +/// Connects to the tethered iPhone over USB (usbmuxd), demuxes the +/// AVLiveWire stream, republishes skeleton payloads (keyed by pid) +/// and forwards decoded camera frames. Blocking transport runs on a +/// dedicated background thread; only `@Published` writes hop to main. +final class USBSkeletonConsumer: ObservableObject { + /// 91-joint skeleton payloads keyed by pid. + @Published var skeletons: [Int: SkeletonPayload] = [:] + @Published var connected = false + + /// Called on the main queue for every decoded camera frame. + var onVideoFrame: ((CVPixelBuffer) -> Void)? + + /// TCP port the iPhone `USBServer` listens on. + static let devicePort: UInt16 = 7000 + + private let videoDecoder = VideoDecoder() + private let stateLock = NSLock() + private var running = false + private var thread: Thread? + + init() { + videoDecoder.onFrame = { [weak self] pixelBuffer in + DispatchQueue.main.async { + self?.onVideoFrame?(pixelBuffer) + } + } + } + + private var isRunning: Bool { + stateLock.lock(); defer { stateLock.unlock() } + return running + } + + func start() { + stateLock.lock() + if running { stateLock.unlock(); return } + running = true + stateLock.unlock() + let t = Thread { [weak self] in self?.loop() } + t.name = "cc.avlive.usbconsumer" + t.start() + thread = t + } + + func stop() { + stateLock.lock(); running = false; stateLock.unlock() + } + + private func loop() { + while isRunning { + guard let transport = UnixMuxTransport() else { + NSLog("USBSkeletonConsumer: no usbmuxd; retry") + Thread.sleep(forTimeInterval: 1.0); continue + } + let client = USBClient(transport: transport) + let devices = client.listDevices() + guard let dev = devices.first, + client.connect(deviceID: dev, + port: Self.devicePort) else { + NSLog("USBSkeletonConsumer: no device; retry") + transport.close() + Thread.sleep(forTimeInterval: 1.0); continue + } + NSLog("USBSkeletonConsumer: connected to device %d", dev) + publishConnected(true) + var demux = StreamDemuxer() + while isRunning { + guard let chunk = transport.readStream(), + !chunk.isEmpty else { break } + for frame in demux.feed(chunk) { route(frame) } + } + transport.close() + publishConnected(false) + NSLog("USBSkeletonConsumer: disconnected") + if isRunning { Thread.sleep(forTimeInterval: 1.0) } + } + } + + private func route(_ frame: StreamDemuxer.Frame) { + switch frame.header.tag { + case .skeleton: + guard let payload = + SkeletonPayload(decoding: frame.payload) else { return } + let pid = Int(frame.header.pid) + DispatchQueue.main.async { [weak self] in + self?.skeletons[pid] = payload + } + case .video: + guard let payload = + VideoPayload(decoding: frame.payload) else { return } + videoDecoder.decode(payload) + case .meta: + break + } + } + + private func publishConnected(_ value: Bool) { + DispatchQueue.main.async { [weak self] in + self?.connected = value + } + } +} +``` + +- [ ] **Step 2: Build** + +```bash +cd avlivebody-mac && xcodegen generate && \ +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' build +``` + +Expected: `** BUILD SUCCEEDED **`. + +- [ ] **Step 3: Commit** + +```bash +git add avlivebody-mac/Sources/AVLiveBody/usb/USBSkeletonConsumer.swift +git commit -m "feat(avlivebody-mac): usb skeleton consumer" +``` + +--- + +## Task 4: Migrate MultiHMRCoreML + BodyFusion (cleaned) + +Copy `MultiHMRCoreML.swift` verbatim. Adapt `BodyFusion` to take +`[Int: SkeletonPayload]` instead of the legacy `ArkitBodyFrame`. + +**Files:** +- Create: `avlivebody-mac/Sources/AVLiveBody/usb/MultiHMRCoreML.swift` (copy) +- Create: `avlivebody-mac/Sources/AVLiveBody/usb/BodyFusion.swift` +- Test: `avlivebody-mac/Tests/AVLiveBodyTests/BodyFusionTests.swift` + +- [ ] **Step 1: Copy MultiHMRCoreML.swift** + +```bash +cp launcher/AV-Live-Body/Sources/AVLiveBody/MultiHMRCoreML.swift \ + avlivebody-mac/Sources/AVLiveBody/usb/ +``` + +- [ ] **Step 2: Write the BodyFusion test** + +`avlivebody-mac/Tests/AVLiveBodyTests/BodyFusionTests.swift`: + +```swift +import XCTest +import AVLiveWire +@testable import AVLiveBody + +final class BodyFusionTests: XCTestCase { + private func skeleton(pelvisZ: Float) -> SkeletonPayload { + var p = SkeletonPayload() + p.joints[0] = SIMD3(0, 0, pelvisZ) + p.valid[0] = true + return p + } + + func testPelvisDepthOverride() { + let mesh = MultiHMRPerson( + vertices: [SIMD3](repeating: .zero, count: 1), + translation: SIMD3(0, 0, -1.0), score: 0.9) + let fused = BodyFusion.fuse( + persons: [mesh], skeletons: [0: skeleton(pelvisZ: -2.5)]) + XCTAssertEqual(fused[0].translation.z, -2.5, accuracy: 1e-4) + } + + func testPassthroughWhenNoSkeleton() { + let mesh = MultiHMRPerson( + vertices: [SIMD3](repeating: .zero, count: 1), + translation: SIMD3(0, 0, -1.0), score: 0.9) + let fused = BodyFusion.fuse(persons: [mesh], skeletons: [:]) + XCTAssertEqual(fused[0].translation.z, -1.0, accuracy: 1e-4) + } +} +``` + +- [ ] **Step 3: Run the test to verify it fails** + +Run: `cd avlivebody-mac && xcodegen generate && xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody -destination 'platform=macOS' test` +Expected: FAIL — `BodyFusion` undefined. + +- [ ] **Step 4: Write BodyFusion.swift** + +`avlivebody-mac/Sources/AVLiveBody/usb/BodyFusion.swift`: + +```swift +import AVLiveWire +import Foundation +import simd + +/// Associates Multi-HMR meshes with USB skeletons and corrects the +/// mesh pelvis depth. Pure, stateless — unit-testable. +enum BodyFusion { + /// SMPL-X / ARKit body root (hips) joint index. + static let pelvisJoint = 0 + + static func fuse(persons: [MultiHMRPerson], + skeletons: [Int: SkeletonPayload]) + -> [MultiHMRPerson] { + let pelvisZs: [Float] = skeletons.values.compactMap { s in + guard pelvisJoint < s.valid.count, + s.valid[pelvisJoint] else { return nil } + return s.joints[pelvisJoint].z + } + guard !pelvisZs.isEmpty, + let primaryIdx = persons.indices.max(by: { + persons[$0].score < persons[$1].score + }) else { return persons } + var out = persons + out[primaryIdx].translation.z = pelvisZs[0] + return out + } +} +``` + +- [ ] **Step 5: Run the test to verify it passes** + +Run the `test` command. Expected: `BodyFusionTests` pass. + +- [ ] **Step 6: Commit** + +```bash +git add avlivebody-mac/Sources/AVLiveBody/usb/MultiHMRCoreML.swift avlivebody-mac/Sources/AVLiveBody/usb/BodyFusion.swift avlivebody-mac/Tests/AVLiveBodyTests/BodyFusionTests.swift +git commit -m "feat(avlivebody-mac): multi-hmr and body fusion" +``` + +--- + +## Task 5: SceneController + SceneView + +`SceneController` owns the RealityKit scene, an orbital camera, and +the entity roots. `SceneView` is the `NSViewRepresentable` bridge. + +**Files:** +- Create: `avlivebody-mac/Sources/AVLiveBody/SceneController.swift` +- Create: `avlivebody-mac/Sources/AVLiveBody/SceneView.swift` + +- [ ] **Step 1: Write `SceneController.swift`** + +```swift +import Foundation +import RealityKit +import simd + +/// Owns the single RealityKit scene: the video quad, the body root, +/// and an orbital camera. The app calls `updateVideo/updateSkeleton/ +/// updateMesh` from the main queue. +@MainActor +final class SceneController { + let arView = ARView(frame: .zero) + + private let cameraAnchor = AnchorEntity(world: .zero) + private let camera = PerspectiveCamera() + private let worldAnchor = AnchorEntity(world: .zero) + + private(set) var videoQuad: VideoQuad? + private(set) var skeleton: SkeletonEntity? + private(set) var mesh: MeshEntity? + + /// Orbital camera state. + private var orbitYaw: Float = 0 + private var orbitPitch: Float = 0 + private var orbitRadius: Float = 3.0 + + func setUp() { + arView.environment.background = .color(.black) + arView.scene.addAnchor(worldAnchor) + + camera.camera.fieldOfViewInDegrees = 55 + cameraAnchor.addChild(camera) + arView.scene.addAnchor(cameraAnchor) + applyCamera() + + let q = VideoQuad() + worldAnchor.addChild(q.entity) + videoQuad = q + + let s = SkeletonEntity() + worldAnchor.addChild(s.root) + skeleton = s + + let m = MeshEntity() + worldAnchor.addChild(m.root) + mesh = m + + installOrbitGestures() + } + + func updateVideo(_ pixelBuffer: CVPixelBuffer) { + videoQuad?.update(pixelBuffer) + } + + func updateSkeleton(_ skeletons: [Int: SkeletonPayload]) { + skeleton?.update(skeletons) + } + + func updateMesh(_ persons: [MultiHMRPerson]) { + mesh?.update(persons) + } + + // MARK: - Orbital camera + + private func applyCamera() { + let cy = cos(orbitYaw), sy = sin(orbitYaw) + let cp = cos(orbitPitch), sp = sin(orbitPitch) + let pos = SIMD3(orbitRadius * cp * sy, + orbitRadius * sp, + orbitRadius * cp * cy) + cameraAnchor.transform.translation = pos + camera.look(at: .zero, from: pos, relativeTo: nil) + } + + private func installOrbitGestures() { + let pan = NSPanGestureRecognizer( + target: OrbitTarget.shared, action: #selector( + OrbitTarget.handlePan(_:))) + OrbitTarget.shared.controller = self + arView.addGestureRecognizer(pan) + } + + fileprivate func orbit(dx: Float, dy: Float) { + orbitYaw += dx * 0.01 + orbitPitch = max(-1.4, min(1.4, orbitPitch + dy * 0.01)) + applyCamera() + } +} + +/// Bridges the AppKit pan gesture to `SceneController.orbit`. +final class OrbitTarget: NSObject { + static let shared = OrbitTarget() + weak var controller: SceneController? + private var last: CGPoint = .zero + + @objc func handlePan(_ g: NSPanGestureRecognizer) { + let p = g.translation(in: g.view) + if g.state == .began { last = p } + let dx = Float(p.x - last.x) + let dy = Float(p.y - last.y) + last = p + Task { @MainActor in + self.controller?.orbit(dx: dx, dy: -dy) + } + } +} +``` + +- [ ] **Step 2: Write `SceneView.swift`** + +```swift +import RealityKit +import SwiftUI + +/// SwiftUI bridge that hands the SceneController's ARView to the +/// window and runs `setUp()` once. +struct SceneView: NSViewRepresentable { + let controller: SceneController + + func makeNSView(context: Context) -> ARView { + controller.setUp() + return controller.arView + } + + func updateNSView(_ view: ARView, context: Context) {} +} +``` + +- [ ] **Step 3: Build** + +`VideoQuad`, `SkeletonEntity`, `MeshEntity` do not exist yet — this +task will not build alone. Proceed to Tasks 6-8, then build at Task 8. +(Stub note: the build is verified at the end of Task 8.) + +- [ ] **Step 4: Commit** + +```bash +git add avlivebody-mac/Sources/AVLiveBody/SceneController.swift avlivebody-mac/Sources/AVLiveBody/SceneView.swift +git commit -m "feat(avlivebody-mac): scene controller + view" +``` + +--- + +## Task 6: SkeletonEntity + +91 joint marker spheres under a root entity. + +**Files:** +- Create: `avlivebody-mac/Sources/AVLiveBody/SkeletonEntity.swift` + +- [ ] **Step 1: Write the file** + +```swift +import AVLiveWire +import Foundation +import RealityKit +import simd + +/// Renders 91-joint skeletons as yellow marker spheres. One marker +/// pool per pid. ARKit world coords -> RealityKit space (x, -y, -z). +@MainActor +final class SkeletonEntity { + let root = Entity() + + private static let jointCount = 91 + private static let markerRadius: Float = 0.012 + + private var pools: [Int: [ModelEntity]] = [:] + private let mesh = MeshResource.generateSphere(radius: markerRadius) + private let material = SimpleMaterial( + color: .systemYellow, roughness: 0.6, isMetallic: false) + + func update(_ skeletons: [Int: SkeletonPayload]) { + // Drop pools for pids no longer present. + for pid in pools.keys where skeletons[pid] == nil { + pools[pid]?.forEach { $0.removeFromParent() } + pools.removeValue(forKey: pid) + } + for (pid, payload) in skeletons { + let pool = pools[pid] ?? makePool() + pools[pid] = pool + let n = min(Self.jointCount, payload.joints.count, + payload.valid.count) + for i in 0..(j.x, -j.y, -j.z) + marker.isEnabled = true + } else { + marker.isEnabled = false + } + } + } + } + + private func makePool() -> [ModelEntity] { + var pool: [ModelEntity] = [] + pool.reserveCapacity(Self.jointCount) + for _ in 0..(0, 0, Self.zBack) + } + + /// Replace the plane's texture from a decoded camera frame. + func update(_ pixelBuffer: CVPixelBuffer) { + let ci = CIImage(cvPixelBuffer: pixelBuffer) + guard let cg = ciContext.createCGImage( + ci, from: ci.extent) else { return } + guard let texture = try? TextureResource( + image: cg, + options: .init(semantic: .color)) else { return } + var material = UnlitMaterial() + material.color = .init(tint: .white, + texture: .init(texture)) + entity.model?.materials = [material] + } +} +``` + +Note: per-frame `CGImage` + `TextureResource` creation is the known +performance hot spot. It is isolated here so a later iteration can +switch to `LowLevelTexture` / a Metal-backed update without touching +callers. + +- [ ] **Step 2: Commit** (build verified at Task 8) + +```bash +git add avlivebody-mac/Sources/AVLiveBody/VideoQuad.swift +git commit -m "feat(avlivebody-mac): video quad" +``` + +--- + +## Task 8: MeshEntity + +Renders the SMPL-X dense mesh (10475 vertices) from Multi-HMR. The +triangle indices are loaded from a bundled `smplx_faces.bin` (the same +face-index binary the old app used: a flat array of `UInt32` triplets). + +**Files:** +- Create (copy): `avlivebody-mac/Sources/AVLiveBody/Resources/smplx_faces.bin` +- Create: `avlivebody-mac/Sources/AVLiveBody/MeshEntity.swift` + +- [ ] **Step 1: Copy the face-index resource** + +```bash +mkdir -p avlivebody-mac/Sources/AVLiveBody/Resources +cp launcher/AV-Live-Body/Sources/AVLiveBody/Resources/smplx_faces.bin \ + avlivebody-mac/Sources/AVLiveBody/Resources/ +``` + +Declare it in `project.yml` — add under the `AVLiveBody` target a +`resources` style copy by adding to `sources` a buildPhase, or simply +keep it in `Sources/AVLiveBody/Resources` (xcodegen copies unknown +files as resources of the folder reference). Verify after Step 3 that +`Bundle.main.url(forResource: "smplx_faces", withExtension: "bin")` +resolves; if not, add an explicit resource entry to `project.yml`. + +- [ ] **Step 2: Write `MeshEntity.swift`** + +```swift +import Foundation +import RealityKit +import simd + +/// Renders SMPL-X dense body meshes (10475 vertices) from Multi-HMR. +/// Triangle indices come from the bundled `smplx_faces.bin` +/// (flat UInt32 triplets). +@MainActor +final class MeshEntity { + let root = Entity() + + private static let vertexCount = 10475 + private let faces: [UInt32] + private var pools: [Int: ModelEntity] = [:] + private let material = SimpleMaterial( + color: .init(white: 0.8, alpha: 1.0), + roughness: 0.5, isMetallic: false) + + init() { + faces = MeshEntity.loadFaces() + } + + /// Build/refresh one mesh per detected person. + func update(_ persons: [MultiHMRPerson]) { + for (idx, person) in persons.enumerated() { + let entity = pools[idx] ?? { + let e = ModelEntity() + root.addChild(e) + pools[idx] = e + return e + }() + guard let mesh = buildMesh(person.vertices) else { continue } + entity.model = ModelComponent(mesh: mesh, + materials: [material]) + // RealityKit space conversion + fused translation. + let t = person.translation + entity.transform.translation = + SIMD3(t.x, -t.y, -t.z) + entity.isEnabled = true + } + for idx in pools.keys where idx >= persons.count { + pools[idx]?.isEnabled = false + } + } + + private func buildMesh(_ verts: [SIMD3]) + -> MeshResource? { + guard verts.count == Self.vertexCount, + !faces.isEmpty else { return nil } + var descriptor = MeshDescriptor(name: "smplx") + // Model->RealityKit space (x, -y, -z). + descriptor.positions = MeshBuffer(verts.map { + SIMD3($0.x, -$0.y, -$0.z) + }) + descriptor.primitives = .triangles(faces) + return try? MeshResource.generate(from: [descriptor]) + } + + private static func loadFaces() -> [UInt32] { + guard let url = Bundle.main.url( + forResource: "smplx_faces", withExtension: "bin"), + let data = try? Data(contentsOf: url) else { + NSLog("MeshEntity: smplx_faces.bin missing") + return [] + } + return data.withUnsafeBytes { raw in + Array(raw.bindMemory(to: UInt32.self)) + } + } +} +``` + +- [ ] **Step 3: Build the whole app (Tasks 5-8 together)** + +```bash +cd avlivebody-mac && xcodegen generate && \ +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' build +``` + +Expected: `** BUILD SUCCEEDED **` — `SceneController`, `SceneView`, +`SkeletonEntity`, `VideoQuad`, `MeshEntity` all compile together. Fix +any RealityKit signature mismatch minimally (the RealityKit mesh / +texture APIs are the likely friction points; preserve behavior). + +- [ ] **Step 4: Commit** + +```bash +git add avlivebody-mac/Sources/AVLiveBody/MeshEntity.swift avlivebody-mac/Sources/AVLiveBody/Resources avlivebody-mac/project.yml +git commit -m "feat(avlivebody-mac): smpl-x mesh entity" +``` + +--- + +## Task 9: Wire the app together + +Replace the placeholder window with the scene, own the consumer + the +CoreML pipeline, show a status bar. + +**Files:** +- Modify: `avlivebody-mac/Sources/AVLiveBody/AVLiveBodyApp.swift` +- Create: `avlivebody-mac/Sources/AVLiveBody/StatusBar.swift` +- Create (copy): `avlivebody-mac/Sources/AVLiveBody/Resources/multihmr_full_672_s.mlpackage` + +- [ ] **Step 1: Bundle the CoreML model** + +```bash +cp -R ~/.cache/av-live-multihmr/multihmr_full_672_s.mlpackage \ + avlivebody-mac/Sources/AVLiveBody/Resources/ +``` + +It is gitignored (`*.mlpackage`) — a build input, never committed. If +absent, STOP — see voie 2 / `data_only_viz/scripts/coreml_full_probe.py`. + +- [ ] **Step 2: Write `StatusBar.swift`** + +```swift +import SwiftUI + +/// A thin overlay showing the USB connection state. +struct StatusBar: View { + @ObservedObject var consumer: USBSkeletonConsumer + + var body: some View { + HStack(spacing: 6) { + Circle() + .fill(consumer.connected ? Color.green : Color.orange) + .frame(width: 9, height: 9) + Text(consumer.connected + ? "iPhone connected (USB)" + : "waiting for iPhone…") + .font(.caption) + .foregroundStyle(.white) + Spacer() + } + .padding(8) + .background(.black.opacity(0.5)) + } +} +``` + +- [ ] **Step 3: Rewrite `AVLiveBodyApp.swift`** + +```swift +import Cocoa +import CoreVideo +import SwiftUI + +final class AppDelegate: NSObject, NSApplicationDelegate { + func applicationDidFinishLaunching(_ notification: Notification) { + NSApp.setActivationPolicy(.regular) + NSApp.activate() + } +} + +@main +struct AVLiveBodyApp: App { + @NSApplicationDelegateAdaptor(AppDelegate.self) + private var appDelegate + + var body: some Scene { + WindowGroup { + ContentView() + .frame(minWidth: 900, minHeight: 600) + } + } +} + +struct ContentView: View { + @StateObject private var consumer = USBSkeletonConsumer() + @State private var controller = SceneController() + private let multiHMR = MultiHMRCoreML() + /// Placeholder intrinsics until a `.meta` frame supplies real ones. + private let cameraK: [Float] = [ + 672, 0, 336, 0, 672, 336, 0, 0, 1, + ] + + var body: some View { + ZStack(alignment: .top) { + SceneView(controller: controller) + StatusBar(consumer: consumer) + } + .onAppear { wire() } + .onReceive(consumer.$skeletons) { skeletons in + controller.updateSkeleton(skeletons) + } + } + + private func wire() { + consumer.onVideoFrame = { pixelBuffer in + controller.updateVideo(pixelBuffer) + if let hmr = multiHMR { + let raw = hmr.infer(pixelBuffer, cameraK: cameraK) + let fused = BodyFusion.fuse( + persons: raw, skeletons: consumer.skeletons) + controller.updateMesh(fused) + } + } + consumer.start() + } +} +``` + +- [ ] **Step 4: Build** + +```bash +cd avlivebody-mac && xcodegen generate && \ +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' build +``` + +Expected: `** BUILD SUCCEEDED **`. + +- [ ] **Step 5: Commit** + +```bash +git add avlivebody-mac/Sources/AVLiveBody/AVLiveBodyApp.swift avlivebody-mac/Sources/AVLiveBody/StatusBar.swift avlivebody-mac/project.yml +git commit -m "feat(avlivebody-mac): wire scene, consumer, status" +``` + +--- + +## Task 10: Archive the old app + final verification + +**Files:** +- Modify: `launcher/CLAUDE.md` (note the archival) + +- [ ] **Step 1: Archive the old AVLiveBody** + +```bash +git mv launcher/AV-Live-Body launcher/_archive-AV-Live-Body +``` + +Add a one-line note at the top of `launcher/_archive-AV-Live-Body/` +(create `ARCHIVED.md`): "Superseded by `avlivebody-mac/` on +2026-05-18 — see `docs/superpowers/specs/2026-05-18-avlivebody-macos-rewrite-design.md`." + +- [ ] **Step 2: Full clean build + tests** + +```bash +cd avlivebody-mac && xcodegen generate && \ +xcodebuild -project AVLiveBody.xcodeproj -scheme AVLiveBody \ + -destination 'platform=macOS' clean build test +``` + +Expected: `** BUILD SUCCEEDED **` and `** TEST SUCCEEDED **`. + +- [ ] **Step 3: Commit** + +```bash +git add launcher/_archive-AV-Live-Body/ARCHIVED.md launcher/CLAUDE.md +git commit -m "chore: archive legacy AV-Live-Body" +``` + +--- + +## Self-Review + +- **Spec coverage:** Every spec component is built — scaffold (T1), + USB transport (T2), `USBSkeletonConsumer` (T3), `MultiHMRCoreML` + + `BodyFusion` (T4), `SceneController`/`SceneView` (T5), + `SkeletonEntity` (T6), `VideoQuad` (T7), `MeshEntity` (T8), app + wiring + `StatusBar` (T9), archival (T10). The data flow + (consumer → controller → entities) is wired in T9. +- **Placeholders:** none — every step has concrete code or an exact + command. T5 Step 3 deliberately defers the build to T8 because + `SceneController` references entities created in T6-T8; this + ordering note is explicit, not a placeholder. +- **Type consistency:** `USBSkeletonConsumer` publishes + `[Int: SkeletonPayload]`; `SceneController.updateSkeleton` and + `SkeletonEntity.update` consume exactly that. `BodyFusion.fuse` + takes `[Int: SkeletonPayload]` (cleaned from the legacy + `ArkitBodyFrame`) and `[MultiHMRPerson]`; `MultiHMRCoreML.infer` + produces `[MultiHMRPerson]`; `MeshEntity.update` consumes it. + `VideoDecoder.onFrame`/`USBSkeletonConsumer.onVideoFrame`/ + `SceneController.updateVideo` all carry `CVPixelBuffer`. +- **Known risks** (from the spec): `VideoQuad`'s per-frame + `CGImage`+`TextureResource` rebuild is a perf hot spot — isolated in + one unit. `SceneController`'s orbital camera uses an AppKit pan + gesture bridged via `OrbitTarget`. The RealityKit `MeshDescriptor`/ + `TextureResource` APIs are the most likely to need a minor + signature fix on the live SDK — T8 Step 3 calls this out. diff --git a/docs/superpowers/plans/2026-05-18-iphone-capture.md b/docs/superpowers/plans/2026-05-18-iphone-capture.md new file mode 100644 index 0000000..058c046 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-iphone-capture.md @@ -0,0 +1,430 @@ +# iPhone Capture Implementation Plan (Plan 2 of 3) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the iOS `ARBodyTracker` app stream the camera RGB video (HEVC) over the USB transport alongside the ARKit skeleton, and retire the legacy OSC/UDP sender — so the iPhone is a self-contained, network-free capture source. + +**Architecture:** `ARBodySession` already captures the ARKit 91-joint skeleton and sends it as `AVLiveWire` `.skeleton` frames through `USBServer` (built in Plan 1). This plan adds a `VideoEncoder` (VideoToolbox hardware HEVC) that encodes each `ARFrame.capturedImage` and sends it as `.video` frames through the same `USBServer`. The OSC/UDP fanout (`/body3d/kp` to `host:57128/57129`) and its `ContentView` config fields are removed. + +**Tech Stack:** Swift 5.10, ARKit, VideoToolbox, CoreMedia, `AVLiveWire` (local package), iOS 17. Build verification via `xcodebuild`. + +**Companion spec:** `docs/superpowers/specs/2026-05-18-iphone-usb-body-link-design.md` +**Prerequisite:** Plan 1 (`docs/superpowers/plans/2026-05-18-iphone-usb-transport.md`) — merged. + +--- + +## Verification note + +The iOS app is an iOS-only target; it cannot be built with `swift build` +on a macOS host. The verification command for every task is: + +```bash +cd iphone-arbody && xcodegen generate && \ +xcodebuild -project ARBodyTracker.xcodeproj -scheme ARBodyTracker \ + -sdk iphonesimulator -destination 'generic/platform=iOS Simulator' \ + -configuration Debug build +``` + +Expected: `** BUILD SUCCEEDED **`. VideoToolbox HEVC encoding and ARKit +body tracking only run fully on a physical device — runtime behavior is +an owner on-device check, out of this plan's automated scope. + +--- + +## File Structure + +| File | Responsibility | +|------|----------------| +| `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/VideoEncoder.swift` | NEW. VideoToolbox HEVC hardware encoder: `CVPixelBuffer` → `VideoPayload` via callback | +| `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift` | MODIFY. Add video encoding in `session(_:didUpdate:)`; remove OSC fanout | +| `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift` | MODIFY. Remove OSC host/port config UI | + +--- + +## Task 1: VideoEncoder + +`VideoEncoder` wraps a `VTCompressionSession` configured for HEVC. It +accepts `CVPixelBuffer`s and invokes `onPayload` with a `VideoPayload` +(keyframe flag + the access-unit bytes; for keyframes the HEVC +parameter sets are prepended so the Mac decoder is self-sufficient). + +**Files:** +- Create: `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/VideoEncoder.swift` + +- [ ] **Step 1: Create the file** + +```swift +import AVLiveWire +import CoreMedia +import CoreVideo +import Foundation +import VideoToolbox + +/// Hardware HEVC encoder. Feed `CVPixelBuffer`s from ARKit frames in; +/// receive one `VideoPayload` per encoded access unit via `onPayload`. +/// Keyframe payloads carry the VPS/SPS/PPS parameter sets prepended, +/// each as a 4-byte-length-prefixed NAL unit, so the Mac decoder can +/// build its format description without a side channel. +final class VideoEncoder { + var onPayload: ((VideoPayload) -> Void)? + + private var session: VTCompressionSession? + private let lock = NSLock() + + /// Create the compression session for a given frame size. + func start(width: Int32, height: Int32) { + stop() + var s: VTCompressionSession? + let status = VTCompressionSessionCreate( + allocator: kCFAllocatorDefault, + width: width, height: height, + codecType: kCMVideoCodecType_HEVC, + encoderSpecification: nil, + imageBufferAttributes: nil, + compressedDataAllocator: nil, + outputCallback: nil, + refcon: nil, + compressionSessionOut: &s) + guard status == noErr, let s else { + NSLog("VideoEncoder: VTCompressionSessionCreate failed %d", + status) + return + } + VTSessionSetProperty(s, key: kVTCompressionPropertyKey_RealTime, + value: kCFBooleanTrue) + VTSessionSetProperty(s, + key: kVTCompressionPropertyKey_AllowFrameReordering, + value: kCFBooleanFalse) + VTSessionSetProperty(s, + key: kVTCompressionPropertyKey_MaxKeyFrameInterval, + value: 30 as CFNumber) + VTCompressionSessionPrepareToEncodeFrames(s) + session = s + } + + /// Encode one frame. `pts` is the capture timestamp in seconds. + func encode(_ pixelBuffer: CVPixelBuffer, pts: Double) { + lock.lock(); let s = session; lock.unlock() + guard let s else { return } + let time = CMTime(seconds: pts, preferredTimescale: 1_000_000) + VTCompressionSessionEncodeFrame( + s, imageBuffer: pixelBuffer, presentationTimeStamp: time, + duration: .invalid, frameProperties: nil, + infoFlagsOut: nil) { [weak self] status, _, sample in + guard status == noErr, let sample else { return } + self?.handle(sample) + } + } + + func stop() { + lock.lock(); let s = session; session = nil; lock.unlock() + if let s { + VTCompressionSessionInvalidate(s) + } + } + + deinit { stop() } + + // MARK: - Sample → VideoPayload + + private func handle(_ sample: CMSampleBuffer) { + let isKeyframe = !Self.notSync(sample) + var out = Data() + if isKeyframe, let fmt = CMSampleBufferGetFormatDescription(sample) { + out.append(Self.parameterSets(fmt)) + } + if let block = CMSampleBufferGetDataBuffer(sample) { + var lengthOut = 0 + var ptr: UnsafeMutablePointer? + if CMBlockBufferGetDataPointer( + block, atOffset: 0, lengthAtOffsetOut: nil, + totalLengthOut: &lengthOut, + dataPointerOut: &ptr) == noErr, let ptr { + out.append(UnsafeBufferPointer( + start: UnsafeRawPointer(ptr) + .assumingMemoryBound(to: UInt8.self), + count: lengthOut)) + } + } + guard !out.isEmpty else { return } + onPayload?(VideoPayload(isKeyframe: isKeyframe, data: out)) + } + + /// True if the sample is NOT a sync (key) frame. + private static func notSync(_ sample: CMSampleBuffer) -> Bool { + guard let arr = CMSampleBufferGetSampleAttachmentsArray( + sample, createIfNecessary: false), + CFArrayGetCount(arr) > 0 else { return false } + let dict = unsafeBitCast(CFArrayGetValueAtIndex(arr, 0), + to: CFDictionary.self) + let key = Unmanaged.passUnretained( + kCMSampleAttachmentKey_NotSync).toOpaque() + return CFDictionaryContainsKey(dict, key) + } + + /// Concatenate the HEVC VPS/SPS/PPS parameter sets, each as a + /// 4-byte big-endian length prefix followed by the NAL bytes. + private static func parameterSets( + _ fmt: CMFormatDescription) -> Data { + var count = 0 + CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + fmt, parameterSetIndex: 0, parameterSetPointerOut: nil, + parameterSetSizeOut: nil, parameterSetCountOut: &count, + nalUnitHeaderLengthOut: nil) + var data = Data() + for i in 0..? + var size = 0 + guard CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + fmt, parameterSetIndex: i, + parameterSetPointerOut: &ptr, + parameterSetSizeOut: &size, + parameterSetCountOut: nil, + nalUnitHeaderLengthOut: nil) == noErr, + let ptr else { continue } + var be = UInt32(size).bigEndian + withUnsafeBytes(of: &be) { data.append(contentsOf: $0) } + data.append(UnsafeBufferPointer(start: ptr, count: size)) + } + return data + } +} +``` + +- [ ] **Step 2: Verify it compiles** + +Run the verification command from the "Verification note" section above. +Expected: `** BUILD SUCCEEDED **` (the new file compiles within the +target). + +- [ ] **Step 3: Commit** + +```bash +git add iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/VideoEncoder.swift +git commit -m "feat(ios): VideoToolbox HEVC encoder" +``` + +(subject ≤50 chars; add a short body — the commit hook rejects +subject-only messages; no AI attribution.) + +--- + +## Task 2: Stream video from ARBodySession + +Wire `VideoEncoder` into the ARKit frame loop. On each `didUpdate` +frame already processed for skeletons, also encode `capturedImage` and +send the resulting `VideoPayload` over the existing `USBServer`. + +**Files:** +- Modify: `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift` + +- [ ] **Step 1: Add the encoder property and its payload wiring** + +In `ARBodySession`, next to `private let usb = USBServer()` (currently +line 45), add: + +```swift + private let videoEncoder = VideoEncoder() + private var videoStarted = false +``` + +In `init()`, after the `usb.onState = { ... }` block, add the encoder +output wiring: + +```swift + videoEncoder.onPayload = { [weak self] payload in + Task { @MainActor in + guard let self, self.usbState == .connected else { + return + } + self.usb.send(tag: .video, pid: -1, + timestamp: self.lastFrameTime, + payload: payload.encoded()) + } + } +``` + +- [ ] **Step 2: Encode the captured image in the frame loop** + +In `session(_:didUpdate:)`, inside the `Task { @MainActor in ... }` +block, after `self.lastFrameTime = t` and before the anchor loop, +add video encoding: + +```swift + // Start the encoder lazily once the first frame size is + // known, then encode every (throttled) frame. + let img = frame.capturedImage + let w = Int32(CVPixelBufferGetWidth(img)) + let h = Int32(CVPixelBufferGetHeight(img)) + if !self.videoStarted, w > 0, h > 0 { + self.videoEncoder.start(width: w, height: h) + self.videoStarted = true + } + if self.videoStarted { + self.videoEncoder.encode(img, pts: t) + } +``` + +- [ ] **Step 3: Stop the encoder on stop()** + +In `stop()`, after `usb.stop()`, add: + +```swift + videoEncoder.stop() + videoStarted = false +``` + +- [ ] **Step 4: Verify it compiles** + +Run the verification command. Expected: `** BUILD SUCCEEDED **`. + +- [ ] **Step 5: Commit** + +```bash +git add iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift +git commit -m "feat(ios): stream HEVC video over USB" +``` + +--- + +## Task 3: Remove the legacy OSC sender + +The OSC/UDP fanout is the network dependency the autonomous USB design +removes. Delete it from `ARBodySession`. + +**Files:** +- Modify: `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift` + +- [ ] **Step 1: Delete OSC members and methods** + +In `ARBodySession.swift`, delete: +- the stored properties `host`, `pythonPort`, `swiftPort` (currently + lines 39-41) and `conns` (line 44); +- the `configure(host:pythonPort:swiftPort:sendEnvMesh:)` method — + replace it with a parameterless `configure(sendEnvMesh:)`: + ```swift + func configure(sendEnvMesh: Bool) { + self.sendEnvMesh = sendEnvMesh + } + ``` +- the call `openUDP()` in `start()`; +- in `stop()`, the lines `for c in conns { c.cancel() }` and + `conns.removeAll()`; +- the entire `// MARK: - UDP fanout` section: `openUDP()` and + `sendDatagram(_:)`; +- the `publishJoints(pid:body:)` method and its call site in + `session(_:didUpdate:)` (`self.publishJoints(pid: count, body: body)`); +- the `sendOSC(addr:args:)` call for `/body3d/count` in + `session(_:didUpdate:)`; +- the `// MARK: - OSC minimal encoder` section: the `OSCArg` enum, + `sendOSC(addr:args:)`, and `appendOSCString(_:into:)`. + +After deletion, `Network` is still needed (`USBServer` uses it +indirectly — actually `USBServer` imports its own `Network`). Remove +`import Network` from `ARBodySession.swift` only if no symbol from it +remains; if `NWConnection`/`NWEndpoint` no longer appear in the file, +remove the import. + +- [ ] **Step 2: Verify it compiles** + +Run the verification command. Expected: `** BUILD SUCCEEDED **`. If the +build reports an unused `import` or an unresolved symbol, fix it +minimally (remove the dead import, or keep it if still referenced). + +- [ ] **Step 3: Commit** + +```bash +git add iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift +git commit -m "refactor(ios): drop legacy OSC sender" +``` + +--- + +## Task 4: Simplify ContentView + +`ContentView` exposes OSC host/port text fields that no longer have a +backing. Remove them; keep the USB status indicator and Start/Stop. + +**Files:** +- Modify: `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift` + +- [ ] **Step 1: Remove OSC state and UI** + +In `ContentView`: +- delete the `@State` properties `host`, `pythonPort`, `swiftPort` + (currently lines 7-9); +- in `controlPanel`, delete the `HStack { Text("Host") ... }` block and + the `HStack { Text("Py") ... Text("Swift") ... }` block (the two + rows of OSC text fields, currently lines 85-102); +- in the Start/Stop button action, replace the `session.configure( + host:pythonPort:swiftPort:sendEnvMesh:)` call with + `session.configure(sendEnvMesh: sendEnvMesh)`. + +Keep: `sendEnvMesh` toggle, Start/Stop button, status text, the USB +status dot/label, and the bodies/frames/jointsPerSec line. + +- [ ] **Step 2: Verify it compiles** + +Run the verification command. Expected: `** BUILD SUCCEEDED **`. The +three `#Preview` blocks at the end of the file construct +`ContentView(useMockBackground:useMockSkeleton:)` — those parameters +are unaffected; the previews must still compile. + +- [ ] **Step 3: Commit** + +```bash +git add iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift +git commit -m "refactor(ios): drop OSC config from ContentView" +``` + +--- + +## Task 5: Final build verification + +- [ ] **Step 1: Full clean build** + +```bash +cd iphone-arbody && xcodegen generate && \ +xcodebuild -project ARBodyTracker.xcodeproj -scheme ARBodyTracker \ + -sdk iphonesimulator -destination 'generic/platform=iOS Simulator' \ + -configuration Debug clean build +``` + +Expected: `** BUILD SUCCEEDED **`, zero errors. + +- [ ] **Step 2: Confirm no OSC references remain** + +```bash +grep -rn -E "OSC|57128|57129|openUDP|sendDatagram" \ + iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ +``` + +Expected: no matches in `ARBodySession.swift` or `ContentView.swift`. +(`USBServer.swift` and `VideoEncoder.swift` never had OSC.) Comments +mentioning history are acceptable; live OSC code is not. + +- [ ] **Step 3: Commit any cleanup** (only if Step 2 found stragglers) + +--- + +## Self-Review + +- **Spec coverage:** This plan implements the spec's `VideoEncoder` + unit and the `ARBodySession` "exposes video frames / OSC sender + removed" requirement. `ARBodySession` already builds and sends + `SkeletonPayload` over `USBServer` (delivered via the recovery + branch + Plan 1), so no skeleton-path task is needed. `ContentView` + simplification follows from OSC removal. +- **Placeholders:** none — every step has concrete code or an exact + command and expected output. +- **Type consistency:** `VideoPayload`, `FrameTag.video`, + `USBServer.send(tag:pid:timestamp:payload:)` are used consistently + with their Plan 1 / `AVLiveWire` definitions. `VideoEncoder.start` + takes `Int32` width/height matching `CVPixelBufferGetWidth`'s `Int` + cast to `Int32`. +- **Known risk:** the `VideoEncoder` VideoToolbox code compiles on the + simulator but HEVC hardware encoding and the exact access-unit / + parameter-set byte layout can only be validated on a physical + device. Plan 3's `VideoDecoder` must agree with the framing chosen + here (length-prefixed parameter sets prepended to keyframe payloads); + this is the integration seam to verify when Plan 3 is built. diff --git a/docs/superpowers/plans/2026-05-18-iphone-usb-transport.md b/docs/superpowers/plans/2026-05-18-iphone-usb-transport.md new file mode 100644 index 0000000..50bd3e3 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-iphone-usb-transport.md @@ -0,0 +1,1098 @@ +# iPhone USB Transport Implementation Plan (Plan 1 of 3) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the network-free USB byte-pipe between the iOS +`ARBodyTracker` app and the macOS `AVLiveBody` app: a shared wire +format, a native `usbmux` client, an iOS TCP listener, and an +incremental stream demuxer. + +**Architecture:** A shared SwiftPM package `AVLiveWire` defines the +binary frame codec used by both apps. The iOS app serves frames on a +local TCP port; the macOS app reaches that port through Apple's +`usbmuxd` daemon over the USB cable (no IP network). A demuxer +reassembles frames from the byte stream. + +**Tech Stack:** Swift 5.9+, SwiftPM, `Network.framework` (iOS +listener), raw Unix-domain socket + property-list serialization +(macOS usbmux client), `XCTest`. + +**Companion spec:** `docs/superpowers/specs/2026-05-18-iphone-usb-body-link-design.md` + +--- + +## File Structure + +| File | Responsibility | +|------|----------------| +| `shared/AVLiveWire/Package.swift` | SwiftPM manifest for the shared library | +| `shared/AVLiveWire/Sources/AVLiveWire/FrameHeader.swift` | Fixed 19-byte frame header encode/decode | +| `shared/AVLiveWire/Sources/AVLiveWire/WirePayloads.swift` | Skeleton / video / meta payload codecs | +| `shared/AVLiveWire/Sources/AVLiveWire/StreamDemuxer.swift` | Incremental byte-stream → frames, resync on partial buffers | +| `shared/AVLiveWire/Tests/AVLiveWireTests/*.swift` | Unit tests for the above | +| `launcher/AV-Live-Body/Sources/AVLiveBody/USBMuxProtocol.swift` | usbmux message framing (header + plist) | +| `launcher/AV-Live-Body/Sources/AVLiveBody/USBClient.swift` | usbmux device list, connect-to-port, attach/detach, byte stream | +| `launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBMuxProtocolTests.swift` | usbmux codec tests | +| `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/USBServer.swift` | iOS TCP `NWListener`, frame send queue | +| `launcher/AV-Live-Body/Package.swift` | add `AVLiveWire` dependency (modify) | +| `iphone-arbody/ARBodyTracker.swiftpm/Package.swift` | add `AVLiveWire` dependency (modify) | + +`AVLiveWire` is a plain SwiftPM library so both the macOS package and +the iOS `.swiftpm` app can depend on it via a local path — the wire +format is defined exactly once. + +--- + +## Task 1: Shared package skeleton + +**Files:** +- Create: `shared/AVLiveWire/Package.swift` +- Create: `shared/AVLiveWire/Sources/AVLiveWire/AVLiveWire.swift` +- Create: `shared/AVLiveWire/Tests/AVLiveWireTests/SmokeTests.swift` + +- [ ] **Step 1: Create the package manifest** + +`shared/AVLiveWire/Package.swift`: + +```swift +// swift-tools-version:5.9 +import PackageDescription + +let package = Package( + name: "AVLiveWire", + platforms: [.macOS(.v13), .iOS(.v17)], + products: [ + .library(name: "AVLiveWire", targets: ["AVLiveWire"]), + ], + targets: [ + .target(name: "AVLiveWire"), + .testTarget(name: "AVLiveWireTests", dependencies: ["AVLiveWire"]), + ] +) +``` + +- [ ] **Step 2: Create a placeholder source + smoke test** + +`shared/AVLiveWire/Sources/AVLiveWire/AVLiveWire.swift`: + +```swift +/// AVLiveWire — binary frame format shared by ARBodyTracker (iOS) +/// and AVLiveBody (macOS) over the USB transport. +public enum AVLiveWire { + public static let protocolVersion: UInt8 = 1 +} +``` + +`shared/AVLiveWire/Tests/AVLiveWireTests/SmokeTests.swift`: + +```swift +import XCTest +@testable import AVLiveWire + +final class SmokeTests: XCTestCase { + func testProtocolVersion() { + XCTAssertEqual(AVLiveWire.protocolVersion, 1) + } +} +``` + +- [ ] **Step 3: Run the test to verify the package builds** + +Run: `cd shared/AVLiveWire && swift test` +Expected: PASS, 1 test. + +- [ ] **Step 4: Commit** + +```bash +git add shared/AVLiveWire +git commit -m "feat(avlivewire): shared wire package skeleton" +``` + +--- + +## Task 2: Frame header codec + +The frame header is a fixed 19-byte big-endian record: +`tag (u8) | pid (i16) | timestamp (f64) | length (u32)` — total +1+2+8+4 = 15 bytes, plus a 4-byte magic prefix `0x41 0x56 0x4C 0x31` +("AVL1") = 19 bytes. The magic lets the demuxer resync. + +**Files:** +- Create: `shared/AVLiveWire/Sources/AVLiveWire/FrameHeader.swift` +- Test: `shared/AVLiveWire/Tests/AVLiveWireTests/FrameHeaderTests.swift` + +- [ ] **Step 1: Write the failing test** + +`shared/AVLiveWire/Tests/AVLiveWireTests/FrameHeaderTests.swift`: + +```swift +import XCTest +@testable import AVLiveWire + +final class FrameHeaderTests: XCTestCase { + func testRoundTrip() { + let h = FrameHeader(tag: .skeleton, pid: 7, + timestamp: 12.5, length: 1092) + let bytes = h.encoded() + XCTAssertEqual(bytes.count, FrameHeader.byteCount) + let decoded = FrameHeader(decoding: bytes) + XCTAssertEqual(decoded, h) + } + + func testRejectsBadMagic() { + var bytes = FrameHeader(tag: .video, pid: -1, + timestamp: 0, length: 0).encoded() + bytes[0] = 0x00 + XCTAssertNil(FrameHeader(decoding: bytes)) + } + + func testRejectsShortBuffer() { + XCTAssertNil(FrameHeader(decoding: Data([0x41, 0x56]))) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd shared/AVLiveWire && swift test --filter FrameHeaderTests` +Expected: FAIL — `FrameHeader` is undefined. + +- [ ] **Step 3: Write the implementation** + +`shared/AVLiveWire/Sources/AVLiveWire/FrameHeader.swift`: + +```swift +import Foundation + +public enum FrameTag: UInt8 { + case skeleton = 1 + case video = 2 + case meta = 3 +} + +/// Fixed-size frame header. Layout (big-endian): +/// magic[4]=`AVL1` | tag u8 | pid i16 | timestamp f64 | length u32 +public struct FrameHeader: Equatable { + public static let magic: [UInt8] = [0x41, 0x56, 0x4C, 0x31] + public static let byteCount = 19 + + public var tag: FrameTag + public var pid: Int16 + public var timestamp: Double + public var length: UInt32 + + public init(tag: FrameTag, pid: Int16, + timestamp: Double, length: UInt32) { + self.tag = tag; self.pid = pid + self.timestamp = timestamp; self.length = length + } + + public func encoded() -> Data { + var d = Data(Self.magic) + d.append(tag.rawValue) + d.appendBE(UInt16(bitPattern: pid)) + d.appendBE(timestamp.bitPattern) + d.appendBE(length) + return d + } + + public init?(decoding data: Data) { + guard data.count >= Self.byteCount else { return nil } + let b = [UInt8](data.prefix(Self.byteCount)) + guard Array(b[0..<4]) == Self.magic, + let t = FrameTag(rawValue: b[4]) else { return nil } + tag = t + pid = Int16(bitPattern: UInt16(bigEndianBytes: b[5...6])) + timestamp = Double(bitPattern: UInt64(bigEndianBytes: b[7...14])) + length = UInt32(bigEndianBytes: b[15...18]) + } +} + +extension Data { + mutating func appendBE(_ v: UInt16) { + appendBE(UInt64(v), width: 2) } + mutating func appendBE(_ v: UInt32) { + appendBE(UInt64(v), width: 4) } + mutating func appendBE(_ v: UInt64, width: Int = 8) { + for i in stride(from: width - 1, through: 0, by: -1) { + append(UInt8((v >> (8 * i)) & 0xFF)) + } + } +} + +extension UInt16 { + init(bigEndianBytes s: S) where S.Element == UInt8 { + self = s.reduce(0) { ($0 << 8) | UInt16($1) } + } +} +extension UInt32 { + init(bigEndianBytes s: S) where S.Element == UInt8 { + self = s.reduce(0) { ($0 << 8) | UInt32($1) } + } +} +extension UInt64 { + init(bigEndianBytes s: S) where S.Element == UInt8 { + self = s.reduce(0) { ($0 << 8) | UInt64($1) } + } +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd shared/AVLiveWire && swift test --filter FrameHeaderTests` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Commit** + +```bash +git add shared/AVLiveWire +git commit -m "feat(avlivewire): fixed 19-byte frame header codec" +``` + +--- + +## Task 3: Payload codecs + +**Files:** +- Create: `shared/AVLiveWire/Sources/AVLiveWire/WirePayloads.swift` +- Test: `shared/AVLiveWire/Tests/AVLiveWireTests/WirePayloadsTests.swift` + +- [ ] **Step 1: Write the failing test** + +`shared/AVLiveWire/Tests/AVLiveWireTests/WirePayloadsTests.swift`: + +```swift +import XCTest +@testable import AVLiveWire + +final class WirePayloadsTests: XCTestCase { + func testSkeletonRoundTrip() { + var f = SkeletonPayload() + f.joints[0] = SIMD3(1, 2, 3) + f.valid[0] = true + f.joints[90] = SIMD3(-4, 5, -6) + f.valid[90] = true + let decoded = SkeletonPayload(decoding: f.encoded()) + XCTAssertEqual(decoded, f) + } + + func testSkeletonRejectsWrongSize() { + XCTAssertNil(SkeletonPayload(decoding: Data([0, 1, 2]))) + } + + func testVideoPayloadRoundTrip() { + let p = VideoPayload(isKeyframe: true, + data: Data([9, 8, 7, 6])) + let decoded = VideoPayload(decoding: p.encoded()) + XCTAssertEqual(decoded, p) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd shared/AVLiveWire && swift test --filter WirePayloadsTests` +Expected: FAIL — `SkeletonPayload` / `VideoPayload` undefined. + +- [ ] **Step 3: Write the implementation** + +`shared/AVLiveWire/Sources/AVLiveWire/WirePayloads.swift`: + +```swift +import Foundation +import simd + +/// 91 ARKit joints in world space + a per-joint validity flag. +public struct SkeletonPayload: Equatable { + public static let jointCount = 91 + /// 91 * 3 * 4 bytes (floats) + 91 validity bytes. + public static let byteCount = jointCount * 12 + jointCount + + public var joints: [SIMD3] + public var valid: [Bool] + + public init() { + joints = Array(repeating: .zero, count: Self.jointCount) + valid = Array(repeating: false, count: Self.jointCount) + } + + public func encoded() -> Data { + var d = Data(capacity: Self.byteCount) + for j in joints { + d.appendBE(j.x.bitPattern); d.appendBE(j.y.bitPattern) + d.appendBE(j.z.bitPattern) + } + for v in valid { d.append(v ? 1 : 0) } + return d + } + + public init?(decoding data: Data) { + guard data.count == Self.byteCount else { return nil } + let b = [UInt8](data) + self.init() + var o = 0 + for i in 0.. Float { + let v = Float(bitPattern: + UInt32(bigEndianBytes: b[o.. Data { + var d = Data([isKeyframe ? 1 : 0]) + d.append(data) + return d + } + + public init?(decoding data: Data) { + guard let first = data.first else { return nil } + isKeyframe = first != 0 + self.data = data.dropFirst() + } +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd shared/AVLiveWire && swift test --filter WirePayloadsTests` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Commit** + +```bash +git add shared/AVLiveWire +git commit -m "feat(avlivewire): skeleton and video payload codecs" +``` + +--- + +## Task 4: Stream demuxer + +The demuxer is fed arbitrary byte chunks (TCP delivers +non-frame-aligned data) and emits complete `(FrameHeader, Data)` +frames. It resyncs on the magic prefix if the stream is corrupt. + +**Files:** +- Create: `shared/AVLiveWire/Sources/AVLiveWire/StreamDemuxer.swift` +- Test: `shared/AVLiveWire/Tests/AVLiveWireTests/StreamDemuxerTests.swift` + +- [ ] **Step 1: Write the failing test** + +`shared/AVLiveWire/Tests/AVLiveWireTests/StreamDemuxerTests.swift`: + +```swift +import XCTest +@testable import AVLiveWire + +final class StreamDemuxerTests: XCTestCase { + private func frame(_ tag: FrameTag, _ payload: Data) -> Data { + let h = FrameHeader(tag: tag, pid: 0, timestamp: 1, + length: UInt32(payload.count)) + return h.encoded() + payload + } + + func testSingleFrame() { + var d = StreamDemuxer() + let out = d.feed(frame(.video, Data([1, 2, 3]))) + XCTAssertEqual(out.count, 1) + XCTAssertEqual(out[0].payload, Data([1, 2, 3])) + } + + func testSplitAcrossChunks() { + var d = StreamDemuxer() + let whole = frame(.video, Data([9, 9, 9, 9, 9])) + XCTAssertTrue(d.feed(whole.prefix(10)).isEmpty) + let out = d.feed(whole.dropFirst(10)) + XCTAssertEqual(out.count, 1) + XCTAssertEqual(out[0].payload, Data([9, 9, 9, 9, 9])) + } + + func testTwoFramesOneChunk() { + var d = StreamDemuxer() + let out = d.feed(frame(.meta, Data([1])) + + frame(.video, Data([2, 2]))) + XCTAssertEqual(out.count, 2) + } + + func testResyncAfterGarbage() { + var d = StreamDemuxer() + let out = d.feed(Data([0xDE, 0xAD]) + + frame(.video, Data([7]))) + XCTAssertEqual(out.count, 1) + XCTAssertEqual(out[0].payload, Data([7])) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd shared/AVLiveWire && swift test --filter StreamDemuxerTests` +Expected: FAIL — `StreamDemuxer` undefined. + +- [ ] **Step 3: Write the implementation** + +`shared/AVLiveWire/Sources/AVLiveWire/StreamDemuxer.swift`: + +```swift +import Foundation + +public struct StreamDemuxer { + public struct Frame: Equatable { + public let header: FrameHeader + public let payload: Data + } + + private var buffer = Data() + public init() {} + + /// Append bytes; return every complete frame now available. + public mutating func feed(_ chunk: Data) -> [Frame] { + buffer.append(chunk) + var out: [Frame] = [] + while true { + guard let start = findMagic() else { + // keep at most 3 trailing bytes (partial magic) + if buffer.count > 3 { + buffer = buffer.suffix(3) + } + break + } + if start > 0 { buffer.removeFirst(start) } + guard buffer.count >= FrameHeader.byteCount, + let h = FrameHeader(decoding: buffer) else { break } + let total = FrameHeader.byteCount + Int(h.length) + guard buffer.count >= total else { break } + let payloadStart = buffer.index( + buffer.startIndex, offsetBy: FrameHeader.byteCount) + let payloadEnd = buffer.index( + buffer.startIndex, offsetBy: total) + out.append(Frame(header: h, + payload: Data(buffer[payloadStart.. Int? { + let m = FrameHeader.magic + let bytes = [UInt8](buffer) + guard bytes.count >= m.count else { return nil } + for i in 0...(bytes.count - m.count) { + if Array(bytes[i.. Data { + let body = (try? PropertyListSerialization.data( + fromPropertyList: plist, format: .xml, options: 0)) + ?? Data() + var d = Data() + appendLE32(&d, UInt32(16 + body.count)) // length + appendLE32(&d, 1) // version + appendLE32(&d, 8) // message: plist + appendLE32(&d, tag) + d.append(body) + return d + } + + static func decode(_ packet: Data) -> [String: Any]? { + guard packet.count >= 16 else { return nil } + let body = packet.dropFirst(16) + return (try? PropertyListSerialization.propertyList( + from: body, options: [], format: nil)) as? [String: Any] + } + + static func appendLE32(_ d: inout Data, _ v: UInt32) { + for i in 0..<4 { d.append(UInt8((v >> (8 * i)) & 0xFF)) } + } + + static func readLE32(_ d: Data, _ offset: Int) -> UInt32 { + let b = [UInt8](d) + var v: UInt32 = 0 + for i in 0..<4 { v |= UInt32(b[offset + i]) << (8 * i) } + return v + } +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd launcher/AV-Live-Body && swift test --filter USBMuxProtocolTests` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Commit** + +```bash +git add launcher/AV-Live-Body +git commit -m "feat(av-live-body): usbmux message codec" +``` + +--- + +## Task 6: USBClient — device discovery + connect + +`USBClient` opens `/var/run/usbmuxd`, lists attached devices, and +connects to a chosen device's TCP port — yielding a byte stream that +is physically tunneled over USB. It is tested against an injectable +socket transport so no real device is needed. + +**Files:** +- Create: `launcher/AV-Live-Body/Sources/AVLiveBody/USBClient.swift` +- Test: `launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBClientTests.swift` + +- [ ] **Step 1: Write the failing test** + +`launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBClientTests.swift`: + +```swift +import XCTest +@testable import AVLiveBody + +/// In-memory stand-in for the usbmuxd Unix socket. +final class MockMuxTransport: MuxTransport { + var sent: [Data] = [] + var canned: [Data] = [] + func send(_ data: Data) { sent.append(data) } + func receivePacket() -> Data? { + canned.isEmpty ? nil : canned.removeFirst() + } + func close() {} +} + +final class USBClientTests: XCTestCase { + func testListDevicesParsesDeviceIDs() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode(plist: [ + "DeviceList": [ + ["DeviceID": 42, + "Properties": ["ConnectionType": "USB"]], + ]], tag: 0)] + let client = USBClient(transport: mock) + let devices = client.listDevices() + XCTAssertEqual(devices, [42]) + } + + func testConnectSendsConnectRequest() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode( + plist: ["MessageType": "Result", "Number": 0], tag: 0)] + let client = USBClient(transport: mock) + let ok = client.connect(deviceID: 42, port: 7000) + XCTAssertTrue(ok) + let req = USBMuxProtocol.decode(mock.sent.last!) + XCTAssertEqual(req?["MessageType"] as? String, "Connect") + XCTAssertEqual(req?["DeviceID"] as? Int, 42) + // usbmux expects the port byte-swapped to big-endian + XCTAssertEqual(req?["PortNumber"] as? Int, + Int((UInt16(7000) << 8) | (UInt16(7000) >> 8))) + } + + func testConnectFailsOnNonZeroResult() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode( + plist: ["MessageType": "Result", "Number": 3], tag: 0)] + let client = USBClient(transport: mock) + XCTAssertFalse(client.connect(deviceID: 1, port: 7000)) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd launcher/AV-Live-Body && swift test --filter USBClientTests` +Expected: FAIL — `MuxTransport` / `USBClient` undefined. + +- [ ] **Step 3: Write the implementation** + +`launcher/AV-Live-Body/Sources/AVLiveBody/USBClient.swift`: + +```swift +import Foundation + +/// Transport abstraction over the usbmuxd Unix socket. The real +/// implementation wraps a `socket(AF_UNIX)`; tests inject a mock. +protocol MuxTransport { + func send(_ data: Data) + func receivePacket() -> Data? + func close() +} + +/// usbmux client: device discovery + connect-to-port. After a +/// successful `connect`, the same transport carries the raw tunneled +/// byte stream from the device. +final class USBClient { + private let transport: MuxTransport + private var tag: UInt32 = 0 + + init(transport: MuxTransport) { + self.transport = transport + } + + func listDevices() -> [Int] { + tag += 1 + transport.send(USBMuxProtocol.encode( + plist: ["MessageType": "ListDevices"], tag: tag)) + guard let reply = transport.receivePacket(), + let plist = USBMuxProtocol.decode(reply), + let list = plist["DeviceList"] as? [[String: Any]] + else { return [] } + return list.compactMap { $0["DeviceID"] as? Int } + } + + /// Returns true once the transport is tunneled to `port` on the + /// device. usbmux wants the TCP port in big-endian order. + func connect(deviceID: Int, port: UInt16) -> Bool { + tag += 1 + let swapped = Int((port << 8) | (port >> 8)) + transport.send(USBMuxProtocol.encode(plist: [ + "MessageType": "Connect", + "DeviceID": deviceID, + "PortNumber": swapped, + ], tag: tag)) + guard let reply = transport.receivePacket(), + let plist = USBMuxProtocol.decode(reply), + let number = plist["Number"] as? Int + else { return false } + return number == 0 + } +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd launcher/AV-Live-Body && swift test --filter USBClientTests` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Commit** + +```bash +git add launcher/AV-Live-Body +git commit -m "feat(av-live-body): usbmux device discovery + connect" +``` + +--- + +## Task 7: Real Unix-socket transport + +`UnixMuxTransport` is the production `MuxTransport`: a blocking +`AF_UNIX` socket to `/var/run/usbmuxd`. It has no unit test (it needs +the daemon); it is exercised by the end-to-end smoke task. + +**Files:** +- Modify: `launcher/AV-Live-Body/Sources/AVLiveBody/USBClient.swift` + +- [ ] **Step 1: Append the real transport** + +Append to `USBClient.swift`: + +```swift +import Darwin + +/// Production transport: blocking AF_UNIX socket to usbmuxd. +final class UnixMuxTransport: MuxTransport { + private var fd: Int32 = -1 + + init?(path: String = "/var/run/usbmuxd") { + fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { return nil } + var addr = sockaddr_un() + addr.sun_family = sa_family_t(AF_UNIX) + _ = path.withCString { src in + withUnsafeMutablePointer(to: &addr.sun_path) { + $0.withMemoryRebound(to: CChar.self, capacity: 104) { + strcpy($0, src) + } + } + } + let size = socklen_t(MemoryLayout.size) + let ok = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.connect(fd, $0, size) + } + } + if ok != 0 { Darwin.close(fd); return nil } + } + + func send(_ data: Data) { + data.withUnsafeBytes { _ = Darwin.write(fd, $0.baseAddress, data.count) } + } + + /// Read one usbmux packet: 4-byte LE length prefix then body. + func receivePacket() -> Data? { + guard let head = readN(4) else { return nil } + let total = Int(USBMuxProtocol.readLE32(head, 0)) + guard total >= 16, let rest = readN(total - 4) else { return nil } + return head + rest + } + + /// Read raw tunneled bytes after a successful Connect. + func readStream(max: Int = 65536) -> Data? { + readN(max, exact: false) + } + + private func readN(_ n: Int, exact: Bool = true) -> Data? { + var buf = [UInt8](repeating: 0, count: n) + var got = 0 + while got < n { + let r = buf.withUnsafeMutableBytes { + Darwin.read(fd, $0.baseAddress!.advanced(by: got), n - got) + } + if r <= 0 { return got > 0 && !exact + ? Data(buf[0..= 0 { Darwin.close(fd) } } +} +``` + +- [ ] **Step 2: Run the existing suite to confirm no regression** + +Run: `cd launcher/AV-Live-Body && swift test --filter USBClientTests` +Expected: PASS, 3 tests (mock-based tests unaffected). + +- [ ] **Step 3: Commit** + +```bash +git add launcher/AV-Live-Body +git commit -m "feat(av-live-body): real usbmuxd unix socket transport" +``` + +--- + +## Task 8: iOS USBServer + +`USBServer` runs inside ARBodyTracker: an `NWListener` on a fixed +local TCP port. usbmuxd exposes that port to the tethered Mac. It +sends `AVLiveWire` frames and exposes a connection-state callback. + +**Files:** +- Create: `iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/USBServer.swift` + +- [ ] **Step 1: Write the implementation** + +`iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/USBServer.swift`: + +```swift +import Foundation +import Network +import AVLiveWire + +/// TCP listener on a fixed local port. usbmuxd tunnels it to the +/// tethered Mac — the port is never advertised on any network. +final class USBServer { + static let port: UInt16 = 7000 + + enum State { case idle, listening, connected } + var onState: ((State) -> Void)? + + private var listener: NWListener? + private var connection: NWConnection? + private let queue = DispatchQueue(label: "cc.avlive.usbserver") + + func start() { + let params = NWParameters.tcp + params.allowLocalEndpointReuse = true + listener = try? NWListener(using: params, + on: NWEndpoint.Port(rawValue: Self.port)!) + listener?.newConnectionHandler = { [weak self] conn in + self?.adopt(conn) + } + listener?.start(queue: queue) + onState?(.listening) + } + + private func adopt(_ conn: NWConnection) { + connection?.cancel() + connection = conn + conn.stateUpdateHandler = { [weak self] st in + switch st { + case .ready: self?.onState?(.connected) + case .failed, .cancelled: self?.onState?(.listening) + default: break + } + } + conn.start(queue: queue) + } + + /// Send one framed message. Drops silently if no peer. + func send(tag: FrameTag, pid: Int16, timestamp: Double, + payload: Data) { + guard let conn = connection else { return } + let header = FrameHeader(tag: tag, pid: pid, + timestamp: timestamp, length: UInt32(payload.count)) + conn.send(content: header.encoded() + payload, + completion: .contentProcessed { _ in }) + } + + func stop() { + connection?.cancel(); listener?.cancel() + onState?(.idle) + } +} +``` + +- [ ] **Step 2: Verify the iOS app target builds** + +Run: `cd iphone-arbody/ARBodyTracker.swiftpm && swift build` +Expected: build succeeds (the `AVLiveWire` dependency is added in +Task 9; until then this step is expected to fail on the missing +`import AVLiveWire` — proceed to Task 9, then re-run). + +- [ ] **Step 3: Commit** + +```bash +git add iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/USBServer.swift +git commit -m "feat(ios): USB TCP frame server" +``` + +--- + +## Task 9: Wire AVLiveWire into both apps + +**Files:** +- Modify: `launcher/AV-Live-Body/Package.swift` +- Modify: `iphone-arbody/ARBodyTracker.swiftpm/Package.swift` + +- [ ] **Step 1: Add the dependency to AV-Live-Body** + +In `launcher/AV-Live-Body/Package.swift`, add to `dependencies:`: + +```swift +.package(path: "../../shared/AVLiveWire"), +``` + +and add `"AVLiveWire"` to the `AVLiveBody` target's `dependencies` +array: + +```swift +.product(name: "AVLiveWire", package: "AVLiveWire"), +``` + +- [ ] **Step 2: Add the dependency to ARBodyTracker** + +In `iphone-arbody/ARBodyTracker.swiftpm/Package.swift`, add the same +`.package(path:)` entry (path relative to the `.swiftpm`: +`../../shared/AVLiveWire`) and the same `.product` to the app target. + +- [ ] **Step 3: Build both apps** + +Run: `cd launcher/AV-Live-Body && swift build` +Expected: build succeeds, `import AVLiveWire` resolves. + +Run: `cd iphone-arbody/ARBodyTracker.swiftpm && swift build` +Expected: build succeeds. + +- [ ] **Step 4: Commit** + +```bash +git add launcher/AV-Live-Body/Package.swift +git add iphone-arbody/ARBodyTracker.swiftpm/Package.swift +git commit -m "build: depend on shared AVLiveWire package" +``` + +--- + +## Task 10: End-to-end loopback smoke test + +Verify the transport end to end without a device, using a local TCP +loopback in place of the USB tunnel: a server sends frames, a client +demuxes them. + +**Files:** +- Test: `shared/AVLiveWire/Tests/AVLiveWireTests/LoopbackTests.swift` + +- [ ] **Step 1: Write the test** + +`shared/AVLiveWire/Tests/AVLiveWireTests/LoopbackTests.swift`: + +```swift +import XCTest +@testable import AVLiveWire + +/// Feeds an encoded frame stream through the demuxer in 7-byte +/// chunks — the worst-case fragmentation a TCP tunnel can produce. +final class LoopbackTests: XCTestCase { + func testManyFramesChunked() { + var skel = SkeletonPayload() + skel.valid[0] = true + skel.joints[0] = SIMD3(1, 1, 1) + + var stream = Data() + for i in 0..<20 { + let payload = skel.encoded() + let h = FrameHeader(tag: .skeleton, pid: Int16(i), + timestamp: Double(i), + length: UInt32(payload.count)) + stream += h.encoded() + payload + } + + var demux = StreamDemuxer() + var frames: [StreamDemuxer.Frame] = [] + var offset = 0 + while offset < stream.count { + let end = min(offset + 7, stream.count) + frames += demux.feed(stream[ + stream.index(stream.startIndex, offsetBy: offset)..< + stream.index(stream.startIndex, offsetBy: end)]) + offset = end + } + + XCTAssertEqual(frames.count, 20) + XCTAssertEqual(frames[5].header.pid, 5) + XCTAssertEqual(SkeletonPayload(decoding: frames[5].payload), + skel) + } +} +``` + +- [ ] **Step 2: Run the test** + +Run: `cd shared/AVLiveWire && swift test --filter LoopbackTests` +Expected: PASS, 1 test. + +- [ ] **Step 3: Run the full suites** + +Run: `cd shared/AVLiveWire && swift test` +Expected: PASS, all tests (Tasks 1-4 + loopback). + +Run: `cd launcher/AV-Live-Body && swift test --filter USBMuxProtocolTests --filter USBClientTests` +Expected: PASS, all usbmux tests. + +- [ ] **Step 4: Commit** + +```bash +git add shared/AVLiveWire +git commit -m "test(avlivewire): end-to-end chunked loopback" +``` + +--- + +## Manual verification (requires a tethered iPhone) + +Not a coded task — performed once the iOS app from Plan 2 streams +real frames: + +1. Tether the iPhone by USB; trust the Mac if prompted. +2. From the Mac, `UnixMuxTransport()` + `USBClient.listDevices()` + returns the device ID. +3. `connect(deviceID:port:7000)` succeeds while ARBodyTracker runs. +4. Bytes read from the transport, fed to `StreamDemuxer`, yield + `skeleton` frames. + +--- + +## Self-Review + +- **Spec coverage:** This plan covers the spec's `WireFormat`, + `StreamDemuxer`, `USBClient`, and `USBServer` units, and the + "usbmux native Swift client" decision. `VideoEncoder`, + `VideoDecoder`, `MultiHMRCoreML`, `BodyFusion`, `ARBodySession` + changes, and `PoseOSCBridge` are deliberately deferred to Plans 2 + and 3. +- **Placeholders:** none — every step carries complete code or an + exact command. +- **Type consistency:** `FrameHeader`, `FrameTag`, `SkeletonPayload`, + `VideoPayload`, `StreamDemuxer.Frame`, `MuxTransport`, `USBClient`, + `USBMuxProtocol`, `USBServer` are used with consistent signatures + across tasks. `USBServer.send` builds a `FrameHeader` exactly as + `StreamDemuxer` expects to parse it. +- **Known ordering note:** Task 8 Step 2 cannot fully build until + Task 9 adds the dependency — flagged inline in Task 8. diff --git a/docs/superpowers/plans/2026-05-18-macos-multihmr.md b/docs/superpowers/plans/2026-05-18-macos-multihmr.md new file mode 100644 index 0000000..d10475c --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-macos-multihmr.md @@ -0,0 +1,549 @@ +# macOS Multi-HMR Mesh Implementation Plan (Plan 3b of 3) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add the dense-mesh half of the macOS pipeline — run Multi-HMR (CoreML) on the USB video stream inside `AVLiveBody`, fuse the result with the ARKit skeleton, and render the SMPL-X body mesh. + +**Architecture:** `VideoDecoder` (Plan 3a) already turns `.video` frames into `CVPixelBuffer`s. This plan adds `MultiHMRCoreML`, a Swift wrapper around the bundled `multihmr_full_672_s.mlpackage`: it preprocesses a pixel buffer into the model's two `MLMultiArray` inputs, runs inference, and parses up to 4 detected persons (10475-vertex SMPL-X meshes). `BodyFusion` associates each mesh with the ARKit skeleton from `USBSkeletonConsumer` and corrects pelvis depth. The existing `MeshRenderer` (which already renders 10475-vertex SMPL-X meshes from its OSC server) is fed from the fusion output. + +**Tech Stack:** Swift 5, macOS 15, CoreML, CoreVideo/CoreImage, RealityKit, `AVLiveWire`, `XCTest`. Build verifies on the host with `swift build` / `swift test`. + +**Companion spec:** `docs/superpowers/specs/2026-05-18-iphone-usb-body-link-design.md` +**Prerequisites:** Plan 1, 2, 3a (merged); the working CoreML model (voie 2). + +--- + +## The model — exact I/O contract + +The reference implementation is `data_only_viz/multihmr_coreml.py` (Python, validated). The Swift wrapper must mirror it: + +- **File:** `~/.cache/av-live-multihmr/multihmr_full_672_s.mlpackage` (204 MB, FP32). Not in git (`*.mlpackage` is gitignored). +- **Load:** an `.mlpackage` must be compiled to `.mlmodelc` (`MLModel.compileModel(at:)`) before `MLModel(contentsOf:configuration:)`. Use `MLComputeUnits.cpuAndGPU` (benched best: ~139 ms standalone). +- **Inputs** (an `MLDictionaryFeatureProvider` with two `MLMultiArray`s): + - `"image"` — shape `[1, 3, 672, 672]`, Float32, RGB, **ImageNet-normalized**: `(v - mean) / std`, mean `[0.485, 0.456, 0.406]`, std `[0.229, 0.224, 0.225]` per channel. Feeding raw `[0,1]` collapses all scores (the "0 detections" bug). + - `"cam_K"` — shape `[1, 3, 3]`, Float32, camera intrinsics. +- **Outputs** (fixed K=4 persons): + - `var_2420` — v3d `[4, 10475, 3]` vertices + - `var_2423` — transl `[4, 1, 3]` pelvis translation + - `var_2436` — scores `[4]` + - `var_2439` — betas `[4, 10]`, `var_2442` — expression `[4, 10]` (unused here) +- **Detection:** keep person `k` when `scores[k] >= 0.3`. + +--- + +## File Structure + +| File | Responsibility | +|------|----------------| +| `launcher/AV-Live-Body/Sources/AVLiveBody/Resources/multihmr_full_672_s.mlpackage` | NEW (build input, gitignored). Copied from `~/.cache/av-live-multihmr/` by a setup step | +| `launcher/AV-Live-Body/Package.swift` | MODIFY. Declare the `.mlpackage` as a `.copy` resource | +| `launcher/AV-Live-Body/Sources/AVLiveBody/MultiHMRCoreML.swift` | NEW. Load the model; `CVPixelBuffer` → inputs → inference → `[MultiHMRPerson]` | +| `launcher/AV-Live-Body/Sources/AVLiveBody/BodyFusion.swift` | NEW. Associate ARKit skeleton ↔ Multi-HMR person; pelvis-depth correction | +| `launcher/AV-Live-Body/Tests/AVLiveBodyTests/BodyFusionTests.swift` | NEW. Pure association/correction logic tests | +| `launcher/AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift` | MODIFY. Drive `VideoDecoder` → `MultiHMRCoreML` → `BodyFusion` → `MeshRenderer` | +| `launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift` | REFERENCE — reuse its existing `updatePersons`-style entry point for 10475-vertex meshes | + +--- + +## Task 1: Bundle the model + loader + +**Files:** +- Create (copy): `launcher/AV-Live-Body/Sources/AVLiveBody/Resources/multihmr_full_672_s.mlpackage` +- Modify: `launcher/AV-Live-Body/Package.swift` + +- [ ] **Step 1: Copy the model into the package resources** + +The model is a build input that cannot live in git. Copy it: + +```bash +mkdir -p launcher/AV-Live-Body/Sources/AVLiveBody/Resources +cp -R ~/.cache/av-live-multihmr/multihmr_full_672_s.mlpackage \ + launcher/AV-Live-Body/Sources/AVLiveBody/Resources/ +``` + +Verify it is gitignored (root `.gitignore` has `*.mlpackage`): + +```bash +git check-ignore launcher/AV-Live-Body/Sources/AVLiveBody/Resources/multihmr_full_672_s.mlpackage +``` + +Expected: the path is printed (it is ignored — it must NOT be committed). + +If the source file is absent, STOP — Plan 3b is blocked until voie 2's +`.mlpackage` is regenerated (`data_only_viz/scripts/coreml_full_probe.py`). + +- [ ] **Step 2: Declare the resource in Package.swift** + +In `launcher/AV-Live-Body/Package.swift`, add to the `AVLiveBody` +executable target's `resources:` array (next to the existing +`smplx_faces.bin` / `scene.metal` copies): + +```swift + .copy("Resources/multihmr_full_672_s.mlpackage"), +``` + +- [ ] **Step 3: Verify the build still resolves resources** + +Run: `cd launcher/AV-Live-Body && swift build` +Expected: build succeeds; the `.mlpackage` is copied into the bundle. + +- [ ] **Step 4: Commit (Package.swift only — the model is gitignored)** + +```bash +git add launcher/AV-Live-Body/Package.swift +git commit -m "build(av-live-body): bundle Multi-HMR mlpackage" +``` + +--- + +## Task 2: MultiHMRCoreML + +`MultiHMRCoreML` loads the bundled model, preprocesses a `CVPixelBuffer` +into the two model inputs, runs inference, and returns detected persons. + +**Files:** +- Create: `launcher/AV-Live-Body/Sources/AVLiveBody/MultiHMRCoreML.swift` + +- [ ] **Step 1: Write the implementation** + +`launcher/AV-Live-Body/Sources/AVLiveBody/MultiHMRCoreML.swift`: + +```swift +import CoreML +import CoreVideo +import CoreImage +import Foundation + +/// One detected SMPL-X body from Multi-HMR. +struct MultiHMRPerson { + var vertices: [SIMD3] // 10475 SMPL-X verts, model space + var translation: SIMD3 // pelvis translation + var score: Float +} + +/// CoreML wrapper around the bundled `multihmr_full_672_s.mlpackage`. +/// Mirrors `data_only_viz/multihmr_coreml.py`: two MLMultiArray inputs +/// (`image` 1x3x672x672 ImageNet-normalized, `cam_K` 1x3x3), fixed +/// K=4 person outputs. +final class MultiHMRCoreML { + static let inputSize = 672 + static let vertexCount = 10475 + static let maxPersons = 4 + private static let detThreshold: Float = 0.3 + private static let normMean: [Float] = [0.485, 0.456, 0.406] + private static let normStd: [Float] = [0.229, 0.224, 0.225] + + private let model: MLModel + private let ciContext = CIContext() + + /// Loads the bundled model. Returns nil if the resource or load + /// fails — callers fall back to skeleton-only rendering. + init?() { + guard let url = Bundle.module.url( + forResource: "multihmr_full_672_s", + withExtension: "mlpackage") else { + NSLog("MultiHMRCoreML: mlpackage resource missing") + return nil + } + let cfg = MLModelConfiguration() + cfg.computeUnits = .cpuAndGPU + do { + let compiled = try MLModel.compileModel(at: url) + model = try MLModel(contentsOf: compiled, configuration: cfg) + } catch { + NSLog("MultiHMRCoreML: load failed %@", + String(describing: error)) + return nil + } + } + + /// Run inference on one camera frame. `cameraK` is the 3x3 camera + /// intrinsics row-major. + func infer(_ pixelBuffer: CVPixelBuffer, + cameraK: [Float]) -> [MultiHMRPerson] { + guard let image = makeImageInput(pixelBuffer), + let k = makeKInput(cameraK) else { return [] } + let inputs: [String: MLFeatureValue] = [ + "image": MLFeatureValue(multiArray: image), + "cam_K": MLFeatureValue(multiArray: k), + ] + guard let provider = try? MLDictionaryFeatureProvider( + dictionary: inputs), + let out = try? model.prediction(from: provider) else { + return [] + } + return parse(out) + } + + // MARK: - Input preprocessing + + /// `CVPixelBuffer` -> [1,3,672,672] Float32, RGB, ImageNet-normed. + private func makeImageInput(_ pb: CVPixelBuffer) -> MLMultiArray? { + let n = Self.inputSize + // Resize to n x n BGRA via CoreImage. + let ci = CIImage(cvPixelBuffer: pb) + let sx = CGFloat(n) / ci.extent.width + let sy = CGFloat(n) / ci.extent.height + let scaled = ci.transformed( + by: CGAffineTransform(scaleX: sx, y: sy)) + var dst: CVPixelBuffer? + CVPixelBufferCreate(kCFAllocatorDefault, n, n, + kCVPixelFormatType_32BGRA, nil, &dst) + guard let dst else { return nil } + ciContext.render(scaled, to: dst) + CVPixelBufferLockBaseAddress(dst, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(dst, .readOnly) } + guard let base = CVPixelBufferGetBaseAddress(dst) else { + return nil + } + let rowBytes = CVPixelBufferGetBytesPerRow(dst) + let px = base.assumingMemoryBound(to: UInt8.self) + guard let arr = try? MLMultiArray( + shape: [1, 3, NSNumber(value: n), NSNumber(value: n)], + dataType: .float32) else { return nil } + let ptr = arr.dataPointer.assumingMemoryBound(to: Float.self) + let plane = n * n + for y in 0.. [1,3,3] Float32. + private func makeKInput(_ k: [Float]) -> MLMultiArray? { + guard k.count == 9, + let arr = try? MLMultiArray( + shape: [1, 3, 3], dataType: .float32) else { return nil } + let ptr = arr.dataPointer.assumingMemoryBound(to: Float.self) + for i in 0..<9 { ptr[i] = k[i] } + return arr + } + + // MARK: - Output parsing + + private func parse(_ out: MLFeatureProvider) -> [MultiHMRPerson] { + guard let v3d = out.featureValue(for: "var_2420")? + .multiArrayValue, + let transl = out.featureValue(for: "var_2423")? + .multiArrayValue, + let scores = out.featureValue(for: "var_2436")? + .multiArrayValue else { return [] } + var persons: [MultiHMRPerson] = [] + let vc = Self.vertexCount + for k in 0..]( + repeating: .zero, count: vc) + let base = k * vc * 3 + for i in 0.. ArkitOSCListener.ArkitBodyFrame { + var f = ArkitOSCListener.ArkitBodyFrame() + f.pid = 0 + // ARKit body skeleton joint 0 is the hips/pelvis root. + f.joints[0] = SIMD3(0, 0, pelvisZ) + f.hasJoint[0] = true + return f + } + + func testPelvisDepthOverride() { + let mesh = MultiHMRPerson( + vertices: [SIMD3](repeating: .zero, count: 1), + translation: SIMD3(0, 0, -1.0), score: 0.9) + let fused = BodyFusion.fuse( + persons: [mesh], skeletons: [0: skeleton(pelvisZ: -2.5)]) + XCTAssertEqual(fused.count, 1) + XCTAssertEqual(fused[0].translation.z, -2.5, accuracy: 1e-4) + } + + func testPassthroughWhenNoSkeleton() { + let mesh = MultiHMRPerson( + vertices: [SIMD3](repeating: .zero, count: 1), + translation: SIMD3(0, 0, -1.0), score: 0.9) + let fused = BodyFusion.fuse(persons: [mesh], skeletons: [:]) + XCTAssertEqual(fused[0].translation.z, -1.0, accuracy: 1e-4) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd launcher/AV-Live-Body && swift test --filter BodyFusionTests` +Expected: FAIL — `BodyFusion` undefined. + +- [ ] **Step 3: Write the implementation** + +`launcher/AV-Live-Body/Sources/AVLiveBody/BodyFusion.swift`: + +```swift +import AVLiveWire +import Foundation +import simd + +/// Associates Multi-HMR meshes with ARKit skeletons and corrects the +/// mesh pelvis depth. Pure, stateless — unit-testable. +enum BodyFusion { + /// ARKit body skeleton root (hips) joint index. + static let pelvisJoint = 0 + + /// Returns the persons with `translation.z` of each replaced by + /// the matching ARKit skeleton's pelvis Z when one is available. + /// Association is nearest-translation; with a single skeleton and + /// a single dominant person this is exact. + static func fuse(persons: [MultiHMRPerson], + skeletons: [Int: ArkitOSCListener.ArkitBodyFrame]) + -> [MultiHMRPerson] { + // Collect candidate ARKit pelvis depths. + let pelvisZs: [Float] = skeletons.values.compactMap { s in + guard pelvisJoint < s.hasJoint.count, + s.hasJoint[pelvisJoint] else { return nil } + return s.joints[pelvisJoint].z + } + guard !pelvisZs.isEmpty else { return persons } + // Highest-scoring person is the primary; lock its depth to the + // single ARKit skeleton (ARKit tracks one body). Others pass + // through unchanged. + guard let primaryIdx = persons.indices.max(by: { + persons[$0].score < persons[$1].score + }) else { return persons } + var out = persons + out[primaryIdx].translation.z = pelvisZs[0] + return out + } +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd launcher/AV-Live-Body && swift test --filter BodyFusionTests` +Expected: PASS, 2 tests. + +- [ ] **Step 5: Run the full suite + commit** + +Run: `cd launcher/AV-Live-Body && swift test` — Expected: all pass +(9: prior 7 + 2). + +```bash +git add launcher/AV-Live-Body/Sources/AVLiveBody/BodyFusion.swift launcher/AV-Live-Body/Tests/AVLiveBodyTests/BodyFusionTests.swift +git commit -m "feat(av-live-body): ARKit-to-mesh body fusion" +``` + +--- + +## Task 4: Wire the mesh pipeline + +Drive the chain: `USBSkeletonConsumer.onVideo` → `VideoDecoder` → +`MultiHMRCoreML` → `BodyFusion` → `MeshRenderer`. + +**Files:** +- Modify: `launcher/AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift` +- Reference: `launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift` + +- [ ] **Step 1: Read `MeshRenderer.swift`** + +Identify the method that ingests SMPL-X persons (the OSC `SMPX` server +path calls it — likely `updatePersons(_:)` taking per-person 10475 +vertex arrays). Note its exact signature and the vertex/coordinate +convention it expects. + +- [ ] **Step 2: Add the mesh pipeline to `USBSkeletonConsumer`** + +Give `USBSkeletonConsumer` an optional mesh pipeline. Add stored +properties: + +```swift + private let videoDecoder = VideoDecoder() + private let multiHMR = MultiHMRCoreML() + /// Set by the app to receive fused mesh persons on the main queue. + var onMeshPersons: (([MultiHMRPerson]) -> Void)? + /// Camera intrinsics (row-major 3x3) for Multi-HMR; a sane default + /// is the iPhone main-camera focal at 672 px until a `.meta` frame + /// supplies the real values. + private var cameraK: [Float] = [ + 672, 0, 336, + 0, 672, 336, + 0, 0, 1, + ] +``` + +In `init()` (or `start()`), wire the decoder to the model: + +```swift + videoDecoder.onFrame = { [weak self] pixelBuffer in + guard let self else { return } + guard let hmr = self.multiHMR else { return } + let raw = hmr.infer(pixelBuffer, cameraK: self.cameraK) + let latestSkeletons = self.bodies + let fused = BodyFusion.fuse( + persons: raw, skeletons: latestSkeletons) + DispatchQueue.main.async { + self.onMeshPersons?(fused) + } + } +``` + +Change the `.video` branch of `route(_:)` so it feeds the decoder +instead of only forwarding the payload: + +```swift + case .video: + guard let payload = + VideoPayload(decoding: frame.payload) else { return } + videoDecoder.decode(payload) +``` + +(`onVideo` may be kept for diagnostics or removed — keeping it is +harmless; if removed, delete its declaration too.) + +- [ ] **Step 3: Feed `MeshRenderer` from the app** + +In `AVLiveBodyApp.swift`'s `ContentView` `.onAppear` (or where the +renderers are wired), set `usbConsumer.onMeshPersons` to call the +`MeshRenderer` ingest method identified in Step 1, converting +`[MultiHMRPerson]` (vertices + fused translation) into whatever shape +that method expects. The translation from `BodyFusion` positions each +mesh; the 10475 vertices are the SMPL-X surface. + +If `MeshRenderer`'s ingest method is not reachable from `ContentView` +(it may be owned by `BodyView`), thread an `onMeshPersons` closure the +same way `usbConsumer` itself was threaded in Plan 3a Task 4. + +- [ ] **Step 4: Verify build + tests** + +Run: `cd launcher/AV-Live-Body && swift build && swift test` +Expected: build succeeds; all tests pass (9). + +- [ ] **Step 5: Commit** + +```bash +git add launcher/AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift +git commit -m "feat(av-live-body): wire Multi-HMR mesh pipeline" +``` + +(Include `BodyView.swift` in the commit if Step 3 threaded a closure +through it.) + +--- + +## Task 5: Final verification + +- [ ] **Step 1: Clean build + full test suite** + +```bash +cd launcher/AV-Live-Body && swift build && swift test +``` + +Expected: build succeeds; all 9 tests pass. + +- [ ] **Step 2: Confirm the model is bundled, not committed** + +```bash +git status --porcelain | grep mlpackage || echo "model not staged — correct" +ls -d launcher/AV-Live-Body/Sources/AVLiveBody/Resources/multihmr_full_672_s.mlpackage +``` + +Expected: the model directory exists on disk but is NOT staged in git. + +--- + +## Self-Review + +- **Spec coverage:** This plan implements the spec's `MultiHMRCoreML`, + `BodyFusion`, and the mesh-render wiring — the dense-mesh half + deferred from Plan 3a. With Plan 3b done, the full spec + (`USBClient`/`StreamDemuxer`/`VideoDecoder`/`MultiHMRCoreML`/ + `BodyFusion` + renderers) is covered. +- **Placeholders:** none — new files carry complete code; modify tasks + cite exact files and instruct reading `MeshRenderer.swift` for the + one signature this plan cannot reproduce blind. +- **Type consistency:** `MultiHMRPerson` is produced by + `MultiHMRCoreML.infer` and consumed by `BodyFusion.fuse` and + `onMeshPersons`. The model I/O names (`image`, `cam_K`, `var_2420`, + `var_2423`, `var_2436`) match `multihmr_coreml.py` exactly. +- **Known risks:** + 1. **Bundling 204 MB** — `swift build` copies the `.mlpackage` into + the app bundle; build is slower and the app is large. Acceptable + per the owner's decision (FP32, validated). + 2. **`CVPixelBuffer` → tensor** — the CoreImage resize + manual + BGRA→normalized-CHW packing is the most error-prone code here and + needs on-device validation against `multihmr_coreml.py`'s output + on the same frame. It also runs per-frame on the CPU — a perf + hotspot; revisit with `vImage`/Metal if frame rate suffers. + 3. **~7.6 fps** — Multi-HMR is far below 30 fps; the mesh layer is + slow while the skeleton (Plan 3a) stays real-time. `MeshRenderer` + already interpolates meshes to ~60 fps between worker frames — + reuse that, do not block the USB read loop on inference (the + `videoDecoder.onFrame` callback already runs off the main queue). + 4. **`cameraK`** — a placeholder intrinsics matrix is used until a + `.meta` frame carries the real values; absolute depth scale will + be approximate until then. A future iteration should send camera + intrinsics from the iPhone in a `.meta` frame. diff --git a/docs/superpowers/plans/2026-05-18-macos-usb-consumer.md b/docs/superpowers/plans/2026-05-18-macos-usb-consumer.md new file mode 100644 index 0000000..e517221 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-macos-usb-consumer.md @@ -0,0 +1,655 @@ +# macOS USB Consumer Implementation Plan (Plan 3a of 3) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the macOS `AVLiveBody` app consume the iPhone's USB stream — connect via `usbmuxd`, demux `AVLiveWire` frames, render the 91-joint skeleton on screen, and HEVC-decode the video — without the Multi-HMR dense-mesh step (deferred to Plan 3b). + +**Architecture:** A new `USBSkeletonConsumer` runs the blocking `UnixMuxTransport`/`USBClient` read loop on a dedicated background thread, feeds bytes through `StreamDemuxer`, and republishes `.skeleton` frames as `@Published` ARKit-shaped body frames plus a `.video` callback. `Skeleton3DRenderer`'s long-standing `// TODO: render yellow ARKit markers` (line 138) is completed so the 91-joint USB skeleton actually draws. A new `VideoDecoder` turns `.video` `VideoPayload`s into `CVPixelBuffer`s via `VTDecompressionSession`. + +**Tech Stack:** Swift 5 (language mode v5), macOS 15, RealityKit, VideoToolbox, `AVLiveWire` (already a dependency of `AV-Live-Body`), `XCTest`. + +**Companion spec:** `docs/superpowers/specs/2026-05-18-iphone-usb-body-link-design.md` +**Prerequisites:** Plan 1 (transport, merged), Plan 2 (iOS capture, merged). +**Out of scope:** `MultiHMRCoreML`, `BodyFusion`, dense-mesh rendering — Plan 3b, gated on a confirmed CoreML Multi-HMR `.mlpackage`. + +--- + +## Verification + +`AV-Live-Body` is a macOS target — it builds on the host: + +```bash +cd launcher/AV-Live-Body && swift build +cd launcher/AV-Live-Body && swift test +``` + +Each task ends with `swift build` (and `swift test` where a test was +added) succeeding. + +--- + +## File Structure + +| File | Responsibility | +|------|----------------| +| `launcher/AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift` | NEW. Background USB read loop → `StreamDemuxer` → `@Published` body frames + video callback | +| `launcher/AV-Live-Body/Sources/AVLiveBody/VideoDecoder.swift` | NEW. `VTDecompressionSession` HEVC decode: `VideoPayload` → `CVPixelBuffer` | +| `launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBSkeletonConsumerTests.swift` | NEW. Unit test for the `SkeletonPayload` → `ArkitBodyFrame` mapping | +| `launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift` | MODIFY. Complete the line-138 TODO: draw 91 USB-skeleton joint markers | +| `launcher/AV-Live-Body/Sources/AVLiveBody/ArkitOSCListener.swift` | REFERENCE only — reuse its nested `ArkitBodyFrame` type | +| `launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift` | MODIFY. Own a `USBSkeletonConsumer`, start it in `.onAppear` | +| `launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift` | MODIFY. Thread the consumer into `Skeleton3DRenderer.attach` | + +--- + +## Task 1: USBSkeletonConsumer + +`USBSkeletonConsumer` owns the blocking USB read loop on a background +`Thread`. It reconnects on drop. It republishes `.skeleton` frames as +`ArkitOSCListener.ArkitBodyFrame` (the existing 91-joint body type, so +`Skeleton3DRenderer` can consume them with no new type) and forwards +`.video` payloads via a callback. It is **not** `@MainActor`: the loop +runs off-main and hops to main only for `@Published` writes — the same +pattern as `ArkitOSCListener`. + +**Files:** +- Create: `launcher/AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift` +- Test: `launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBSkeletonConsumerTests.swift` + +- [ ] **Step 1: Write the failing test** + +`launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBSkeletonConsumerTests.swift`: + +```swift +import XCTest +import AVLiveWire +@testable import AVLiveBody + +final class USBSkeletonConsumerTests: XCTestCase { + func testSkeletonPayloadMapsToBodyFrame() { + var p = SkeletonPayload() + p.joints[0] = SIMD3(1, 2, 3) + p.valid[0] = true + p.joints[90] = SIMD3(-4, 5, -6) + p.valid[90] = true + let frame = USBSkeletonConsumer.bodyFrame(pid: 7, from: p) + XCTAssertEqual(frame.pid, 7) + XCTAssertEqual(frame.joints.count, 91) + XCTAssertEqual(frame.hasJoint.count, 91) + XCTAssertEqual(frame.joints[0], SIMD3(1, 2, 3)) + XCTAssertTrue(frame.hasJoint[0]) + XCTAssertEqual(frame.joints[90], SIMD3(-4, 5, -6)) + XCTAssertFalse(frame.hasJoint[1]) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd launcher/AV-Live-Body && swift test --filter USBSkeletonConsumerTests` +Expected: FAIL — `USBSkeletonConsumer` undefined. + +- [ ] **Step 3: Write the implementation** + +`launcher/AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift`: + +```swift +import AVLiveWire +import Combine +import Foundation + +/// Connects to the tethered iPhone over USB (usbmuxd), demuxes the +/// AVLiveWire stream, and republishes skeleton frames (as the existing +/// 91-joint `ArkitOSCListener.ArkitBodyFrame`) plus video payloads. +/// The blocking transport runs on a dedicated background thread; only +/// `@Published` writes hop to the main queue. +final class USBSkeletonConsumer: ObservableObject { + /// 91-joint body frames keyed by pid — same shape `Skeleton3DRenderer` + /// already consumes from `ArkitOSCListener`. + @Published var bodies: [Int: ArkitOSCListener.ArkitBodyFrame] = [:] + @Published var connected = false + + /// Called (on the main queue) for every decoded `.video` frame. + var onVideo: ((VideoPayload) -> Void)? + + /// TCP port the iPhone `USBServer` listens on (must match the iOS + /// app's `USBServer.port`). + static let devicePort: UInt16 = 7000 + + private let stateLock = NSLock() + private var running = false + private var thread: Thread? + + private var isRunning: Bool { + stateLock.lock(); defer { stateLock.unlock() } + return running + } + + func start() { + stateLock.lock() + if running { stateLock.unlock(); return } + running = true + stateLock.unlock() + let t = Thread { [weak self] in self?.loop() } + t.name = "cc.avlive.usbconsumer" + t.start() + thread = t + } + + func stop() { + stateLock.lock(); running = false; stateLock.unlock() + } + + /// Pure mapping `SkeletonPayload` -> `ArkitBodyFrame`. Static so it + /// is unit-testable without a transport. + static func bodyFrame(pid: Int, from p: SkeletonPayload) + -> ArkitOSCListener.ArkitBodyFrame { + var f = ArkitOSCListener.ArkitBodyFrame() + f.pid = pid + f.joints = p.joints + f.hasJoint = p.valid + f.seenAt = CFAbsoluteTimeGetCurrent() + return f + } + + // MARK: - Background read loop + + private func loop() { + while isRunning { + guard let transport = UnixMuxTransport() else { + Thread.sleep(forTimeInterval: 1.0); continue + } + let client = USBClient(transport: transport) + guard let dev = client.listDevices().first, + client.connect(deviceID: dev, + port: Self.devicePort) else { + transport.close() + Thread.sleep(forTimeInterval: 1.0); continue + } + publishConnected(true) + var demux = StreamDemuxer() + while isRunning { + guard let chunk = transport.readStream(), + !chunk.isEmpty else { break } + for frame in demux.feed(chunk) { route(frame) } + } + transport.close() + publishConnected(false) + if isRunning { Thread.sleep(forTimeInterval: 1.0) } + } + } + + private func route(_ frame: StreamDemuxer.Frame) { + switch frame.header.tag { + case .skeleton: + guard let payload = + SkeletonPayload(decoding: frame.payload) else { return } + let pid = Int(frame.header.pid) + let body = Self.bodyFrame(pid: pid, from: payload) + DispatchQueue.main.async { [weak self] in + self?.bodies[pid] = body + } + case .video: + guard let payload = + VideoPayload(decoding: frame.payload) else { return } + DispatchQueue.main.async { [weak self] in + self?.onVideo?(payload) + } + case .meta: + break + } + } + + private func publishConnected(_ value: Bool) { + DispatchQueue.main.async { [weak self] in + self?.connected = value + } + } +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `cd launcher/AV-Live-Body && swift test --filter USBSkeletonConsumerTests` +Expected: PASS, 1 test. + +If `ArkitOSCListener.ArkitBodyFrame` has no memberwise mutability or a +different field set than `pid`/`joints`/`hasJoint`/`seenAt`, read +`ArkitOSCListener.swift` and adjust `bodyFrame` to match the actual +struct (it is a `struct ArkitBodyFrame: Equatable` with `var pid`, +`var joints: [SIMD3]`, `var hasJoint: [Bool]`, `var seenAt`). + +- [ ] **Step 5: Run the full suite + commit** + +Run: `cd launcher/AV-Live-Body && swift test` +Expected: PASS, all tests (7: prior 6 + this 1). + +```bash +git add launcher/AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift launcher/AV-Live-Body/Tests/AVLiveBodyTests/USBSkeletonConsumerTests.swift +git commit -m "feat(av-live-body): USB skeleton consumer" +``` + +(subject ≤50 chars; add a short body — the hook rejects subject-only.) + +--- + +## Task 2: VideoDecoder + +`VideoDecoder` turns `.video` `VideoPayload`s into `CVPixelBuffer`s. A +keyframe payload carries the HEVC parameter sets prepended (each as a +4-byte big-endian length prefix + NAL bytes — the format Plan 2's iOS +`VideoEncoder` produces); the decoder builds its +`CMVideoFormatDescription` from those, then decodes subsequent access +units. + +**Files:** +- Create: `launcher/AV-Live-Body/Sources/AVLiveBody/VideoDecoder.swift` + +- [ ] **Step 1: Write the implementation** + +`launcher/AV-Live-Body/Sources/AVLiveBody/VideoDecoder.swift`: + +```swift +import AVLiveWire +import CoreMedia +import CoreVideo +import Foundation +import VideoToolbox + +/// HEVC decoder. Feed `VideoPayload`s in; receive `CVPixelBuffer`s via +/// `onFrame`. Keyframe payloads must carry the VPS/SPS/PPS parameter +/// sets prepended as 4-byte-length-prefixed NAL units (the layout the +/// iOS `VideoEncoder` emits); the decoder (re)builds its format +/// description from those. +final class VideoDecoder { + var onFrame: ((CVPixelBuffer) -> Void)? + + private var session: VTDecompressionSession? + private var formatDesc: CMVideoFormatDescription? + + /// Decode one access unit. + func decode(_ payload: VideoPayload) { + var au = payload.data + if payload.isKeyframe { + // Split the prepended parameter sets from the frame data. + let (params, rest) = Self.splitParameterSets(au) + if !params.isEmpty { + rebuildFormat(params) + } + au = rest + } + guard let fmt = formatDesc, !au.isEmpty else { return } + if session == nil { makeSession(fmt) } + guard let session else { return } + guard let block = Self.blockBuffer(au) else { return } + var sample: CMSampleBuffer? + var sampleSize = au.count + guard CMSampleBufferCreateReady( + allocator: kCFAllocatorDefault, dataBuffer: block, + formatDescription: fmt, sampleCount: 1, sampleTimingEntryCount: 0, + sampleTimingArray: nil, sampleSizeEntryCount: 1, + sampleSizeArray: &sampleSize, + sampleBufferOut: &sample) == noErr, let sample else { return } + VTDecompressionSessionDecodeFrame( + session, sampleBuffer: sample, flags: [], + infoFlagsOut: nil) { [weak self] status, _, image, _, _ in + guard status == noErr, let image else { return } + self?.onFrame?(image) + } + } + + func stop() { + if let session { VTDecompressionSessionInvalidate(session) } + session = nil + formatDesc = nil + } + + deinit { stop() } + + // MARK: - Helpers + + /// Parameter sets are 4-byte-length-prefixed NAL units at the head + /// of a keyframe payload. The first NAL whose type is a VCL slice + /// marks the start of frame data — but to stay simple and robust, + /// we treat every leading NAL as a parameter set until the running + /// concatenation can build a valid HEVC format description; the + /// remainder is the frame. Returns (parameterSetData, frameData). + private static func splitParameterSets(_ data: Data) + -> (Data, Data) { + // Parameter set NALs for HEVC: VPS=32, SPS=33, PPS=34 + // (nal_unit_type = (firstByte >> 1) & 0x3F). + var offset = 0 + let bytes = [UInt8](data) + var paramEnd = 0 + while offset + 4 <= bytes.count { + let len = (Int(bytes[offset]) << 24) + | (Int(bytes[offset + 1]) << 16) + | (Int(bytes[offset + 2]) << 8) + | Int(bytes[offset + 3]) + let nalStart = offset + 4 + guard len > 0, nalStart + len <= bytes.count else { break } + let nalType = (Int(bytes[nalStart]) >> 1) & 0x3F + if nalType == 32 || nalType == 33 || nalType == 34 { + offset = nalStart + len + paramEnd = offset + } else { + break + } + } + return (data.prefix(paramEnd), + data.suffix(from: data.startIndex + .advanced(by: paramEnd))) + } + + private func rebuildFormat(_ paramData: Data) { + var sets: [[UInt8]] = [] + let bytes = [UInt8](paramData) + var offset = 0 + while offset + 4 <= bytes.count { + let len = (Int(bytes[offset]) << 24) + | (Int(bytes[offset + 1]) << 16) + | (Int(bytes[offset + 2]) << 8) + | Int(bytes[offset + 3]) + let start = offset + 4 + guard len > 0, start + len <= bytes.count else { break } + sets.append(Array(bytes[start..= 3 else { return } + let pointers = sets.map { UnsafePointer($0) } + let sizes = sets.map { $0.count } + var fmt: CMFormatDescription? + let status = pointers.withUnsafeBufferPointer { pBuf in + sizes.withUnsafeBufferPointer { sBuf in + CMVideoFormatDescriptionCreateFromHEVCParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: sets.count, + parameterSetPointers: pBuf.baseAddress!, + parameterSetSizes: sBuf.baseAddress!, + nalUnitHeaderLength: 4, extensions: nil, + formatDescriptionOut: &fmt) + } + } + if status == noErr, let fmt { + formatDesc = fmt + if let session { VTDecompressionSessionInvalidate(session) } + session = nil + } + } + + private func makeSession(_ fmt: CMVideoFormatDescription) { + let attrs: [CFString: Any] = [ + kCVPixelBufferPixelFormatTypeKey: + kCVPixelFormatType_32BGRA, + ] + VTDecompressionSessionCreate( + allocator: kCFAllocatorDefault, formatDescription: fmt, + decoderSpecification: nil, + imageBufferAttributes: attrs as CFDictionary, + outputCallback: nil, decompressionSessionOut: &session) + } + + private static func blockBuffer(_ data: Data) -> CMBlockBuffer? { + var block: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, memoryBlock: nil, + blockLength: data.count, blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, offsetToData: 0, + dataLength: data.count, flags: 0, + blockBufferOut: &block) == noErr, let block else { + return nil + } + var ok = false + data.withUnsafeBytes { raw in + if CMBlockBufferReplaceDataBytes( + with: raw.baseAddress!, blockBuffer: block, + offsetIntoDestination: 0, + dataLength: data.count) == noErr { ok = true } + } + return ok ? block : nil + } +} +``` + +- [ ] **Step 2: Verify it compiles** + +Run: `cd launcher/AV-Live-Body && swift build` +Expected: build succeeds. If a VideoToolbox/CoreMedia signature differs +on this SDK, fix minimally — the behavior (build a format description +from the prepended parameter sets, decode the rest) must be preserved. + +- [ ] **Step 3: Commit** + +```bash +git add launcher/AV-Live-Body/Sources/AVLiveBody/VideoDecoder.swift +git commit -m "feat(av-live-body): HEVC video decoder" +``` + +--- + +## Task 3: Render the 91-joint USB skeleton + +`Skeleton3DRenderer` already subscribes to a 91-joint ARKit body +publisher into `lastArkit` but never draws it — `Skeleton3DRenderer.swift:138` +is `// TODO: render yellow ARKit markers from lastArkit in update()`. +Complete it: draw the 91 joints as small yellow spheres. + +**Files:** +- Modify: `launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift` + +- [ ] **Step 1: Read the renderer** + +Read `Skeleton3DRenderer.swift` fully. Note: `PersonEntities` (the +per-pid entity struct), `lastArkit: [Int: ArkitOSCListener.ArkitBodyFrame]`, +`makePerson(pid:parent:)`, the `update(frames:)` 30 fps tick, and the +RealityKit space conversion used for MediaPipe joints +(`SIMD3(k.x, -k.y, -k.z)`). + +- [ ] **Step 2: Add 91 ARKit marker entities to `PersonEntities`** + +In the `PersonEntities` struct, add a field: + +```swift + var arkitMarkers: [ModelEntity] // 91 yellow ARKit joint spheres +``` + +In `makePerson(pid:parent:)`, after the hand spheres are built, create +91 yellow marker spheres (reuse the `jointRadius`-sized sphere mesh, a +yellow `SimpleMaterial`), parent them to `root`, start them disabled, +and include `arkitMarkers:` in the returned `PersonEntities(...)`: + +```swift + let arkitMat = SimpleMaterial( + color: .systemYellow, roughness: 0.6, isMetallic: false) + var arkitMarkers: [ModelEntity] = [] + arkitMarkers.reserveCapacity(91) + for _ in 0..<91 { + let e = ModelEntity(mesh: sphereMesh, materials: [arkitMat]) + e.isEnabled = false + root.addChild(e) + arkitMarkers.append(e) + } +``` + +- [ ] **Step 3: Draw the ARKit markers each tick** + +Replace the line `// TODO: render yellow ARKit markers from lastArkit in update()` +(`Skeleton3DRenderer.swift:138`) — leave the comment removed — and add, +at the end of `update(frames:)` (after the existing per-pid loop), a +call to a new private method `applyArkit()`. Then add the method: + +```swift + /// Draw the 91-joint ARKit/USB skeletons as yellow joint markers. + /// ARKit joints are world-space metric; convert to RealityKit + /// space (x, y, z) -> (x, -y, -z) like the MediaPipe path. + private func applyArkit() { + for (pid, entities) in persons { + guard let frame = lastArkit[pid] else { + for m in entities.arkitMarkers { m.isEnabled = false } + continue + } + let n = min(91, entities.arkitMarkers.count, + frame.joints.count) + for i in 0..(j.x, -j.y, -j.z) + marker.isEnabled = true + } else { + marker.isEnabled = false + } + } + for i in n.. **Status:** design approved (brainstorming), pending implementation plan. +> **Date:** 2026-05-18 + +## Goal + +Rebuild the macOS `AVLiveBody` app from scratch as a clean, native +Xcode application focused solely on the iPhone-USB body pipeline: +display the iPhone camera video and the tracked body (91-joint +skeleton + SMPL-X mesh) in a single RealityKit 3D scene. Drop all the +legacy components that have made incremental work fragile. + +## Motivation + +The existing `launcher/AV-Live-Body` is a SwiftPM package carrying +years of unrelated functionality — MediaPipe OSC listeners, openFrame- +works-style Metal "viz mode" scenes, a data-feeds HUD, a 33-joint +MediaPipe skeleton renderer, Mac-webcam capture, viz-mode hotkeys, a +multi-layer `BodyView`. Bolting the iPhone-USB pipeline onto it caused +recurring friction: the skeleton render tick was coupled to the +MediaPipe publisher, the app could not take keyboard focus when run as +a bare SwiftPM executable, the camera defaulted to the Mac webcam. A +clean, purpose-built app removes that whole class of problems. + +## Decisions (brainstorming outcomes) + +1. **Fresh native macOS app, Xcode project**, xcodegen-managed + (`project.yml` → `.xcodeproj`, matching the `iphone-arbody` iOS + app). New directory `avlivebody-mac/` in the AV-Live monorepo. The + old `launcher/AV-Live-Body/` is archived. +2. **Reuse the clean USB pipeline** built previously — the + `AVLiveWire` package plus `USBMuxProtocol`, `USBClient`, + `UnixMuxTransport`, `VideoDecoder`, `USBSkeletonConsumer`, + `MultiHMRCoreML`, `BodyFusion`. These migrate into the new app + unchanged (they are tested and reviewed). +3. **Rendering: a single RealityKit 3D scene** — the iPhone video is a + texture on a quad at the back of the scene; the body (skeleton + + mesh) is in front; an orbitable camera. +4. **Drop all legacy** — MediaPipe OSC listeners, the `SceneRenderer` + Metal viz modes, `DataFeedsOSCListener` + HUD, the 33-joint + `Skeleton3DRenderer`, Mac-webcam capture, viz-mode hotkeys, the + layered `BodyView`, `PoseOSCListener`. +5. Built as a proper `.app` via Xcode, which resolves the keyboard- + focus problem that affected the `swift run` executable. + +## Architecture + +A SwiftUI `@main App` with one window. An `AppDelegate` sets +`NSApplication` activation policy to `.regular`. The window hosts one +RealityKit `ARView` (used purely as a general 3D view on macOS — no +ARKit). The `ARView` holds a single scene containing: + +- a **video quad** — a flat plane entity at the back, its material + texture replaced from each decoded iPhone `CVPixelBuffer`; +- the **body** — 91 skeleton joint markers and the dense SMPL-X mesh, + positioned in front of the video quad; +- an **orbitable camera**. + +The USB pipeline (reused components) feeds the scene. The app is a +strict consumer: no network, the only input is the USB cable. + +## Components + +### Reused — the USB pipeline (migrated unchanged) + +| Unit | Responsibility | +|------|----------------| +| `AVLiveWire` (SwiftPM package, stays in `shared/`) | 19-byte frame format, `FrameHeader`/`FrameTag`, `SkeletonPayload`/`VideoPayload`, `StreamDemuxer` | +| `USBMuxProtocol` | usbmux 16-byte-header + plist codec | +| `USBClient` / `MuxTransport` / `UnixMuxTransport` | usbmux device discovery, connect, `AF_UNIX` socket | +| `VideoDecoder` | HEVC `VideoPayload` → `CVPixelBuffer` (`VTDecompressionSession`) | +| `USBSkeletonConsumer` | background USB read loop → `StreamDemuxer`; republishes `.skeleton` body frames + decoded `.video` pixel buffers; auto-reconnect | +| `MultiHMRCoreML` | bundled CoreML model → N SMPL-X persons | +| `BodyFusion` | associate ARKit skeleton ↔ Multi-HMR person, pelvis-depth correction | + +These move from `launcher/AV-Live-Body/Sources/AVLiveBody/` into the +new app's source tree. `AVLiveWire` stays in `shared/AVLiveWire`; the +new app declares it as a local package dependency. + +### New — rendering (clean, zero legacy) + +| Unit | Responsibility | +|------|----------------| +| `AVLiveBodyApp` | `@main` SwiftUI `App`; `AppDelegate` forces `.regular` activation; one window | +| `SceneView` | `NSViewRepresentable` wrapping the RealityKit `ARView` | +| `SceneController` | owns the scene, the orbital camera, assembles the entities; exposes `updateSkeleton`, `updateMesh`, `updateVideo` | +| `VideoQuad` | the back plane entity; updates its `TextureResource` from a `CVPixelBuffer` per frame | +| `SkeletonEntity` | 91 joint marker entities (native 91-joint, no MediaPipe 33-joint schema) | +| `MeshEntity` | the SMPL-X mesh entity (10475 vertices); mesh-building logic cleanly adapted from the old `MeshRenderer` | +| `StatusBar` | a small SwiftUI overlay showing connection state from `USBSkeletonConsumer.connected` | + +## Data flow + +``` +iPhone ──USB── UnixMuxTransport → USBClient → StreamDemuxer → USBSkeletonConsumer + ├─ .skeleton → SceneController.updateSkeleton → SkeletonEntity + └─ .video → VideoDecoder → CVPixelBuffer ─┬─ SceneController.updateVideo → VideoQuad + └─ MultiHMRCoreML → BodyFusion → SceneController.updateMesh → MeshEntity +``` + +Two rates: the skeleton streams at ~30 fps (smooth markers); video and +Multi-HMR run slower (~7 fps for the mesh). The video quad texture +refreshes at the video frame rate. + +## Error handling + +- **USB disconnect / no iPhone** — `USBSkeletonConsumer` retries every + second; `StatusBar` shows "waiting for iPhone". +- **CoreML model absent or failing** — the app runs skeleton-only (no + mesh); not a fatal error. +- **Video decode failure** — the frame is skipped. +- **Reconnect** — handled by the consumer's loop; entities holding + stale data are cleared after a timeout. + +## Testing + +- The reused USB components keep their existing unit tests + (`AVLiveWireTests`, `USBMuxProtocolTests`, `USBClientTests`, + `BodyFusionTests`, `USBSkeletonConsumerTests`) — carried into the new + app's test target. +- New rendering units (`VideoQuad`, `SkeletonEntity`, `MeshEntity`, + `SceneController`) depend on RealityKit/CoreML/VideoToolbox — + verified by build + on-device/manual run. Any extractable pure logic + (coordinate mapping, mesh index construction) gets unit tests. +- Build verification: a real Xcode project — + `xcodebuild -scheme AVLiveBody -destination 'platform=macOS' build`. + +## Scope + +**In scope** + +- New `avlivebody-mac/` Xcode app (xcodegen `project.yml`). +- Migrate the USB pipeline components into the new app. +- The RealityKit scene: `VideoQuad`, `SkeletonEntity`, `MeshEntity`, + `SceneController`, orbital camera. +- Connection-status UI. +- Archive `launcher/AV-Live-Body/`. + +**Out of scope** + +- All legacy AVLiveBody functionality (MediaPipe pose, Metal viz + modes, data-feeds HUD, Mac webcam, viz-mode hotkeys) — deliberately + dropped, not migrated. +- Changes to the iOS `ARBodyTracker` app or to `AVLiveWire`. + +## Migration notes + +- The USB component files currently live in + `launcher/AV-Live-Body/Sources/AVLiveBody/`. They are copied into the + new app's source tree; the old directory is then archived (moved + aside / removed from the active build), not deleted from git history. +- `AVLiveWire` is untouched in `shared/AVLiveWire`. +- The new app's `project.yml` declares the local `AVLiveWire` package + dependency and bundles the Multi-HMR `.mlpackage` as a resource + (per the earlier owner decision: bundle the validated FP32 model). + +## Risks + +- **Video-as-texture in RealityKit** — RealityKit has no direct + "stream of `CVPixelBuffer` → texture" path (`VideoMaterial` is + driven by an `AVPlayer`, not a decoded buffer stream). `VideoQuad` + must replace a `TextureResource` (or use `LowLevelTexture`) per + frame. This is the app's hardest technical point; the implementation + plan isolates it in `VideoQuad` so it can be iterated independently. +- **macOS RealityKit camera control** — `ARView` on macOS is a general + 3D view; an orbital camera must be set up explicitly (RealityKit + does not provide macOS orbit controls out of the box). +- **Multi-HMR throughput** — ~7 fps; the mesh layer is slow while the + skeleton stays real-time. Acceptable; mesh interpolation can be + added later if needed. diff --git a/docs/superpowers/specs/2026-05-18-iphone-usb-body-link-design.md b/docs/superpowers/specs/2026-05-18-iphone-usb-body-link-design.md new file mode 100644 index 0000000..9c31939 --- /dev/null +++ b/docs/superpowers/specs/2026-05-18-iphone-usb-body-link-design.md @@ -0,0 +1,201 @@ +# iPhone USB Body-Tracking Link — Design + +> **Status:** design approved (brainstorming), pending implementation plan. +> **Date:** 2026-05-18 + +## Goal + +Replace the network (OSC/UDP over WiFi) link between the iOS +`ARBodyTracker` app and the macOS `AVLiveBody` app with a **wired USB +link**, so the body-tracking pipeline runs autonomously on one +iPhone + one Mac with no WiFi, no router, no hotspot, no remote +worker. + +## Motivation + +AV-Live's body pipeline is currently distributed: the Mac camera +feeds Multi-HMR (on a remote host), and the iPhone ARKit data only +*corrects* it over OSC/UDP. This depends on the network. The owner +wants a self-contained, network-free system. + +## Decisions (brainstorming outcomes) + +1. **iPhone is the source.** ARKit body tracking + LiDAR + RGB video + all originate on the iPhone. The Mac no longer uses its own camera. +2. **iPhone streams video.** Multi-HMR is an image-to-SMPL-X model, so + the iPhone sends the RGB video (not just the skeleton); the Mac runs + Multi-HMR on that video. The ARKit skeleton + LiDAR correct scale + and depth. +3. **Transport is USB.** Bluetooth cannot carry video bandwidth; WiFi + is a network. The cable is the only network-free, high-bandwidth, + low-latency option. +4. **Single native macOS app.** `AVLiveBody` becomes one Swift app: + receives USB, runs Multi-HMR in CoreML, renders the mesh. No Python + in the iPhone-USB path. +5. **Multi-person.** Multi-HMR yields N meshes from the video; the + single ARKit skeleton corrects the *primary* body only; others are + Multi-HMR raw. Skeleton-to-mesh association logic is required. +6. **USB transport mechanism:** native Swift `usbmux` client (no + `peertalk` dependency). + +## Architecture + +Two apps, one cable. + +- **ARBodyTracker (iOS)** — extends the existing + `iphone-arbody/ARBodyTracker.swiftpm`. Captures the ARKit 91-joint + skeleton (LiDAR-anchored) and the `ARFrame` RGB image, HEVC-encodes + the video, frames skeleton + video into one stream, and serves it on + a local TCP port that the Mac reaches through `usbmuxd`. +- **AVLiveBody (macOS)** — extends the existing + `launcher/AV-Live-Body` Swift app. Connects to the iPhone over USB, + demuxes the stream, HEVC-decodes the video, runs CoreML Multi-HMR + (N meshes), fuses with the ARKit skeleton, renders the meshes, and + keeps feeding SuperCollider via localhost OSC. + +usbmuxd is Apple's USB device-multiplexing daemon (the channel Xcode +uses for a tethered device). The iOS app's TCP listener is never +exposed to any network; the Mac connects to it through the cable via +`/var/run/usbmuxd`. + +## Components + +### iOS — ARBodyTracker + +| Unit | Responsibility | Depends on | +|------|----------------|------------| +| `ARBodySession` | `ARBodyTrackingConfiguration` → 91-joint skeleton + `ARFrame.capturedImage` | ARKit (exists, extend) | +| `VideoEncoder` | hardware HEVC encode (VideoToolbox): pixel buffer → compressed access unit | VideoToolbox | +| `WireFormat` | binary framing `[tag, pid, timestamp, length, payload]`; pure, testable | — | +| `USBServer` | TCP `NWListener` on a fixed local port; usbmuxd exposes it to the tethered Mac | Network, WireFormat | +| `ContentView` | UI: AR preview, connection status, start/stop | SwiftUI (exists, extend) | + +The existing OSC sender in ARBodyTracker is removed. + +### macOS — AVLiveBody + +| Unit | Responsibility | Depends on | +|------|----------------|------------| +| `USBClient` | native Swift usbmux client: `/var/run/usbmuxd` socket, device list, connect-to-port, attach/detach events, byte stream | — (Unix socket, mockable) | +| `StreamDemuxer` | parse `WireFormat` frames → skeleton frames / video frames; resync on partial buffers | WireFormat | +| `VideoDecoder` | hardware HEVC decode → `CVPixelBuffer` | VideoToolbox | +| `MultiHMRCoreML` | run the CoreML Multi-HMR model on a frame → N SMPL-X meshes | CoreML `.mlpackage` | +| `BodyFusion` | associate the ARKit skeleton with the matching Multi-HMR person; LiDAR scale/depth correction on the primary; others pass through; pure, testable | — | +| `MeshRenderer` / `Skeleton3DRenderer` | RealityKit rendering of meshes/skeletons | RealityKit (exist) | +| `PoseOSCBridge` | emit pose to SuperCollider `:57121` on localhost — preserves AV-Live's audio half | Network (localhost only) | + +`ArkitOSCListener` (network) is retired; `USBClient` takes its role +over USB. + +## Data flow + +``` +iPhone ARKit ──┬─ skeleton 91 joints ─────────────┐ + └─ ARFrame RGB → VideoEncoder HEVC ─┤ + WireFormat ┤→ USBServer (local TCP port) + │ + ═══ USB cable / usbmuxd ═══ + │ +Mac USBClient → StreamDemuxer ─┬─ video → VideoDecoder → MultiHMRCoreML → N meshes ┐ + └─ skeleton ───────────────────────────────────────┤ + BodyFusion ┤ + ┌──────────────────────────────────────────────────────┘ + ├→ MeshRenderer (N meshes) + Skeleton3DRenderer + └→ PoseOSCBridge → SuperCollider :57121 (localhost) +``` + +`BodyFusion` associates the ARKit skeleton with the nearest Multi-HMR +person (by 2D projection / position) and corrects that person's scale +and depth (`pred_cam_t.z`) from the LiDAR-anchored joints. Other +bodies remain Multi-HMR raw. + +**Two rates.** The skeleton streams at ~30 fps (cheap, always fresh). +Video / Multi-HMR runs slower (CoreML throughput, ~2-5 fps on Apple +Silicon). Every frame carries a timestamp; fusion matches a mesh to +the nearest-in-time skeleton. The skeleton is the smooth real-time +layer; the dense mesh is a slower layer, bridged to 30 fps by the +existing mesh interpolation in `AVLiveBody` (commit `0293cde`), +driven by the 30 fps skeleton. + +## Wire format + +Each frame: a fixed header followed by a payload. + +| Field | Type | Notes | +|-------|------|-------| +| `tag` | `u8` | 1 = skeleton, 2 = video, 3 = meta | +| `pid` | `i16` | body id (skeleton/meta); `-1` for video | +| `timestamp` | `f64` | capture time, seconds | +| `length` | `u32` BE | payload byte count | +| `payload` | bytes | per-tag, below | + +- **skeleton** — 91 × `(x, y, z)` `f32` world-space + a 91-bit + validity mask. +- **video** — one HEVC access unit; a flag marks keyframes and + carries parameter sets (VPS/SPS/PPS) when present. +- **meta** — video dimensions, camera intrinsics, body count. + +Exact byte layout is finalized in the implementation plan. + +## Error handling + +- **USB attach/detach** — `USBClient` subscribes to usbmuxd device + events and auto-reconnects. Renderers GC stale persons (existing + `retainSec`). +- **Backpressure** — if Multi-HMR is slower than capture, latest frame + wins: intermediate video frames are dropped, never queued. The + skeleton stream stays fresh independently. +- **HEVC decode failure** — frame skipped. +- **CoreML model absent or failing** — fall back to skeleton-only + rendering (`Skeleton3DRenderer` draws the ARKit skeleton): degraded + but alive. +- **Frame sync** — timestamp-based nearest match in `BodyFusion`. + +## Testing + +| Unit | Test | +|------|------| +| `WireFormat` | pure unit: encode→decode roundtrip, all tags, truncated/corrupt frames | +| `USBClient` | unit: usbmux protocol against a mocked Unix socket (canned plist replies), device-list parse, connect handshake, attach/detach events | +| `StreamDemuxer` | roundtrip + resync on partial (non-frame-aligned) buffers | +| `VideoDecoder` | decode a known HEVC sample → expected dimensions | +| `BodyFusion` | pure logic: synthetic skeleton + synthetic Multi-HMR persons → assert association + scale/depth correction | +| `MultiHMRCoreML` | integration: known frame → mesh, sanity bounds | +| iOS (`VideoEncoder`, `ARBodySession`, `USBServer`) | framing unit-tested; ARKit/VideoToolbox need a device — manual/integration | +| End-to-end | iPhone tethered, both apps, N meshes render + latency budget + USB reconnect | + +## Scope + +**In scope** + +- Extend `iphone-arbody/ARBodyTracker.swiftpm`: `VideoEncoder`, + `WireFormat`, `USBServer`; `ARBodySession` exposes video frames; the + OSC sender is removed. +- Extend `launcher/AV-Live-Body`: `USBClient`, `StreamDemuxer`, + `VideoDecoder`, `MultiHMRCoreML` wiring, `BodyFusion`; + `ArkitOSCListener` retired. +- Keep `PoseOSCBridge` → SuperCollider on localhost. + +**Out of scope** + +- The Python `data_only_viz` pipeline — untouched; it remains the + Mac-camera mode. This project is the iPhone-USB path only. +- CoreML Multi-HMR model *conversion* — assumed already done + (`multihmr_coreml.py` + existing conversion plans). This project + *consumes* the `.mlpackage`. +- LiDAR scene mesh / ICP fusion (separate plan). +- iOS app signing and deployment — owner action. + +## Risks & dependencies + +- **CoreML Multi-HMR readiness** — the dense-mesh half depends on a + working, fast-enough `.mlpackage`. If not ready, that half is + blocked, but the skeleton-only fallback keeps the project useful — + not all-or-nothing. +- **Multi-HMR throughput** — ~2-5 fps measured on Apple Silicon. The + dense mesh updates slowly; the 30 fps skeleton + existing mesh + interpolation cover the gap. +- **Device pairing** — the iPhone must be trusted/paired with the Mac + for usbmuxd to expose it. +- **iOS deployment** — building/signing/installing the iOS app is a + manual owner step. diff --git a/iphone-arbody/ARBodyTracker.swiftpm/.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata b/iphone-arbody/ARBodyTracker.swiftpm/.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..919434a --- /dev/null +++ b/iphone-arbody/ARBodyTracker.swiftpm/.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Package.swift b/iphone-arbody/ARBodyTracker.swiftpm/Package.swift new file mode 100644 index 0000000..ed1a671 --- /dev/null +++ b/iphone-arbody/ARBodyTracker.swiftpm/Package.swift @@ -0,0 +1,23 @@ +// swift-tools-version:5.10 +import PackageDescription + +let package = Package( + name: "ARBodyTracker", + defaultLocalization: "en", + platforms: [.iOS(.v17)], + products: [ + .executable(name: "ARBodyTracker", targets: ["ARBodyTracker"]), + ], + dependencies: [ + .package(path: "../../shared/AVLiveWire"), + ], + targets: [ + .executableTarget( + name: "ARBodyTracker", + dependencies: [ + .product(name: "AVLiveWire", package: "AVLiveWire"), + ], + path: "Sources/ARBodyTracker" + ), + ] +) diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift new file mode 100644 index 0000000..baca4dd --- /dev/null +++ b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodySession.swift @@ -0,0 +1,217 @@ +import ARKit +import AVLiveWire +import Combine +import Foundation +import RealityKit +import SwiftUI + +/// Drives the ARKit body-tracking session and streams the 91-joint +/// skeleton plus HEVC-encoded camera video to the tethered Mac over +/// USB (AVLiveWire frames via usbmuxd). No network involved. +/// +/// Lightweight 2D snapshot of the tracked skeleton, ready for SwiftUI +/// Canvas. Joint indices follow `ARSkeletonDefinition.defaultBody3D`. +struct SkeletonSnapshot: Equatable { + /// Projected joint positions in viewport coordinates, or nil if the + /// joint falls outside the view / is not finite. + let points: [CGPoint?] + /// Per-joint tracking flag (`ARSkeleton.isJointTracked`). + let tracked: [Bool] +} + +@MainActor +final class ARBodySession: NSObject, ObservableObject, ARSessionDelegate { + @Published var running: Bool = false + @Published var status: String = "idle" + @Published var framesSent: Int = 0 + @Published var jointsPerSec: Double = 0 + @Published var bodyCount: Int = 0 + @Published var skeleton2D: SkeletonSnapshot? + @Published var usbState: USBServer.State = .idle + /// Set by the SwiftUI view via GeometryReader so the projection + /// matches the on-screen ARView size. + var viewportSize: CGSize = .zero + private var sendEnvMesh: Bool = false + private let session = ARSession() + private let usb = USBServer() + private let videoEncoder = VideoEncoder() + private var videoStarted = false + private var lastFrameTime: TimeInterval = 0 + private var jointsInSecond: Int = 0 + private var lastSecond: TimeInterval = 0 + private let bodyParents: [Int] = + ARSkeletonDefinition.defaultBody3D.parentIndices + + let arView = ARView(frame: .zero) + + override init() { + super.init() + arView.session = session + arView.session.delegate = self + arView.environment.background = .color(.black) + arView.debugOptions = [] + usb.onState = { [weak self] s in + Task { @MainActor in self?.usbState = s } + } + videoEncoder.onPayload = { [weak self] payload in + Task { @MainActor in + guard let self, self.usbState == .connected else { + return + } + self.usb.send(tag: .video, pid: -1, + timestamp: self.lastFrameTime, + payload: payload.encoded()) + } + } + } + + func configure(sendEnvMesh: Bool) { + self.sendEnvMesh = sendEnvMesh + } + + func start() { + guard ARBodyTrackingConfiguration.isSupported else { + status = "ARBodyTracking unsupported (need A12+, iPhone XR/XS+)" + return + } + let cfg = ARBodyTrackingConfiguration() + var feats: [String] = [] + // No extra frame semantics: `.sceneDepth` is reserved to + // ARWorldTracking, and `.personSegmentationWithDepth` is + // rejected per-frame by ABPKPersonIDTracker in this config + // (spams the console without producing usable depth). + // NOTE: ARBodyTrackingConfiguration does not expose + // sceneReconstruction (that's ARWorldTrackingConfiguration + // territory). Env mesh capture requires a separate ARSession + // with body tracking off — out of scope for this scaffold. + if sendEnvMesh { + feats.append("env-mesh: requires separate session (TODO)") + } + cfg.automaticImageScaleEstimationEnabled = true + usb.start() + session.run(cfg, options: [.resetTracking, .removeExistingAnchors]) + status = feats.isEmpty + ? "running (RGB only)" + : "running (\(feats.joined(separator: ", ")))" + running = true + } + + func stop() { + session.pause() + usb.stop() + videoEncoder.stop() + videoStarted = false + running = false + status = "stopped" + } + + // MARK: - ARSessionDelegate + + nonisolated func session(_ s: ARSession, didUpdate frame: ARFrame) { + let t = frame.timestamp + Task { @MainActor in + // Throttle to 30 fps max. + if t - self.lastFrameTime < 1.0 / 30.0 { return } + self.lastFrameTime = t + + // Encode the camera frame to HEVC and stream it over USB. + let img = frame.capturedImage + let w = Int32(CVPixelBufferGetWidth(img)) + let h = Int32(CVPixelBufferGetHeight(img)) + if !self.videoStarted, w > 0, h > 0 { + self.videoEncoder.start(width: w, height: h) + self.videoStarted = true + } + if self.videoStarted { + self.videoEncoder.encode(img, pts: t) + } + + var count: Int = 0 + var firstBody: ARBodyAnchor? + for anchor in frame.anchors { + guard let body = anchor as? ARBodyAnchor else { continue } + self.publishUSB(pid: count, timestamp: t, body: body) + if count == 0 { firstBody = body } + count += 1 + } + self.framesSent &+= 1 + self.bodyCount = count + self.updateSkeleton2D(body: firstBody, camera: frame.camera) + + let now = Date().timeIntervalSinceReferenceDate + self.jointsInSecond &+= count * 91 + if now - self.lastSecond >= 1.0 { + self.jointsPerSec = Double(self.jointsInSecond) + / max(0.001, now - self.lastSecond) + self.jointsInSecond = 0 + self.lastSecond = now + } + } + } + + private func currentInterfaceOrientation() -> UIInterfaceOrientation { + for scene in UIApplication.shared.connectedScenes { + if let ws = scene as? UIWindowScene { + return ws.interfaceOrientation + } + } + return .portrait + } + + private func updateSkeleton2D(body: ARBodyAnchor?, camera: ARCamera) { + guard let body, viewportSize.width > 1, viewportSize.height > 1 + else { + if skeleton2D != nil { skeleton2D = nil } + return + } + let xforms = body.skeleton.jointModelTransforms + let root = body.transform + let orient = currentInterfaceOrientation() + var pts: [CGPoint?] = Array(repeating: nil, count: xforms.count) + var tracked: [Bool] = Array(repeating: false, count: xforms.count) + for (i, m) in xforms.enumerated() { + let w = root * m + let p3 = simd_make_float3(w.columns.3.x, + w.columns.3.y, + w.columns.3.z) + let p2 = camera.projectPoint(p3, + orientation: orient, + viewportSize: viewportSize) + if p2.x.isFinite && p2.y.isFinite { pts[i] = p2 } + tracked[i] = body.skeleton.isJointTracked(i) + } + skeleton2D = SkeletonSnapshot(points: pts, tracked: tracked) + } + + /// Exposed for SwiftUI overlays that need to wire bone parent + /// indices without re-reading the ARKit skeleton definition. + var bodyParentIndices: [Int] { bodyParents } + + private func publishUSB(pid: Int, timestamp: TimeInterval, + body: ARBodyAnchor) { + guard usbState == .connected else { return } + let skeleton = body.skeleton + let transforms = skeleton.jointModelTransforms + let root = body.transform + var payload = SkeletonPayload() + let n = min(SkeletonPayload.jointCount, transforms.count) + for i in 0.. ARView { session.arView } + func updateUIView(_ uiView: ARView, context: Context) {} +} diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodyTrackerApp.swift b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodyTrackerApp.swift new file mode 100644 index 0000000..5b733a7 --- /dev/null +++ b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ARBodyTrackerApp.swift @@ -0,0 +1,10 @@ +import SwiftUI + +@main +struct ARBodyTrackerApp: App { + var body: some Scene { + WindowGroup { + ContentView() + } + } +} diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift new file mode 100644 index 0000000..63234c8 --- /dev/null +++ b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/ContentView.swift @@ -0,0 +1,237 @@ +import SwiftUI +import ARKit +import RealityKit +import UIKit + +struct ContentView: View { + @StateObject private var session = ARBodySession() + @State private var sendEnvMesh: Bool = false + + /// Replace the live ARView with a gradient placeholder. Camera is + /// unavailable inside Xcode previews; enable this flag there so the + /// panel + skeleton overlay can still be laid out. + var useMockBackground: Bool = false + /// Overlay a synthetic ARKit T-pose so the skeleton renderer can be + /// tuned without running on a device. + var useMockSkeleton: Bool = false + + var body: some View { + GeometryReader { geo in + ZStack(alignment: .topLeading) { + cameraBackground + .ignoresSafeArea() + SkeletonOverlay( + snapshot: useMockSkeleton + ? SkeletonSnapshot.mockTPose(in: geo.size) + : session.skeleton2D, + parents: useMockSkeleton + ? SkeletonSnapshot.mockParents + : session.bodyParentIndices) + .ignoresSafeArea() + .allowsHitTesting(false) + controlPanel + } + .onAppear { + session.viewportSize = geo.size + // Keep the screen awake during streaming sessions; iOS + // would otherwise lock and tear down the USBServer TCP + // listener within seconds of inactivity. + UIApplication.shared.isIdleTimerDisabled = true + } + .onDisappear { + UIApplication.shared.isIdleTimerDisabled = false + } + .onChange(of: geo.size) { _, newSize in + session.viewportSize = newSize + } + } + } + + private var usbDotColor: Color { + switch session.usbState { + case .idle: return .gray + case .listening: return .yellow + case .connected: return .green + } + } + + private var usbStateLabel: String { + switch session.usbState { + case .idle: return "idle" + case .listening: return "listening :\(USBServer.port)" + case .connected: return "connected" + } + } + + @ViewBuilder + private var cameraBackground: some View { + if useMockBackground { + ZStack { + LinearGradient( + colors: [ + Color(red: 0.18, green: 0.20, blue: 0.24), + Color(red: 0.05, green: 0.05, blue: 0.08), + ], + startPoint: .top, + endPoint: .bottom) + Text("camera preview\n(unavailable in Xcode canvas)") + .font(.caption) + .multilineTextAlignment(.center) + .foregroundStyle(.white.opacity(0.35)) + } + } else { + ARViewContainer(session: session) + } + } + + private var controlPanel: some View { + VStack(alignment: .leading, spacing: 8) { + Text("AR Body → AV-Live") + .font(.headline) + .foregroundColor(.white) + Toggle(isOn: $sendEnvMesh) { + Text("Env mesh (LiDAR)").foregroundColor(.white) + } + HStack { + Button(session.running ? "Stop" : "Start") { + if session.running { + session.stop() + } else { + session.configure(sendEnvMesh: sendEnvMesh) + session.start() + } + } + .buttonStyle(.borderedProminent) + Spacer() + Text(session.status) + .font(.caption) + .foregroundColor(.white) + .padding(6) + .background(.black.opacity(0.5)) + .cornerRadius(6) + } + HStack(spacing: 6) { + Circle() + .fill(usbDotColor) + .frame(width: 8, height: 8) + Text("USB \(usbStateLabel)") + .font(.caption2) + .foregroundColor(.white.opacity(0.8)) + Spacer(minLength: 8) + Text("bodies: \(session.bodyCount) frames: \(session.framesSent) j/s: \(Int(session.jointsPerSec))") + .font(.caption2) + .foregroundColor(.white) + } + } + .padding(12) + .background(.black.opacity(0.5)) + .cornerRadius(10) + .padding() + } +} + +extension SkeletonSnapshot { + /// Parent indices for the 16-joint preview stick figure. Each entry + /// is the parent joint index, or -1 for the root (head). + static let mockParents: [Int] = [ + -1, // 0 head + 0, // 1 neck + 1, // 2 lShoulder + 1, // 3 rShoulder + 2, // 4 lElbow + 3, // 5 rElbow + 4, // 6 lWrist + 5, // 7 rWrist + 1, // 8 spine + 8, // 9 pelvis + 9, // 10 lHip + 9, // 11 rHip + 10, // 12 lKnee + 11, // 13 rKnee + 12, // 14 lAnkle + 13, // 15 rAnkle + ] + + /// Synthetic 16-joint stick figure used by Xcode previews. ARKit + /// is not available in the preview canvas, so we cannot rely on + /// `ARSkeletonDefinition.neutralBodySkeleton3D` (returns nil). + static func mockTPose(in size: CGSize) -> SkeletonSnapshot { + // Normalized layout: origin at body center, +y down, ±1 spans + // roughly the full body height. + let layout: [CGPoint] = [ + CGPoint(x: 0.00, y: -0.45), + CGPoint(x: 0.00, y: -0.32), + CGPoint(x: -0.18, y: -0.30), + CGPoint(x: 0.18, y: -0.30), + CGPoint(x: -0.30, y: -0.12), + CGPoint(x: 0.30, y: -0.12), + CGPoint(x: -0.36, y: 0.08), + CGPoint(x: 0.36, y: 0.08), + CGPoint(x: 0.00, y: -0.10), + CGPoint(x: 0.00, y: 0.06), + CGPoint(x: -0.10, y: 0.09), + CGPoint(x: 0.10, y: 0.09), + CGPoint(x: -0.12, y: 0.30), + CGPoint(x: 0.12, y: 0.30), + CGPoint(x: -0.13, y: 0.46), + CGPoint(x: 0.13, y: 0.46), + ] + let scale = min(size.width * 0.9, size.height * 0.8) + let cx = size.width * 0.5 + let cy = size.height * 0.5 + let pts: [CGPoint?] = layout.map { + CGPoint(x: cx + $0.x * scale, + y: cy + $0.y * scale) + } + return SkeletonSnapshot( + points: pts, + tracked: Array(repeating: true, count: pts.count)) + } +} + +/// Draws ARKit body joints + bones over the camera view. Bones are +/// derived from `ARSkeletonDefinition.defaultBody3D` parent indices. +struct SkeletonOverlay: View { + let snapshot: SkeletonSnapshot? + let parents: [Int] + + var body: some View { + Canvas { ctx, _ in + guard let snap = snapshot else { return } + for (i, parent) in parents.enumerated() where parent >= 0 { + guard i < snap.points.count, parent < snap.points.count, + let a = snap.points[i], let b = snap.points[parent] + else { continue } + var path = Path() + path.move(to: a) + path.addLine(to: b) + let solid = snap.tracked[i] && snap.tracked[parent] + ctx.stroke( + path, + with: .color(solid ? .green : .yellow.opacity(0.5)), + lineWidth: solid ? 2 : 1.2) + } + for (i, pt) in snap.points.enumerated() { + guard let pt else { continue } + let r: CGFloat = snap.tracked[i] ? 4 : 2.5 + let rect = CGRect(x: pt.x - r, y: pt.y - r, + width: r * 2, height: r * 2) + ctx.fill(Path(ellipseIn: rect), + with: .color(snap.tracked[i] + ? .cyan : .yellow.opacity(0.8))) + } + } + } +} + +#Preview("iPhone 15 Pro — portrait") { + ContentView(useMockBackground: true, useMockSkeleton: true) +} + +#Preview("iPhone 15 Pro — landscape", traits: .landscapeLeft) { + ContentView(useMockBackground: true, useMockSkeleton: true) +} + +#Preview("Empty camera (no body)") { + ContentView(useMockBackground: true, useMockSkeleton: false) +} diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/Info.plist b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/Info.plist new file mode 100644 index 0000000..82f68df --- /dev/null +++ b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/Info.plist @@ -0,0 +1,39 @@ + + + + + CFBundleDevelopmentRegionen + CFBundleDisplayNameARBody Tracker + CFBundleExecutable$(EXECUTABLE_NAME) + CFBundleIdentifier$(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion6.0 + CFBundleNameARBodyTracker + CFBundlePackageTypeAPPL + CFBundleShortVersionString0.1.0 + CFBundleVersion1 + LSRequiresIPhoneOS + NSCameraUsageDescription + Required for ARKit body tracking and LiDAR depth capture. + NSLocalNetworkUsageDescription + Streams ARKit body tracking and camera video to a tethered Mac over USB. + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UIRequiredDeviceCapabilities + + arm64 + arkit + + UIDeviceFamily + 1 + UIRequiresFullScreen + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UILaunchScreen + + diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/USBServer.swift b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/USBServer.swift new file mode 100644 index 0000000..8ef902f --- /dev/null +++ b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/USBServer.swift @@ -0,0 +1,62 @@ +import Foundation +import Network +import AVLiveWire + +/// TCP listener on a fixed local port. usbmuxd tunnels it to the +/// tethered Mac — the port is never advertised on any network. +final class USBServer { + static let port: UInt16 = 7000 + + enum State { case idle, listening, connected } + var onState: ((State) -> Void)? + + private var listener: NWListener? + private var connection: NWConnection? + private let queue = DispatchQueue(label: "cc.avlive.usbserver") + + func start() { + let params = NWParameters.tcp + params.allowLocalEndpointReuse = true + guard let l = try? NWListener(using: params, + on: NWEndpoint.Port(rawValue: Self.port)!) else { + onState?(.idle) + return + } + listener = l + l.newConnectionHandler = { [weak self] conn in + self?.adopt(conn) + } + l.start(queue: queue) + onState?(.listening) + } + + private func adopt(_ conn: NWConnection) { + connection?.cancel() + connection = conn + conn.stateUpdateHandler = { [weak self] st in + switch st { + case .ready: self?.onState?(.connected) + case .failed, .cancelled: self?.onState?(.listening) + default: break + } + } + conn.start(queue: queue) + } + + /// Send one framed message. Drops silently if no peer. + func send(tag: FrameTag, pid: Int16, timestamp: Double, + payload: Data) { + guard let conn = connection else { return } + guard payload.count <= Int(StreamDemuxer.maxPayloadLength) + else { return } + let header = FrameHeader(tag: tag, pid: pid, + timestamp: timestamp, length: UInt32(payload.count)) + conn.send(content: header.encoded() + payload, + completion: .contentProcessed { _ in }) + } + + func stop() { + connection?.cancel(); listener?.cancel() + onState?(.idle) + } +} diff --git a/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/VideoEncoder.swift b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/VideoEncoder.swift new file mode 100644 index 0000000..20da274 --- /dev/null +++ b/iphone-arbody/ARBodyTracker.swiftpm/Sources/ARBodyTracker/VideoEncoder.swift @@ -0,0 +1,137 @@ +import AVLiveWire +import CoreMedia +import CoreVideo +import Foundation +import VideoToolbox + +/// Hardware HEVC encoder. Feed `CVPixelBuffer`s from ARKit frames in; +/// receive one `VideoPayload` per encoded access unit via `onPayload`. +/// Keyframe payloads carry the VPS/SPS/PPS parameter sets prepended, +/// each as a 4-byte big-endian length prefix followed by the NAL +/// bytes, so the Mac decoder can build its format description without +/// a side channel. +final class VideoEncoder { + var onPayload: ((VideoPayload) -> Void)? + + private var session: VTCompressionSession? + private let lock = NSLock() + + /// Create the compression session for a given frame size. + func start(width: Int32, height: Int32) { + stop() + var s: VTCompressionSession? + let status = VTCompressionSessionCreate( + allocator: kCFAllocatorDefault, + width: width, height: height, + codecType: kCMVideoCodecType_HEVC, + encoderSpecification: nil, + imageBufferAttributes: nil, + compressedDataAllocator: nil, + outputCallback: nil, + refcon: nil, + compressionSessionOut: &s) + guard status == noErr, let s else { + NSLog("VideoEncoder: VTCompressionSessionCreate failed %d", + status) + return + } + VTSessionSetProperty(s, key: kVTCompressionPropertyKey_RealTime, + value: kCFBooleanTrue) + VTSessionSetProperty(s, + key: kVTCompressionPropertyKey_AllowFrameReordering, + value: kCFBooleanFalse) + VTSessionSetProperty(s, + key: kVTCompressionPropertyKey_MaxKeyFrameInterval, + value: 30 as CFNumber) + VTCompressionSessionPrepareToEncodeFrames(s) + lock.lock(); session = s; lock.unlock() + } + + /// Encode one frame. `pts` is the capture timestamp in seconds. + func encode(_ pixelBuffer: CVPixelBuffer, pts: Double) { + lock.lock(); let s = session; lock.unlock() + guard let s else { return } + let time = CMTime(seconds: pts, preferredTimescale: 1_000_000) + VTCompressionSessionEncodeFrame( + s, imageBuffer: pixelBuffer, presentationTimeStamp: time, + duration: .invalid, frameProperties: nil, + infoFlagsOut: nil) { [weak self] status, _, sample in + guard status == noErr, let sample else { return } + self?.handle(sample) + } + } + + func stop() { + lock.lock(); let s = session; session = nil; lock.unlock() + if let s { + VTCompressionSessionInvalidate(s) + } + } + + deinit { stop() } + + // MARK: - Sample -> VideoPayload + + private func handle(_ sample: CMSampleBuffer) { + let isKeyframe = !Self.notSync(sample) + var out = Data() + if isKeyframe, + let fmt = CMSampleBufferGetFormatDescription(sample) { + out.append(Self.parameterSets(fmt)) + } + if let block = CMSampleBufferGetDataBuffer(sample) { + var lengthOut = 0 + var ptr: UnsafeMutablePointer? + if CMBlockBufferGetDataPointer( + block, atOffset: 0, lengthAtOffsetOut: nil, + totalLengthOut: &lengthOut, + dataPointerOut: &ptr) == noErr, let ptr { + out.append(UnsafeBufferPointer( + start: UnsafeRawPointer(ptr) + .assumingMemoryBound(to: UInt8.self), + count: lengthOut)) + } + } + guard !out.isEmpty else { return } + onPayload?(VideoPayload(isKeyframe: isKeyframe, data: out)) + } + + /// True if the sample is NOT a sync (key) frame. + private static func notSync(_ sample: CMSampleBuffer) -> Bool { + guard let arr = CMSampleBufferGetSampleAttachmentsArray( + sample, createIfNecessary: false), + CFArrayGetCount(arr) > 0 else { return false } + let dict = unsafeBitCast(CFArrayGetValueAtIndex(arr, 0), + to: CFDictionary.self) + let key = Unmanaged.passUnretained( + kCMSampleAttachmentKey_NotSync).toOpaque() + return CFDictionaryContainsKey(dict, key) + } + + /// Concatenate the HEVC VPS/SPS/PPS parameter sets, each as a + /// 4-byte big-endian length prefix followed by the NAL bytes. + private static func parameterSets( + _ fmt: CMFormatDescription) -> Data { + var count = 0 + CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + fmt, parameterSetIndex: 0, parameterSetPointerOut: nil, + parameterSetSizeOut: nil, parameterSetCountOut: &count, + nalUnitHeaderLengthOut: nil) + var data = Data() + for i in 0..? + var size = 0 + guard CMVideoFormatDescriptionGetHEVCParameterSetAtIndex( + fmt, parameterSetIndex: i, + parameterSetPointerOut: &ptr, + parameterSetSizeOut: &size, + parameterSetCountOut: nil, + nalUnitHeaderLengthOut: nil) == noErr, + let ptr else { continue } + var be = UInt32(size).bigEndian + withUnsafeBytes(of: &be) { data.append(contentsOf: $0) } + data.append(UnsafeBufferPointer(start: ptr, count: size)) + } + return data + } +} diff --git a/iphone-arbody/CLAUDE.md b/iphone-arbody/CLAUDE.md new file mode 100644 index 0000000..93e5390 --- /dev/null +++ b/iphone-arbody/CLAUDE.md @@ -0,0 +1,94 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +iPhone-only app that streams ARKit `ARBodyAnchor` joints (91 per body) to +the AV-Live stack on a tethered Mac. Part of the parent `AV-Live/` +monorepo — read `../CLAUDE.md` for global conventions (commit style, +French/English split, no emoji, etc.). + +## Two build forms, one source tree + +Both flavours compile the same files under +`ARBodyTracker.swiftpm/Sources/ARBodyTracker/`: + +| Form | When to use | How | +|------|-------------|-----| +| `ARBodyTracker.swiftpm/` (Swift Package) | Quick local iteration; previews | open `Package.swift` in Xcode | +| `ARBodyTracker.xcodeproj` (generated) | Device deploy with reproducible signing | `xcodegen generate` from `project.yml` | + +The `.xcodeproj` is **gitignored** — regenerate after every change to +`project.yml` or after adding sources. `Config/Local.xcconfig` (also +gitignored) carries `DEVELOPMENT_TEAM`; copy from +`Config/Local.xcconfig.example` on a fresh clone. + +## Common commands + +```bash +brew install xcodegen # one-time +cp Config/Local.xcconfig.example Config/Local.xcconfig # set DEVELOPMENT_TEAM +xcodegen generate # writes ARBodyTracker.xcodeproj +open ARBodyTracker.xcodeproj # then ⌘R on a connected iPhone + +# AVLiveWire — shared wire-format package (consumed by both the iOS +# app and launcher/AV-Live-Body on macOS). Tests live with it. +cd ../shared/AVLiveWire && swift test +cd ../shared/AVLiveWire && swift test --filter LoopbackTests +``` + +`xcodegen generate` is the one command to remember: any time `project.yml` +or the dep on `../shared/AVLiveWire` changes, the generated project must +be rebuilt before opening Xcode. + +## Architecture + +``` +ARBodyTrackerApp ── ContentView ── ARViewContainer (ARView) + │ │ + │ └─ ARBodySession (ARSessionDelegate) + │ ├─ OSC fanout (UDP) ─► host:57128 (Python data_only_viz) + │ │ host:57129 (Swift AV-Live-Body) + │ ├─ USBServer (TCP :7000, AVLiveWire frames) + │ └─ @Published skeleton2D ──┐ + └─ SkeletonOverlay (SwiftUI Canvas) ◄─────────┘ +``` + +- **`ARBodySession`** owns the `ARSession` configured with + `ARBodyTrackingConfiguration` (RGB-only; LiDAR `sceneDepth` is not + available on this config, do not re-add it — it crashes the session). + Throttles to 30 fps. Projects joints to 2D via + `ARCamera.projectPoint(_:orientation:viewportSize:)`; the viewport + size is fed back from the SwiftUI `GeometryReader`. +- **Transport** is dual: legacy OSC/UDP (still default), and a new + USB/TCP path using `AVLiveWire` framed messages. The Mac side bridges + via `usbmuxd`; the iOS side just listens on a fixed local port. No + WiFi involved on the USB path. +- **`AVLiveWire`** (in `../shared/AVLiveWire`) defines a fixed 19-byte + big-endian header (`AVL1` magic + tag + pid + timestamp + length) and + an incremental `StreamDemuxer`. Both the iOS `USBServer` and the + macOS consumer link against it so the wire format lives in exactly + one place. +- **`SkeletonOverlay`** draws joints and bones from a published + `SkeletonSnapshot`; bone parent indices come from + `ARSkeletonDefinition.defaultBody3D.parentIndices` at runtime. For + Xcode previews the snapshot/parents are replaced by a 16-joint + stick-figure mock (ARKit's `neutralBodySkeleton3D` returns nil off + device). + +## Gotchas + +- **Frame semantics:** `ARBodyTrackingConfiguration` rejects + `.sceneDepth` (world-tracking only) and spams `ABPKPersonIDTracker: + Portrait image is not supported` per-frame if + `.personSegmentationWithDepth` is added. Keep `feats` empty unless a + newly-needed semantic is verified against + `ARBodyTrackingConfiguration.supportsFrameSemantics(_:)`. +- **Signing without an Apple ID logged into Xcode:** with + `CODE_SIGN_STYLE = Automatic`, Xcode needs the account in + Settings → Accounts to download a profile. If only the keychain + cert is present, the build fails with "No Account for Team …". + Reconnect the account, or fall back to a manually installed profile + via `CODE_SIGN_STYLE = Manual` + `PROVISIONING_PROFILE_SPECIFIER`. +- **`UIDeviceFamily` warning:** `Info.plist` and `TARGETED_DEVICE_FAMILY` + duplicate the device family. The build setting wins; the Info.plist + key only generates a warning and can be removed if desired. diff --git a/iphone-arbody/Config/Local.xcconfig b/iphone-arbody/Config/Local.xcconfig new file mode 100644 index 0000000..e3988a7 --- /dev/null +++ b/iphone-arbody/Config/Local.xcconfig @@ -0,0 +1 @@ +DEVELOPMENT_TEAM = K9KK43329X diff --git a/iphone-arbody/Config/Local.xcconfig.example b/iphone-arbody/Config/Local.xcconfig.example new file mode 100644 index 0000000..bb94c32 --- /dev/null +++ b/iphone-arbody/Config/Local.xcconfig.example @@ -0,0 +1,8 @@ +// Copy this file to Config/Local.xcconfig and fill in your Apple Developer +// Team ID. The Team ID is the 10-character alphanumeric string visible in +// Xcode → Settings → Accounts → (your Apple ID) → Manage Certificates, +// or via: security find-identity -v -p codesigning +// +// Config/Local.xcconfig is gitignored and must never be committed. + +DEVELOPMENT_TEAM = YOUR_TEAM_ID diff --git a/iphone-arbody/Config/Shared.xcconfig b/iphone-arbody/Config/Shared.xcconfig new file mode 100644 index 0000000..3f5e797 --- /dev/null +++ b/iphone-arbody/Config/Shared.xcconfig @@ -0,0 +1,12 @@ +// Shared build settings (committed). Local overrides such as the +// Apple Developer Team ID live in Config/Local.xcconfig (gitignored). +// The `#include?` directive silently skips the local file if absent, +// which is what we want on a fresh clone. + +#include? "Local.xcconfig" + +IPHONEOS_DEPLOYMENT_TARGET = 17.0 +SWIFT_VERSION = 5.10 +TARGETED_DEVICE_FAMILY = 1 +ENABLE_PREVIEWS = YES +CODE_SIGN_STYLE = Automatic diff --git a/iphone-arbody/README.md b/iphone-arbody/README.md new file mode 100644 index 0000000..42fd9e0 --- /dev/null +++ b/iphone-arbody/README.md @@ -0,0 +1,46 @@ +# ARBodyTracker — iPhone OSC emitter for AV-Live + +Streams ARKit ARBodyAnchor joints (91 per body) over OSC/UDP to GrosMac: + +- `:57128` → Python `IphoneOSCListener` in `data_only_viz/` (drives `ArkitFuse` + cam-z lock) +- `:57129` → Swift `ArkitOSCListener` in `launcher/AV-Live-Body/` (diagnostic overlay) + +Optional LiDAR depth (`ARFrameSemantics.sceneDepth`) is enabled when the +device supports it (iPhone 12 Pro and later). + +## Build & run + +The project ships in two forms: + +| Form | Path | Use when | +|------|------|----------| +| Swift Package app | `ARBodyTracker.swiftpm/` | Quick local iteration. Signing has to be set manually in Xcode each clone. | +| Xcode project (xcodegen) | `project.yml` + `Config/*.xcconfig` | Reproducible signing across machines. Generate with `xcodegen generate`. | + +### First-time setup (Xcode project flavour) + +```bash +brew install xcodegen # one-time +cp Config/Local.xcconfig.example Config/Local.xcconfig +# Edit Config/Local.xcconfig and set DEVELOPMENT_TEAM to your Apple Developer Team ID. +xcodegen generate # writes ARBodyTracker.xcodeproj +open ARBodyTracker.xcodeproj +``` + +The generated `ARBodyTracker.xcodeproj/` is gitignored — regenerate any +time `project.yml` or sources change. + +### Updating + +Add Swift sources under `ARBodyTracker.swiftpm/Sources/ARBodyTracker/`, +then `xcodegen generate` to refresh the project. Both the swiftpm form +and the xcodeproj reference the same source tree. + +## Signing + +`Config/Local.xcconfig` (gitignored) holds `DEVELOPMENT_TEAM`. +`Config/Shared.xcconfig` (committed) pulls it via `#include?`, falling +back to no team if the local file is missing — the build will then +fail with the familiar "executable is not codesigned" error, which is +the correct signal that the developer needs to provision their own +team ID before deploying to a device. diff --git a/iphone-arbody/project.yml b/iphone-arbody/project.yml new file mode 100644 index 0000000..62aec4a --- /dev/null +++ b/iphone-arbody/project.yml @@ -0,0 +1,41 @@ +name: ARBodyTracker +options: + bundleIdPrefix: cc.saillant + deploymentTarget: + iOS: "17.0" + createIntermediateGroups: true + generateEmptyDirectories: true + +configFiles: + Debug: Config/Shared.xcconfig + Release: Config/Shared.xcconfig + +packages: + AVLiveWire: + path: ../shared/AVLiveWire + +targets: + ARBodyTracker: + type: application + platform: iOS + deploymentTarget: "17.0" + sources: + - path: ARBodyTracker.swiftpm/Sources/ARBodyTracker + excludes: + - Info.plist + dependencies: + - package: AVLiveWire + product: AVLiveWire + configFiles: + Debug: Config/Shared.xcconfig + Release: Config/Shared.xcconfig + settings: + base: + PRODUCT_NAME: ARBodyTracker + PRODUCT_BUNDLE_IDENTIFIER: cc.saillant.ARBodyTracker + INFOPLIST_FILE: ARBodyTracker.swiftpm/Sources/ARBodyTracker/Info.plist + GENERATE_INFOPLIST_FILE: NO + CODE_SIGN_STYLE: Automatic + SWIFT_VERSION: "5.10" + TARGETED_DEVICE_FAMILY: "1" + ENABLE_PREVIEWS: YES diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift deleted file mode 100644 index 0a6c2f3..0000000 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift +++ /dev/null @@ -1,222 +0,0 @@ -import Combine -import Foundation -import RealityKit -import SwiftUI -import simd - -/// RealityKit renderer for MediaPipe Pose 3D world landmarks (33 joints, -/// metric coords relative to the hip-center). Consumes the `body3d` -/// publisher of `PoseOSCListener` and maintains one entity tree per -/// detected person. -/// -/// Coordinate mapping (MediaPipe -> RealityKit): -/// MediaPipe : x = right, y = down, z = forward (away from cam). -/// RealityKit: x = right, y = up, z = backward (toward cam). -/// => convert with (x, -y, -z). -@MainActor -final class Skeleton3DRenderer: ObservableObject { - /// 32 bones connecting MediaPipe Pose 33 landmarks. Indices are - /// the canonical MediaPipe Pose landmark indices. Source: official - /// `mp.solutions.pose.POSE_CONNECTIONS` (Holistic / Pose Landmarker - /// share the same 33-pt schema). - static let POSE_CONNECTIONS: [(Int, Int, BoneChain)] = [ - // Face (kept light: nose <-> inner eyes <-> outer eyes <-> ears) - (0, 1, .face), (1, 2, .face), (2, 3, .face), (3, 7, .face), - (0, 4, .face), (4, 5, .face), (5, 6, .face), (6, 8, .face), - (9, 10, .face), - // Torso - (11, 12, .trunk), (11, 23, .trunk), (12, 24, .trunk), - (23, 24, .trunk), - // Left arm - (11, 13, .arm), (13, 15, .arm), - (15, 17, .arm), (15, 19, .arm), (15, 21, .arm), (17, 19, .arm), - // Right arm - (12, 14, .arm), (14, 16, .arm), - (16, 18, .arm), (16, 20, .arm), (16, 22, .arm), (18, 20, .arm), - // Left leg - (23, 25, .leg), (25, 27, .leg), - (27, 29, .leg), (27, 31, .leg), (29, 31, .leg), - // Right leg - (24, 26, .leg), (26, 28, .leg), - (28, 30, .leg), (28, 32, .leg), (30, 32, .leg), - ] - - enum BoneChain { - case trunk, arm, leg, face - var color: NSColor { - switch self { - case .trunk: return .white - case .arm: return .systemTeal - case .leg: return .systemPink // approx magenta - case .face: return NSColor(white: 0.7, alpha: 1.0) - } - } - } - - private static let jointRadius: Float = 0.02 // 2 cm - private static let boneRadius: Float = 0.012 // 1.2 cm - private static let minConfidence: Float = 0.3 - private static let retainSec: TimeInterval = 1.0 - - /// Update throttle : tick at most every `updatePeriod` seconds even - /// if the publisher fires faster (Combine debounce-style on a clock). - private static let updatePeriod: TimeInterval = 1.0 / 30.0 - - private struct PersonEntities { - var root: Entity - var joints: [ModelEntity] // 33 spheres - var bones: [ModelEntity] // 32 bone entities, same order as POSE_CONNECTIONS - } - - private var persons: [Int: PersonEntities] = [:] - private var lastSeenAt: [Int: TimeInterval] = [:] - private weak var rootAnchor: Entity? - private var poseSub: AnyCancellable? - private var lastUpdateAt: TimeInterval = 0 - - /// Attach to a scene by giving it an AnchorEntity that owns all - /// skeleton entities, and start observing the listener. - func attach(to anchor: Entity, listener: PoseOSCListener) { - rootAnchor = anchor - poseSub = listener.$body3d - .receive(on: DispatchQueue.main) - .sink { [weak self] frames in - Task { @MainActor in self?.update(frames: frames) } - } - } - - func detach() { - poseSub?.cancel() - poseSub = nil - for (_, p) in persons { p.root.removeFromParent() } - persons.removeAll() - lastSeenAt.removeAll() - } - - // MARK: - Update - - private func update(frames: [Int: PoseOSCListener.Pose3DFrame]) { - let now = CACurrentMediaTime() - if now - lastUpdateAt < Self.updatePeriod { return } - lastUpdateAt = now - - guard let anchor = rootAnchor else { return } - - // Mark fresh pids - for pid in frames.keys { lastSeenAt[pid] = now } - // GC stale persons - let cutoff = now - Self.retainSec - for (pid, p) in persons where (lastSeenAt[pid] ?? 0) < cutoff { - p.root.removeFromParent() - persons.removeValue(forKey: pid) - lastSeenAt.removeValue(forKey: pid) - } - - for (pid, frame) in frames { - let entities = persons[pid] ?? makePerson(pid: pid, parent: anchor) - persons[pid] = entities - apply(frame: frame, to: entities) - } - } - - private func apply(frame: PoseOSCListener.Pose3DFrame, - to entities: PersonEntities) { - // Convert all 33 keypoints to RealityKit space once. - var rk = [SIMD3](repeating: .zero, count: 33) - var valid = [Bool](repeating: false, count: 33) - for i in 0..<33 { - let k = frame.kps[i] - let visible = frame.hasPoint[i] && k.w >= Self.minConfidence - valid[i] = visible - // Mediapipe (x right, y down, z forward) -> RK (x right, y up, z back) - rk[i] = SIMD3(k.x, -k.y, -k.z) - } - - // Joints: position spheres and toggle visibility. - for i in 0..<33 { - let joint = entities.joints[i] - if valid[i] { - joint.transform.translation = rk[i] - joint.isEnabled = true - } else { - joint.isEnabled = false - } - } - - // Bones: orient + scale length between endpoints. - for (bIdx, (a, b, _)) in Self.POSE_CONNECTIONS.enumerated() { - let bone = entities.bones[bIdx] - if !valid[a] || !valid[b] { - bone.isEnabled = false - continue - } - let pa = rk[a] - let pb = rk[b] - let delta = pb - pa - let len = simd_length(delta) - if len < 1e-5 { - bone.isEnabled = false - continue - } - let mid = (pa + pb) * 0.5 - // Bone mesh is a cylinder of height=1 along +Y. Rotate +Y - // onto the (b-a) direction. - let dir = delta / len - let yAxis = SIMD3(0, 1, 0) - let dot = simd_dot(yAxis, dir) - let rot: simd_quatf - if dot > 0.9999 { - rot = simd_quatf(angle: 0, axis: SIMD3(0, 1, 0)) - } else if dot < -0.9999 { - rot = simd_quatf(angle: .pi, axis: SIMD3(1, 0, 0)) - } else { - let axis = simd_normalize(simd_cross(yAxis, dir)) - let angle = acos(dot) - rot = simd_quatf(angle: angle, axis: axis) - } - bone.transform.translation = mid - bone.transform.rotation = rot - // Scale length only on Y, keep XZ at 1 to preserve radius. - bone.transform.scale = SIMD3(1, len, 1) - bone.isEnabled = true - } - } - - // MARK: - Construction - - private func makePerson(pid: Int, parent: Entity) -> PersonEntities { - let root = Entity() - parent.addChild(root) - - // Joint sphere mesh shared across joints (cheap to reuse). - let sphereMesh = MeshResource.generateSphere( - radius: Self.jointRadius) - let jointMat = SimpleMaterial( - color: .white, roughness: 0.6, isMetallic: false) - var joints: [ModelEntity] = [] - joints.reserveCapacity(33) - for _ in 0..<33 { - let e = ModelEntity(mesh: sphereMesh, materials: [jointMat]) - e.isEnabled = false - root.addChild(e) - joints.append(e) - } - - // One cylinder per bone (height=1, scaled at runtime). - let cylMesh = MeshResource.generateCylinder( - height: 1.0, radius: Self.boneRadius) - var bones: [ModelEntity] = [] - bones.reserveCapacity(Self.POSE_CONNECTIONS.count) - for (_, _, chain) in Self.POSE_CONNECTIONS { - let mat = SimpleMaterial( - color: chain.color, roughness: 0.6, isMetallic: false) - let e = ModelEntity(mesh: cylMesh, materials: [mat]) - e.isEnabled = false - root.addChild(e) - bones.append(e) - } - NSLog("Skeleton3DRenderer: spawned pid=%d (33 joints, %d bones)", - pid, bones.count) - return PersonEntities(root: root, joints: joints, bones: bones) - } -} diff --git a/launcher/CLAUDE.md b/launcher/CLAUDE.md index 33ff04d..a535cfc 100644 --- a/launcher/CLAUDE.md +++ b/launcher/CLAUDE.md @@ -1,3 +1,5 @@ +> Note (2026-05-18): the `AV-Live-Body` directory has been archived to `_archive-AV-Live-Body/` — superseded by the new app at `/avlivebody-mac/`. See `docs/superpowers/specs/2026-05-18-avlivebody-macos-rewrite-design.md`. + # launcher — AVLiveLauncher App menubar macOS (SwiftUI, SwiftPM, macOS 11+) qui démarre/arrête `sclang+scsynth`, le serveur web et `oscope-of`, log les sorties, et expose un mode picker. diff --git a/launcher/Sources/AVLiveLauncher/ProcessManager.swift b/launcher/Sources/AVLiveLauncher/ProcessManager.swift index afe5c94..2824539 100644 --- a/launcher/Sources/AVLiveLauncher/ProcessManager.swift +++ b/launcher/Sources/AVLiveLauncher/ProcessManager.swift @@ -757,38 +757,17 @@ final class ProcessManager: ObservableObject { /// Lance l'app SwiftPM AV-Live-Body (RealityKit) qui ecoute les /// vertices SMPL-X sur :57130. Necessite useMultiHMR=true. func startBodyApp() { - // Lancable a la demande depuis n'importe quel mode. Si Multi-HMR - // n'est pas actif, la fenetre affichera juste la cam (pas de mesh) - // mais le panel S et les reglages restent fonctionnels. - guard bodyAppProc == nil else { return } - let pkgDir = URL(fileURLWithPath: metalVizDir) - .deletingLastPathComponent() - .appendingPathComponent("launcher/AV-Live-Body") - guard FileManager.default.fileExists( - atPath: pkgDir.appendingPathComponent("Package.swift").path) else { - append(source: "launcher", - text: "AV-Live-Body missing at \(pkgDir.path)") - return - } - let p = Process() - p.executableURL = URL(fileURLWithPath: "/usr/bin/env") - p.arguments = ["swift", "run", "-c", "release", "AVLiveBody"] - p.currentDirectoryURL = pkgDir - attach(process: p, label: "body") - do { - try p.run() - bodyAppProc = p - DispatchQueue.main.async { self.bodyAppRunning = true } - append(source: "launcher", text: "started AV-Live-Body") - p.terminationHandler = { [weak self] _ in - DispatchQueue.main.async { - self?.bodyAppProc = nil - self?.bodyAppRunning = false - } - } - } catch { - append(source: "launcher", text: "startBodyApp failed: \(error)") - } + // FIXME(rewrite-2026-05-18): wire to avlivebody-mac/AVLiveBody.app + // once installation path is decided. The legacy SwiftPM target at + // launcher/AV-Live-Body has been archived to + // launcher/_archive-AV-Live-Body/ and superseded by the native + // Xcode app at avlivebody-mac/ (xcodegen + xcodebuild). There is + // no `swift run` equivalent for an Xcode app target, so the + // spawn path is disabled until we settle on a post-build .app + // install location (DerivedData vs ~/Applications vs bundled). + NSLog("AVLiveBody is now standalone at avlivebody-mac/ — open from Xcode (build target AVLiveBody)") + append(source: "launcher", + text: "AVLiveBody spawn disabled — open avlivebody-mac/ in Xcode") } func stopBodyApp() { diff --git a/launcher/_archive-AV-Live-Body/ARCHIVED.md b/launcher/_archive-AV-Live-Body/ARCHIVED.md new file mode 100644 index 0000000..bb41a14 --- /dev/null +++ b/launcher/_archive-AV-Live-Body/ARCHIVED.md @@ -0,0 +1,2 @@ +Superseded by `avlivebody-mac/` on 2026-05-18 — see +`docs/superpowers/specs/2026-05-18-avlivebody-macos-rewrite-design.md`. diff --git a/launcher/AV-Live-Body/Package.swift b/launcher/_archive-AV-Live-Body/Package.swift similarity index 52% rename from launcher/AV-Live-Body/Package.swift rename to launcher/_archive-AV-Live-Body/Package.swift index 6ed5fa7..1a3e991 100644 --- a/launcher/AV-Live-Body/Package.swift +++ b/launcher/_archive-AV-Live-Body/Package.swift @@ -4,9 +4,15 @@ import PackageDescription let package = Package( name: "AVLiveBody", platforms: [.macOS(.v15)], + dependencies: [ + .package(path: "../../shared/AVLiveWire"), + ], targets: [ .executableTarget( name: "AVLiveBody", + dependencies: [ + .product(name: "AVLiveWire", package: "AVLiveWire"), + ], path: "Sources/AVLiveBody", resources: [ .copy("Resources/smplx_faces.bin"), @@ -15,6 +21,14 @@ let package = Package( swiftSettings: [ .swiftLanguageMode(.v5), ] - ) + ), + .testTarget( + name: "AVLiveBodyTests", + dependencies: ["AVLiveBody"], + path: "Tests/AVLiveBodyTests", + swiftSettings: [ + .swiftLanguageMode(.v5), + ] + ), ] ) diff --git a/launcher/AV-Live-Body/Resources/smpl_faces.bin b/launcher/_archive-AV-Live-Body/Resources/smpl_faces.bin similarity index 100% rename from launcher/AV-Live-Body/Resources/smpl_faces.bin rename to launcher/_archive-AV-Live-Body/Resources/smpl_faces.bin diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift similarity index 85% rename from launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift index 808e8cb..a12293e 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift @@ -65,14 +65,22 @@ struct ContentView: View { @StateObject private var renderer = MeshRenderer() @StateObject private var settings = RenderSettings() @StateObject private var poseListener = PoseOSCListener() + @StateObject private var arkitListener = ArkitOSCListener() + @StateObject private var dataFeeds = DataFeedsOSCListener() + @StateObject private var usbConsumer = USBSkeletonConsumer() var body: some View { ZStack(alignment: .topLeading) { BodyView(renderer: renderer, settings: settings, - poseListener: poseListener) + poseListener: poseListener, + arkitListener: arkitListener, + usbConsumer: usbConsumer) .onAppear { renderer.startOSCServer() poseListener.start() + arkitListener.start() + dataFeeds.start() + usbConsumer.start() } .onReceive(NotificationCenter.default.publisher( for: .toggleSettings)) { _ in @@ -87,9 +95,22 @@ struct ContentView: View { if let n = note.object as? Int { settings.vizMode = n } } + // Face + hand overlay 2D Canvas retire : les landmarks sont + // maintenant integres au squelette 3D RealityKit (cf. + // Skeleton3DRenderer.applyFace/applyHands). + // HUD coin haut-gauche : mode + touches + pose HUDOverlay(settings: settings, poseListener: poseListener) + // Data feeds HUD : telemetrie open-data (eco2mix, velib, ...) + VStack { + Spacer() + HStack { + DataHUDOverlay(data: dataFeeds, settings: settings) + Spacer() + } + } + // Bouton settings coin haut-droit HStack { Spacer() diff --git a/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/ArkitOSCListener.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/ArkitOSCListener.swift new file mode 100644 index 0000000..29e2c18 --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/ArkitOSCListener.swift @@ -0,0 +1,153 @@ +import Foundation +import Network +import simd + +/// Listener UDP secondaire qui consomme /body3d/kp envoyes par l'app +/// iOS ARBodyTracker. Permet une visualisation diagnostique des joints +/// ARKit (91 joints, LiDAR-anchored) en parallele du flux MediaPipe +/// fusionne cote Python. Port distinct de PoseOSCListener (:57126) +/// pour eviter le clash de bind UDP : l'iPhone doit pousser vers les +/// deux ports si on veut Python ET Swift ; ou utiliser un fanout +/// (proxy UDP one-to-many) si une seule destination est practique. +final class ArkitOSCListener: ObservableObject { + struct ArkitBodyFrame: Equatable { + var pid: Int = -1 + /// 91 ARKit joints world-space (x, y, z meters). + var joints: [SIMD3] = Array(repeating: .zero, count: 91) + /// Per-joint "has been written" flag. + var hasJoint: [Bool] = Array(repeating: false, count: 91) + var seenAt: TimeInterval = 0 + } + + @Published var bodies: [Int: ArkitBodyFrame] = [:] + @Published var count: Int = 0 + + static let defaultPort: UInt16 = 57129 // distinct from :57128 (Python) + private var listener: NWListener? + + func start(port: UInt16 = ArkitOSCListener.defaultPort) { + do { + let params = NWParameters.udp + params.allowLocalEndpointReuse = true + let l = try NWListener(using: params, + on: NWEndpoint.Port(rawValue: port)!) + l.newConnectionHandler = { [weak self] conn in + conn.start(queue: .global(qos: .userInitiated)) + self?.receive(on: conn) + } + l.start(queue: .global()) + self.listener = l + NSLog("ArkitOSCListener: udp :%d up", port) + } catch { + NSLog("ArkitOSCListener: bind :%d failed: %@", + Int(port), String(describing: error)) + } + } + + private func receive(on conn: NWConnection) { + conn.receiveMessage { [weak self] data, _, _, error in + if let d = data, !d.isEmpty { + self?.handle(packet: d) + } + if error == nil { self?.receive(on: conn) } + } + } + + private func handle(packet: Data) { + guard let (address, types, payload) = parseOSCHeader(packet) else { + return + } + let args = parseOSCArgs(types: types, data: payload) + DispatchQueue.main.async { [weak self] in + self?.apply(address: address, args: args) + } + } + + private func apply(address: String, args: [Any]) { + switch address { + case "/body3d/count": + if let n = args.first as? Int32 { count = Int(n) } + if count == 0 { bodies.removeAll(keepingCapacity: true) } + case "/body3d/kp": + // pid (i32), joint_idx (i32), x (f32), y (f32), z (f32) + guard args.count >= 5, + let pid = args[0] as? Int32, + let idx = args[1] as? Int32, + let x = args[2] as? Float, + let y = args[3] as? Float, + let z = args[4] as? Float else { return } + let i = Int(idx) + guard i >= 0 && i < 91 else { return } + var b = bodies[Int(pid)] ?? ArkitBodyFrame() + b.pid = Int(pid) + b.joints[i] = SIMD3(x, y, z) + b.hasJoint[i] = true + b.seenAt = CFAbsoluteTimeGetCurrent() + bodies[Int(pid)] = b + default: + break + } + // Garbage-collect bodies non vus depuis > 2 s + let now = CFAbsoluteTimeGetCurrent() + bodies = bodies.filter { now - $0.value.seenAt < 2.0 } + } + + // MARK: - Minimal OSC parser (mirror of PoseOSCListener helpers) + + private func align4(_ n: Int) -> Int { (n + 3) & ~3 } + + private func parseOSCHeader(_ data: Data + ) -> (String, String, Data)? { + guard let endAddr = data.firstIndex(of: 0) else { return nil } + let address = String(data: data[.. [Any] { + var args: [Any] = [] + var offset = 0 + for t in types { + switch t { + case "i": + guard offset + 4 <= data.count else { return args } + let v = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: Int32.self) + }.bigEndian + args.append(v) + offset += 4 + case "f": + guard offset + 4 <= data.count else { return args } + let raw = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: UInt32.self) + }.bigEndian + args.append(Float(bitPattern: raw)) + offset += 4 + case "s": + let start = offset + while offset < data.count + && data[data.startIndex.advanced(by: offset)] != 0 { + offset += 1 + } + let lo = data.startIndex.advanced(by: start) + let hi = data.startIndex.advanced(by: offset) + let slice = data[lo.. NSView { let container = NSView(frame: .zero) @@ -101,6 +103,17 @@ struct BodyView: NSViewRepresentable { context.coordinator.sceneRenderer = scene context.coordinator.mtkView = mtkView context.coordinator.skeletonOverlay = SkeletonOverlay(parent: bodyAnchor) + // Skeleton 3D RealityKit armature (33 spheres + 32 cylinders bones) + // driven by /pose3d/* OSC from MediaPipe pose_world_landmarks. + // Visible quand toggle showSkeleton ou vizMode==9 (openpos). + let skel3dAnchor = AnchorEntity(world: SIMD3(0, 0, -2.5)) + arView.scene.addAnchor(skel3dAnchor) + let skel3d = Skeleton3DRenderer() + skel3d.attach(to: skel3dAnchor, listener: poseListener, + arkitListener: arkitListener, + usbConsumer: usbConsumer) + context.coordinator.skel3dAnchor = skel3dAnchor + context.coordinator.skel3d = skel3d context.coordinator.keyLight = key context.coordinator.fillLight = fill context.coordinator.rimLight = rim @@ -162,6 +175,12 @@ struct BodyView: NSViewRepresentable { let skelVisible = settings.vizMode == 9 || settings.showSkeleton c.skeletonOverlay?.update(persons: poseListener.persons, visible: skelVisible) + // 3D RealityKit armature : show/hide root anchor in sync with + // the same skelVisible signal as the 2D overlay. Skeleton keeps + // its own hip-relative coords (z=-3 anchor), mesh keeps its own + // world coords — both visible together in mode openpos, no + // spatial fusion (original design from commit f540158). + c.skel3dAnchor?.isEnabled = skelVisible // Pose -> scene uniforms : drive hands3d (mode 8) et openpos // (mode 9) avec la premiere personne detectee. Les wrists pilotent // hand_l/r ; pose_count alimente bg_fragment. @@ -211,6 +230,8 @@ struct BodyView: NSViewRepresentable { var sceneRenderer: SceneRenderer? var mtkView: MTKView? var skeletonOverlay: SkeletonOverlay? + var skel3dAnchor: AnchorEntity? + var skel3d: Skeleton3DRenderer? var kbMonitor: Any? deinit { diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/CameraPreviewLayer.swift diff --git a/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/DataFeedsOSCListener.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/DataFeedsOSCListener.swift new file mode 100644 index 0000000..0b702d7 --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/DataFeedsOSCListener.swift @@ -0,0 +1,177 @@ +import Foundation +import Network + +/// UDP :57127 listener that consumes /data//* messages from +/// the Python data_feeds package and exposes the latest per-feed +/// payloads to the SwiftUI HUD via @Published. +@MainActor +final class DataFeedsOSCListener: ObservableObject { + struct TransientEvent: Equatable, Identifiable { + let id = UUID() + let feed: String + let key: String + let value: Double + let seenAt: TimeInterval + } + + @Published var samples: [String: [String: Double]] = [:] + @Published var counts: [String: Int] = [:] + @Published var heartbeats: [String: TimeInterval] = [:] + @Published var recentEvents: [TransientEvent] = [] + + private var listener: NWListener? + static let defaultPort: UInt16 = 57127 + private static let maxEvents: Int = 32 + + func start(port: UInt16 = DataFeedsOSCListener.defaultPort) { + do { + let params = NWParameters.udp + params.allowLocalEndpointReuse = true + let l = try NWListener(using: params, + on: NWEndpoint.Port(rawValue: port)!) + l.newConnectionHandler = { [weak self] conn in + conn.start(queue: .global(qos: .userInitiated)) + self?.receive(on: conn) + } + l.start(queue: .global()) + self.listener = l + NSLog("DataFeedsOSCListener UDP :%d", port) + } catch { + NSLog("DataFeedsOSCListener bind :%d failed: %@", + Int(port), String(describing: error)) + } + } + + private nonisolated func receive(on conn: NWConnection) { + conn.receiveMessage { [weak self] data, _, _, error in + if let d = data, !d.isEmpty { + self?.handle(packet: d) + } + if error == nil { self?.receive(on: conn) } + } + } + + private nonisolated func handle(packet: Data) { + guard let (address, types, payload) = parseOSCHeader(packet) else { + return + } + let args = parseOSCArgs(types: types, data: payload) + Task { @MainActor [weak self] in + self?.apply(address: address, args: args) + } + } + + private func apply(address: String, args: [Any]) { + // address shape : /data// + let trimmed = address.hasPrefix("/") + ? String(address.dropFirst()) : address + let comps = trimmed.split(separator: "/") + guard comps.count >= 3, comps[0] == "data" else { return } + let feed = String(comps[1]) + let route = String(comps[2]) + switch route { + case "count": + if let n = args.first as? Int32 { + counts[feed] = Int(n) + } + case "heartbeat": + if let t = args.first as? Float { + heartbeats[feed] = TimeInterval(t) + } else if let t = args.first as? Double { + heartbeats[feed] = t + } + case "sample": + guard args.count >= 2 else { return } + let key: String + switch args[0] { + case let s as String: key = s + case let i as Int32: key = String(i) + default: return + } + let value: Double + switch args[1] { + case let f as Float: value = Double(f) + case let d as Double: value = d + case let i as Int32: value = Double(i) + default: return + } + var d = samples[feed] ?? [:] + d[key] = value + samples[feed] = d + let ev = TransientEvent(feed: feed, key: key, value: value, + seenAt: CFAbsoluteTimeGetCurrent()) + recentEvents.append(ev) + if recentEvents.count > Self.maxEvents { + recentEvents.removeFirst(recentEvents.count - Self.maxEvents) + } + default: + break + } + } + + // MARK: - Minimal OSC parser (mirror of ArkitOSCListener helpers) + + private nonisolated func align4(_ n: Int) -> Int { (n + 3) & ~3 } + + private nonisolated func parseOSCHeader(_ data: Data + ) -> (String, String, Data)? { + guard let endAddr = data.firstIndex(of: 0) else { return nil } + let address = String(data: data[.. [Any] { + var args: [Any] = [] + var offset = 0 + for t in types { + switch t { + case "i": + guard offset + 4 <= data.count else { return args } + let v = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: Int32.self) + }.bigEndian + args.append(v) + offset += 4 + case "f": + guard offset + 4 <= data.count else { return args } + let raw = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: UInt32.self) + }.bigEndian + args.append(Float(bitPattern: raw)) + offset += 4 + case "d": + guard offset + 8 <= data.count else { return args } + let raw = data.withUnsafeBytes { + $0.loadUnaligned(fromByteOffset: offset, as: UInt64.self) + }.bigEndian + args.append(Double(bitPattern: raw)) + offset += 8 + case "s": + let start = offset + while offset < data.count + && data[data.startIndex.advanced(by: offset)] != 0 { + offset += 1 + } + let lo = data.startIndex.advanced(by: start) + let hi = data.startIndex.advanced(by: offset) + let slice = data[lo.. Color { + let now = Date().timeIntervalSince1970 + let last = data.heartbeats[feed] ?? 0 + let age = now - last + if age < 60 { return .green } + if age < 600 { return .yellow } + return .red + } + + private func summary(for feed: String) -> String { + let s = data.samples[feed] ?? [:] + if let n = data.counts[feed] { + return "\(s.count) keys / \(n)" + } + return "\(s.count) keys" + } +} diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/FaceHandOverlay.swift diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/HUDOverlay.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/HUDOverlay.swift similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/HUDOverlay.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/HUDOverlay.swift diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift similarity index 96% rename from launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift index 92f3fa9..d040c3f 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift @@ -9,6 +9,11 @@ import SwiftUI @MainActor final class MeshRenderer: ObservableObject { @Published var personEntities: [Int: ModelEntity] = [:] + /// Pelvis world position (RealityKit coords) par pid. Mis a jour + /// chaque frame TCP. Consume par Skeleton3DRenderer pour aligner + /// le squelette 3D openpos avec le mesh dense SMPL-X. + /// SMPL-X pelvis = vertex index 0 par convention canonique. + @Published var pelvisWorld: [Int: SIMD3] = [:] private var faces: [UInt32] = [] private var oscServer: OSCServer? @@ -130,6 +135,7 @@ final class MeshRenderer: ObservableObject { wireframeEntities.removeValue(forKey: pid) interpStates.removeValue(forKey: pid) lastSeenAt.removeValue(forKey: pid) + pelvisWorld.removeValue(forKey: pid) } } for p in persons { @@ -163,6 +169,11 @@ final class MeshRenderer: ObservableObject { displayed: converted, target: converted) } entity.transform.translation = SIMD3.zero + // Track pelvis world position (vertex 0 = SMPL-X pelvis) so + // Skeleton3DRenderer can co-locate openpos joints with mesh. + if !converted.isEmpty { + pelvisWorld[p.pid] = converted[0] + } // Bbox debug : utile une fois par creation if personEntities[p.pid] == nil { let xs = converted.map(\.x) diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/OSCServer.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/OSCServer.swift similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/OSCServer.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/OSCServer.swift diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift similarity index 67% rename from launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift index bdd8c59..2c4336a 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/PoseOSCListener.swift @@ -26,8 +26,39 @@ final class PoseOSCListener: ObservableObject { var seenAt: TimeInterval = 0 } + /// MediaPipe pose_world_landmarks : 33 keypoints in meters, hip-rel. + /// MediaPipe convention : x=right, y=down, z=forward (away from cam). + struct Pose3DFrame: Equatable { + var pid: Int = -1 + var kps: [SIMD4] = Array(repeating: .zero, count: 33) + var hasPoint: [Bool] = Array(repeating: false, count: 33) + var seenAt: TimeInterval = 0 + } + + /// 68 dlib-style facial landmarks (x,y normalises 0..1). + struct FaceFrame: Equatable { + var points: [SIMD2] = Array(repeating: .zero, count: 68) + var hasPoint: [Bool] = Array(repeating: false, count: 68) + var seenAt: TimeInterval = 0 + } + + /// 21 MediaPipe hand landmarks per detected hand. + struct HandFrame: Equatable { + var side: Int = 0 + var points: [SIMD2] = Array(repeating: .zero, count: 21) + var hasPoint: [Bool] = Array(repeating: false, count: 21) + var seenAt: TimeInterval = 0 + } + @Published var persons: [Int: PoseFrame] = [:] @Published var count: Int = 0 + @Published var body3d: [Int: Pose3DFrame] = [:] + @Published var body3dCount: Int = 0 + @Published var faces: [Int: FaceFrame] = [:] + @Published var hands: [Int: HandFrame] = [:] + @Published var faceCount: Int = 0 + @Published var handCountLeft: Int = 0 + @Published var handCountRight: Int = 0 private var listener: NWListener? @@ -148,13 +179,77 @@ final class PoseOSCListener: ObservableObject { p.skeleton = skel p.seenAt = CFAbsoluteTimeGetCurrent() persons[Int(pid)] = p + case "/face/count": + if let n = args.first as? Int32 { faceCount = Int(n) } + if faceCount == 0 { faces.removeAll(keepingCapacity: true) } + case "/face/kp": + guard args.count >= 6, + let pid = args[0] as? Int32, + let slot = args[1] as? Int32, + let x = args[2] as? Float, + let y = args[3] as? Float else { return } + let s = Int(slot) + guard s >= 0 && s < 68 else { return } + var f = faces[Int(pid)] ?? FaceFrame() + f.points[s] = SIMD2(x, y) + f.hasPoint[s] = true + f.seenAt = CFAbsoluteTimeGetCurrent() + faces[Int(pid)] = f + case "/hand/count": + if args.count >= 2, + let l = args[0] as? Int32, let r = args[1] as? Int32 { + handCountLeft = Int(l) + handCountRight = Int(r) + if handCountLeft + handCountRight == 0 { + hands.removeAll(keepingCapacity: true) + } + } + case "/hand/kp": + guard args.count >= 7, + let pid = args[0] as? Int32, + let side = args[1] as? Int32, + let idx = args[2] as? Int32, + let x = args[3] as? Float, + let y = args[4] as? Float else { return } + let i = Int(idx) + guard i >= 0 && i < 21 else { return } + var h = hands[Int(pid)] ?? HandFrame() + h.side = Int(side) + h.points[i] = SIMD2(x, y) + h.hasPoint[i] = true + h.seenAt = CFAbsoluteTimeGetCurrent() + hands[Int(pid)] = h + case "/pose3d/count": + if let n = args.first as? Int32 { + body3dCount = Int(n) + if body3dCount == 0 { + body3d.removeAll(keepingCapacity: true) + } + } + case "/pose3d/kp": + guard args.count >= 6, + let pid = args[0] as? Int32, + let idx = args[1] as? Int32, + let x = args[2] as? Float, + let y = args[3] as? Float, + let z = args[4] as? Float, + let c = args[5] as? Float else { return } + let i = Int(idx) + guard i >= 0 && i < 33 else { return } + var p = body3d[Int(pid)] ?? Pose3DFrame(pid: Int(pid)) + p.pid = Int(pid) + p.kps[i] = SIMD4(x, y, z, c) + p.hasPoint[i] = true + p.seenAt = CFAbsoluteTimeGetCurrent() + body3d[Int(pid)] = p default: break } - // Garbage-collect persons non vues depuis > 2 s + // Garbage-collect persons + body3d non vus depuis > 2 s let now = CFAbsoluteTimeGetCurrent() persons = persons.filter { $0.value.seenAt == 0 || now - $0.value.seenAt < 2.0 } + body3d = body3d.filter { now - $0.value.seenAt < 2.0 } } // MARK: - Minimal OSC parser diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/RenderSettings.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/RenderSettings.swift similarity index 93% rename from launcher/AV-Live-Body/Sources/AVLiveBody/RenderSettings.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/RenderSettings.swift index 74860ce..7010cd9 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/RenderSettings.swift +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/RenderSettings.swift @@ -39,4 +39,7 @@ final class RenderSettings: ObservableObject { // Settings panel visibility @Published var showPanel: Bool = false + + // Data feeds HUD overlay (open-data telemetry chips) + @Published var showDataHUD: Bool = true } diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/Resources/scene.metal b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Resources/scene.metal similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/Resources/scene.metal rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Resources/scene.metal diff --git a/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Resources/smplx_faces.bin b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Resources/smplx_faces.bin new file mode 100644 index 0000000..8d99c4a Binary files /dev/null and b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Resources/smplx_faces.bin differ diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/SettingsPanel.swift diff --git a/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift new file mode 100644 index 0000000..52846be --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift @@ -0,0 +1,486 @@ +import Combine +import Foundation +import RealityKit +import SwiftUI +import simd + +/// RealityKit renderer for MediaPipe Pose 3D world landmarks (33 joints, +/// metric coords relative to the hip-center). Consumes the `body3d` +/// publisher of `PoseOSCListener` and maintains one entity tree per +/// detected person. +/// +/// Coordinate mapping (MediaPipe -> RealityKit): +/// MediaPipe : x = right, y = down, z = forward (away from cam). +/// RealityKit: x = right, y = up, z = backward (toward cam). +/// => convert with (x, -y, -z). +@MainActor +final class Skeleton3DRenderer: ObservableObject { + /// 32 bones connecting MediaPipe Pose 33 landmarks. Indices are + /// the canonical MediaPipe Pose landmark indices. Source: official + /// `mp.solutions.pose.POSE_CONNECTIONS` (Holistic / Pose Landmarker + /// share the same 33-pt schema). + static let POSE_CONNECTIONS: [(Int, Int, BoneChain)] = [ + // Face (kept light: nose <-> inner eyes <-> outer eyes <-> ears) + (0, 1, .face), (1, 2, .face), (2, 3, .face), (3, 7, .face), + (0, 4, .face), (4, 5, .face), (5, 6, .face), (6, 8, .face), + (9, 10, .face), + // Torso + (11, 12, .trunk), (11, 23, .trunk), (12, 24, .trunk), + (23, 24, .trunk), + // Left arm + (11, 13, .arm), (13, 15, .arm), + (15, 17, .arm), (15, 19, .arm), (15, 21, .arm), (17, 19, .arm), + // Right arm + (12, 14, .arm), (14, 16, .arm), + (16, 18, .arm), (16, 20, .arm), (16, 22, .arm), (18, 20, .arm), + // Left leg + (23, 25, .leg), (25, 27, .leg), + (27, 29, .leg), (27, 31, .leg), (29, 31, .leg), + // Right leg + (24, 26, .leg), (26, 28, .leg), + (28, 30, .leg), (28, 32, .leg), (30, 32, .leg), + ] + + enum BoneChain { + case trunk, arm, leg, face + var color: NSColor { + switch self { + case .trunk: return .white + case .arm: return .systemTeal + case .leg: return .systemPink // approx magenta + case .face: return NSColor(white: 0.7, alpha: 1.0) + } + } + } + + // Wireframe pur : joints micro (1 mm, quasi invisibles), bones + // 2 mm = lignes fines. radius=0 fait planter MeshResource.generate. + private static let jointRadius: Float = 0.001 // 1 mm — quasi nul + private static let boneRadius: Float = 0.003 // 3 mm — line-like + private static let faceJointRadius: Float = 0.001 + private static let handJointRadius: Float = 0.001 + private static let handScale3D: Float = 0.18 // typical hand size (m) + private static let faceForwardOffset: Float = 0.05 // push face in front of nose + private static let minConfidence: Float = 0.3 + private static let retainSec: TimeInterval = 1.0 + + /// Update throttle : tick at most every `updatePeriod` seconds even + /// if the publisher fires faster (Combine debounce-style on a clock). + private static let updatePeriod: TimeInterval = 1.0 / 30.0 + + private struct PersonEntities { + var root: Entity + var joints: [ModelEntity] // 33 spheres + var bones: [ModelEntity] // 32 bone entities, same order as POSE_CONNECTIONS + var faceJoints: [ModelEntity] // 68 dlib face landmarks + var leftHandJoints: [ModelEntity] // 21 cyan + var rightHandJoints: [ModelEntity] // 21 magenta + var arkitMarkers: [ModelEntity] // 91 yellow ARKit/USB joints + } + + private var persons: [Int: PersonEntities] = [:] + private var lastSeenAt: [Int: TimeInterval] = [:] + private var lastFace: [Int: PoseOSCListener.FaceFrame] = [:] + private var lastHands: [Int: PoseOSCListener.HandFrame] = [:] + private weak var rootAnchor: Entity? + private var poseSub: AnyCancellable? + private var faceSub: AnyCancellable? + private var handSub: AnyCancellable? + private var arkitSub: AnyCancellable? + private var usbSub: AnyCancellable? + private var lastArkit: [Int: ArkitOSCListener.ArkitBodyFrame] = [:] + private var lastUpdateAt: TimeInterval = 0 + /// Optional per-pid offset to align the skeleton with another + /// renderer's coordinate space (typically MeshRenderer's pelvis). + /// When nil for a pid, falls back to the renderer's default anchor. + private var pelvisOffsets: [Int: SIMD3] = [:] + + /// External wiring : MeshRenderer or any other source publishes a + /// pelvis world position per pid. We translate each person's root + /// entity to that pose so the openpos skeleton co-locates with the + /// dense SMPL-X mesh. + func setPelvisOffsets(_ offsets: [Int: SIMD3]) { + pelvisOffsets = offsets + for (pid, entities) in persons { + if let off = offsets[pid] { + entities.root.transform.translation = off + } else { + entities.root.transform.translation = .zero + } + } + } + + /// Attach to a scene by giving it an AnchorEntity that owns all + /// skeleton entities, and start observing the listener. + func attach(to anchor: Entity, listener: PoseOSCListener, + arkitListener: ArkitOSCListener? = nil, + usbConsumer: USBSkeletonConsumer? = nil) { + rootAnchor = anchor + poseSub = listener.$body3d + .receive(on: DispatchQueue.main) + .sink { [weak self] frames in + Task { @MainActor in self?.update(frames: frames) } + } + faceSub = listener.$faces + .receive(on: DispatchQueue.main) + .sink { [weak self] frames in + Task { @MainActor in self?.lastFace = frames } + } + handSub = listener.$hands + .receive(on: DispatchQueue.main) + .sink { [weak self] frames in + Task { @MainActor in self?.lastHands = frames } + } + if let arkit = arkitListener { + arkitSub = arkit.$bodies + .receive(on: DispatchQueue.main) + .sink { [weak self] frames in + Task { @MainActor in self?.lastArkit = frames } + } + } + if let usb = usbConsumer { + usbSub = usb.$bodies + .receive(on: DispatchQueue.main) + .sink { [weak self] frames in + Task { @MainActor in self?.lastArkit = frames } + } + } + } + + func detach() { + poseSub?.cancel() + poseSub = nil + faceSub?.cancel() + faceSub = nil + handSub?.cancel() + handSub = nil + arkitSub?.cancel() + arkitSub = nil + usbSub?.cancel() + usbSub = nil + for (_, p) in persons { p.root.removeFromParent() } + persons.removeAll() + lastSeenAt.removeAll() + lastFace.removeAll() + lastHands.removeAll() + } + + // MARK: - Update + + private func update(frames: [Int: PoseOSCListener.Pose3DFrame]) { + let now = CACurrentMediaTime() + if now - lastUpdateAt < Self.updatePeriod { return } + lastUpdateAt = now + + guard let anchor = rootAnchor else { return } + + // Mark fresh pids (MediaPipe pose + ARKit/USB skeleton) + for pid in frames.keys { lastSeenAt[pid] = now } + for pid in lastArkit.keys { lastSeenAt[pid] = now } + // GC stale persons + let cutoff = now - Self.retainSec + for (pid, p) in persons where (lastSeenAt[pid] ?? 0) < cutoff { + p.root.removeFromParent() + persons.removeValue(forKey: pid) + lastSeenAt.removeValue(forKey: pid) + } + + for (pid, frame) in frames { + let entities = persons[pid] ?? makePerson(pid: pid, parent: anchor) + persons[pid] = entities + apply(frame: frame, pid: pid, to: entities) + } + + // Ensure entity trees exist for ARKit/USB-only pids, then + // draw their 91 joint markers. + for pid in lastArkit.keys where persons[pid] == nil { + persons[pid] = makePerson(pid: pid, parent: anchor) + } + applyArkit() + } + + /// Draw the 91-joint ARKit/USB skeletons as yellow joint markers. + /// ARKit joints are world-space metric; convert to RealityKit + /// space (x, y, z) -> (x, -y, -z) like the MediaPipe path. + private func applyArkit() { + for (pid, entities) in persons { + guard let frame = lastArkit[pid] else { + for m in entities.arkitMarkers { m.isEnabled = false } + continue + } + let n = min(entities.arkitMarkers.count, + frame.joints.count, frame.hasJoint.count) + for i in 0..(j.x, -j.y, -j.z) + marker.isEnabled = true + } else { + marker.isEnabled = false + } + } + for i in n..](repeating: .zero, count: 33) + var valid = [Bool](repeating: false, count: 33) + for i in 0..<33 { + let k = frame.kps[i] + let visible = frame.hasPoint[i] && k.w >= Self.minConfidence + valid[i] = visible + // Mediapipe (x right, y down, z forward) -> RK (x right, y up, z back) + rk[i] = SIMD3(k.x, -k.y, -k.z) + } + + // Joints: position spheres and toggle visibility. + for i in 0..<33 { + let joint = entities.joints[i] + if valid[i] { + joint.transform.translation = rk[i] + joint.isEnabled = true + } else { + joint.isEnabled = false + } + } + + // Bones: orient + scale length between endpoints. + for (bIdx, (a, b, _)) in Self.POSE_CONNECTIONS.enumerated() { + let bone = entities.bones[bIdx] + if !valid[a] || !valid[b] { + bone.isEnabled = false + continue + } + let pa = rk[a] + let pb = rk[b] + let delta = pb - pa + let len = simd_length(delta) + if len < 1e-5 { + bone.isEnabled = false + continue + } + let mid = (pa + pb) * 0.5 + // Bone mesh is a cylinder of height=1 along +Y. Rotate +Y + // onto the (b-a) direction. + let dir = delta / len + let yAxis = SIMD3(0, 1, 0) + let dot = simd_dot(yAxis, dir) + let rot: simd_quatf + if dot > 0.9999 { + rot = simd_quatf(angle: 0, axis: SIMD3(0, 1, 0)) + } else if dot < -0.9999 { + rot = simd_quatf(angle: .pi, axis: SIMD3(1, 0, 0)) + } else { + let axis = simd_normalize(simd_cross(yAxis, dir)) + let angle = acos(dot) + rot = simd_quatf(angle: angle, axis: axis) + } + bone.transform.translation = mid + bone.transform.rotation = rot + // Scale length only on Y, keep XZ at 1 to preserve radius. + bone.transform.scale = SIMD3(1, len, 1) + bone.isEnabled = true + } + + // --- Face landmarks (68) anchored on nose joint rk[0] --- + applyFace(pid: pid, rk: rk, valid: valid, to: entities) + + // --- Hand landmarks (21 left + 21 right) anchored on wrists --- + applyHands(rk: rk, valid: valid, to: entities) + } + + private func applyFace(pid: Int, + rk: [SIMD3], + valid: [Bool], + to entities: PersonEntities) { + guard let face = lastFace[pid] else { + for j in entities.faceJoints { j.isEnabled = false } + return + } + // Head width in 3D : distance between ears (rk[7] left ear, + // rk[8] right ear). Fall back to a sane default if missing. + let headWidth3D: Float + if valid[7] && valid[8] { + headWidth3D = max(0.10, simd_length(rk[7] - rk[8])) + } else { + headWidth3D = 0.18 + } + // Compute 2D bbox width of face points + centroid. + var minX: Float = .infinity, maxX: Float = -.infinity + var sumX: Float = 0, sumY: Float = 0 + var n: Float = 0 + for i in 0..<68 where face.hasPoint[i] { + let p = face.points[i] + if p.x < minX { minX = p.x } + if p.x > maxX { maxX = p.x } + sumX += p.x + sumY += p.y + n += 1 + } + let nose = valid[0] ? rk[0] : SIMD3(0, 0, 0) + guard n > 4, maxX > minX else { + for j in entities.faceJoints { j.isEnabled = false } + return + } + let face2DWidth = max(maxX - minX, 1e-4) + let scale = headWidth3D / face2DWidth + let cx = sumX / n + let cy = sumY / n + for i in 0..<68 { + let j = entities.faceJoints[i] + if face.hasPoint[i] { + let dx = face.points[i].x - cx + let dy = face.points[i].y - cy + // Flip y : image y down -> RK y up. +z to push in front of nose. + j.transform.translation = nose + SIMD3( + dx * scale, -dy * scale, Self.faceForwardOffset) + j.isEnabled = true + } else { + j.isEnabled = false + } + } + } + + private func applyHands(rk: [SIMD3], + valid: [Bool], + to entities: PersonEntities) { + // Disable by default ; re-enable per side if a matching frame found. + for j in entities.leftHandJoints { j.isEnabled = false } + for j in entities.rightHandJoints { j.isEnabled = false } + + // Iterate cached hand frames (keyed by pid in OSC ; here we route + // purely by `side` since the python emits side=0/1 explicitly). + for (_, hand) in lastHands { + let isLeft = (hand.side == 0) + let wristIdx = isLeft ? 15 : 16 + guard valid[wristIdx] else { continue } + let wrist = rk[wristIdx] + let target = isLeft + ? entities.leftHandJoints + : entities.rightHandJoints + + // Centroid of valid hand points. + var sumX: Float = 0, sumY: Float = 0, n: Float = 0 + for i in 0..<21 where hand.hasPoint[i] { + sumX += hand.points[i].x + sumY += hand.points[i].y + n += 1 + } + guard n > 2 else { continue } + let cx = sumX / n + let cy = sumY / n + let scale = Self.handScale3D + for i in 0..<21 { + let j = target[i] + if hand.hasPoint[i] { + let dx = hand.points[i].x - cx + let dy = hand.points[i].y - cy + j.transform.translation = wrist + SIMD3( + dx * scale, -dy * scale, 0) + j.isEnabled = true + } else { + j.isEnabled = false + } + } + } + } + + // MARK: - Construction + + private func makePerson(pid: Int, parent: Entity) -> PersonEntities { + let root = Entity() + parent.addChild(root) + + // Joint sphere mesh shared across joints (cheap to reuse). + let sphereMesh = MeshResource.generateSphere( + radius: Self.jointRadius) + let jointMat = SimpleMaterial( + color: .white, roughness: 0.6, isMetallic: false) + var joints: [ModelEntity] = [] + joints.reserveCapacity(33) + for _ in 0..<33 { + let e = ModelEntity(mesh: sphereMesh, materials: [jointMat]) + e.isEnabled = false + root.addChild(e) + joints.append(e) + } + + // One cylinder per bone (height=1, scaled at runtime). + let cylMesh = MeshResource.generateCylinder( + height: 1.0, radius: Self.boneRadius) + var bones: [ModelEntity] = [] + bones.reserveCapacity(Self.POSE_CONNECTIONS.count) + for (_, _, chain) in Self.POSE_CONNECTIONS { + let mat = SimpleMaterial( + color: chain.color, roughness: 0.6, isMetallic: false) + let e = ModelEntity(mesh: cylMesh, materials: [mat]) + e.isEnabled = false + root.addChild(e) + bones.append(e) + } + // Face sub-spheres (68 dlib landmarks, neutral light grey). + let faceMesh = MeshResource.generateSphere( + radius: Self.faceJointRadius) + let faceMat = SimpleMaterial( + color: NSColor(white: 0.85, alpha: 1.0), + roughness: 0.7, isMetallic: false) + var faceJoints: [ModelEntity] = [] + faceJoints.reserveCapacity(68) + for _ in 0..<68 { + let e = ModelEntity(mesh: faceMesh, materials: [faceMat]) + e.isEnabled = false + root.addChild(e) + faceJoints.append(e) + } + + // Hand sub-spheres : left=cyan, right=magenta, 21 each. + let handMesh = MeshResource.generateSphere( + radius: Self.handJointRadius) + let leftMat = SimpleMaterial( + color: .systemCyan, roughness: 0.6, isMetallic: false) + let rightMat = SimpleMaterial( + color: .systemPink, roughness: 0.6, isMetallic: false) + var leftHand: [ModelEntity] = [] + var rightHand: [ModelEntity] = [] + leftHand.reserveCapacity(21) + rightHand.reserveCapacity(21) + for _ in 0..<21 { + let el = ModelEntity(mesh: handMesh, materials: [leftMat]) + el.isEnabled = false + root.addChild(el) + leftHand.append(el) + let er = ModelEntity(mesh: handMesh, materials: [rightMat]) + er.isEnabled = false + root.addChild(er) + rightHand.append(er) + } + // ARKit/USB skeleton: 91 yellow joint markers. + let arkitMat = SimpleMaterial( + color: .systemYellow, roughness: 0.6, isMetallic: false) + var arkitMarkers: [ModelEntity] = [] + arkitMarkers.reserveCapacity(91) + for _ in 0..<91 { + let e = ModelEntity(mesh: sphereMesh, materials: [arkitMat]) + e.isEnabled = false + root.addChild(e) + arkitMarkers.append(e) + } + NSLog("Skeleton3DRenderer: spawned pid=%d (%d bones, +91 arkit)", + pid, bones.count) + return PersonEntities(root: root, + joints: joints, + bones: bones, + faceJoints: faceJoints, + leftHandJoints: leftHand, + rightHandJoints: rightHand, + arkitMarkers: arkitMarkers) + } +} diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift similarity index 100% rename from launcher/AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift rename to launcher/_archive-AV-Live-Body/Sources/AVLiveBody/SkeletonOverlay.swift diff --git a/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBClient.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBClient.swift new file mode 100644 index 0000000..443223f --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBClient.swift @@ -0,0 +1,135 @@ +import Foundation +import Darwin + +/// Transport abstraction over the usbmuxd Unix socket. The real +/// implementation wraps a `socket(AF_UNIX)`; tests inject a mock. +protocol MuxTransport { + func send(_ data: Data) + func receivePacket() -> Data? + func close() +} + +/// usbmux client: device discovery + connect-to-port. After a +/// successful `connect`, the same transport carries the raw tunneled +/// byte stream from the device. +final class USBClient { + private let transport: MuxTransport + private var tag: UInt32 = 0 + + init(transport: MuxTransport) { + self.transport = transport + } + + func listDevices() -> [Int] { + tag += 1 + transport.send(USBMuxProtocol.encode( + plist: ["MessageType": "ListDevices"], tag: tag)) + guard let reply = transport.receivePacket(), + let plist = USBMuxProtocol.decode(reply), + let list = plist["DeviceList"] as? [[String: Any]] + else { return [] } + return list.compactMap { $0["DeviceID"] as? Int } + } + + /// Returns true once the transport is tunneled to `port` on the + /// device. usbmux wants the TCP port in big-endian order. + func connect(deviceID: Int, port: UInt16) -> Bool { + tag += 1 + let swapped = Int((port << 8) | (port >> 8)) + transport.send(USBMuxProtocol.encode(plist: [ + "MessageType": "Connect", + "DeviceID": deviceID, + "PortNumber": swapped, + ], tag: tag)) + guard let reply = transport.receivePacket(), + let plist = USBMuxProtocol.decode(reply), + let number = plist["Number"] as? Int + else { return false } + return number == 0 + } +} + +/// Production transport: blocking AF_UNIX socket to usbmuxd. +final class UnixMuxTransport: MuxTransport { + private var fd: Int32 = -1 + + init?(path: String = "/var/run/usbmuxd") { + fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { return nil } + var addr = sockaddr_un() + addr.sun_family = sa_family_t(AF_UNIX) + precondition(path.utf8.count < 104, + "usbmuxd socket path exceeds sun_path limit") + _ = path.withCString { src in + withUnsafeMutablePointer(to: &addr.sun_path) { + $0.withMemoryRebound(to: CChar.self, capacity: 104) { + strcpy($0, src) + } + } + } + let size = socklen_t(MemoryLayout.size) + let ok = withUnsafePointer(to: &addr) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.connect(fd, $0, size) + } + } + if ok != 0 { Darwin.close(fd); return nil } + } + + func send(_ data: Data) { + guard fd >= 0 else { return } + data.withUnsafeBytes { buf in + guard let base = buf.baseAddress else { return } + var off = 0 + while off < data.count { + let w = Darwin.write(fd, base.advanced(by: off), + data.count - off) + if w <= 0 { + if w < 0 && errno == EINTR { continue } + break + } + off += w + } + } + } + + /// Read one usbmux packet: 4-byte LE length prefix then body. + func receivePacket() -> Data? { + guard let head = readN(4) else { return nil } + guard let len = USBMuxProtocol.readLE32(head, 0) else { return nil } + let total = Int(len) + guard total >= 16, let rest = readN(total - 4) else { return nil } + return head + rest + } + + /// Read raw tunneled bytes after a successful Connect. + func readStream(max: Int = 65536) -> Data? { + readN(max, exact: false) + } + + private func readN(_ n: Int, exact: Bool = true) -> Data? { + var buf = [UInt8](repeating: 0, count: n) + var got = 0 + while got < n { + let r = buf.withUnsafeMutableBytes { + Darwin.read(fd, $0.baseAddress!.advanced(by: got), n - got) + } + if r < 0 { + if errno == EINTR { continue } + return got > 0 && !exact ? Data(buf[0.. 0 && !exact ? Data(buf[0..= 0 { Darwin.close(fd); fd = -1 } + } +} diff --git a/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBMuxProtocol.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBMuxProtocol.swift new file mode 100644 index 0000000..40c17f6 --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBMuxProtocol.swift @@ -0,0 +1,38 @@ +import Foundation + +/// Codec for the usbmuxd request/response protocol. 16-byte +/// little-endian header (length, version=1, message=8, tag) then an +/// XML property list. +enum USBMuxProtocol { + static func encode(plist: [String: Any], tag: UInt32) -> Data { + let body = (try? PropertyListSerialization.data( + fromPropertyList: plist, format: .xml, options: 0)) + ?? Data() + var d = Data() + appendLE32(&d, UInt32(16 + body.count)) // length + appendLE32(&d, 1) // version + appendLE32(&d, 8) // message: plist + appendLE32(&d, tag) + d.append(body) + return d + } + + static func decode(_ packet: Data) -> [String: Any]? { + guard packet.count >= 16 else { return nil } + let body = packet.dropFirst(16) + return (try? PropertyListSerialization.propertyList( + from: body, options: [], format: nil)) as? [String: Any] + } + + static func appendLE32(_ d: inout Data, _ v: UInt32) { + for i in 0..<4 { d.append(UInt8((v >> (8 * i)) & 0xFF)) } + } + + static func readLE32(_ d: Data, _ offset: Int) -> UInt32? { + guard offset >= 0, d.count >= offset + 4 else { return nil } + let b = [UInt8](d) + var v: UInt32 = 0 + for i in 0..<4 { v |= UInt32(b[offset + i]) << (8 * i) } + return v + } +} diff --git a/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift new file mode 100644 index 0000000..a59e7ad --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/USBSkeletonConsumer.swift @@ -0,0 +1,112 @@ +import AVLiveWire +import Combine +import Foundation + +/// Connects to the tethered iPhone over USB (usbmuxd), demuxes the +/// AVLiveWire stream, and republishes skeleton frames (as the existing +/// 91-joint `ArkitOSCListener.ArkitBodyFrame`) plus video payloads. +/// The blocking transport runs on a dedicated background thread; only +/// `@Published` writes hop to the main queue. +final class USBSkeletonConsumer: ObservableObject { + /// 91-joint body frames keyed by pid — same shape + /// `Skeleton3DRenderer` already consumes from `ArkitOSCListener`. + @Published var bodies: [Int: ArkitOSCListener.ArkitBodyFrame] = [:] + @Published var connected = false + + /// Called (on the main queue) for every decoded `.video` frame. + var onVideo: ((VideoPayload) -> Void)? + + /// TCP port the iPhone `USBServer` listens on (must match the iOS + /// app's `USBServer.port`). + static let devicePort: UInt16 = 7000 + + private let stateLock = NSLock() + private var running = false + private var thread: Thread? + + private var isRunning: Bool { + stateLock.lock(); defer { stateLock.unlock() } + return running + } + + func start() { + stateLock.lock() + if running { stateLock.unlock(); return } + running = true + stateLock.unlock() + let t = Thread { [weak self] in self?.loop() } + t.name = "cc.avlive.usbconsumer" + t.start() + thread = t + } + + func stop() { + stateLock.lock(); running = false; stateLock.unlock() + } + + /// Pure mapping `SkeletonPayload` -> `ArkitBodyFrame`. Static so it + /// is unit-testable without a transport. + static func bodyFrame(pid: Int, from p: SkeletonPayload) + -> ArkitOSCListener.ArkitBodyFrame { + var f = ArkitOSCListener.ArkitBodyFrame() + f.pid = pid + f.joints = p.joints + f.hasJoint = p.valid + f.seenAt = CFAbsoluteTimeGetCurrent() + return f + } + + // MARK: - Background read loop + + private func loop() { + while isRunning { + guard let transport = UnixMuxTransport() else { + Thread.sleep(forTimeInterval: 1.0); continue + } + let client = USBClient(transport: transport) + guard let dev = client.listDevices().first, + client.connect(deviceID: dev, + port: Self.devicePort) else { + transport.close() + Thread.sleep(forTimeInterval: 1.0); continue + } + publishConnected(true) + var demux = StreamDemuxer() + while isRunning { + guard let chunk = transport.readStream(), + !chunk.isEmpty else { break } + for frame in demux.feed(chunk) { route(frame) } + } + transport.close() + publishConnected(false) + if isRunning { Thread.sleep(forTimeInterval: 1.0) } + } + } + + private func route(_ frame: StreamDemuxer.Frame) { + switch frame.header.tag { + case .skeleton: + guard let payload = + SkeletonPayload(decoding: frame.payload) else { return } + let pid = Int(frame.header.pid) + let body = Self.bodyFrame(pid: pid, from: payload) + DispatchQueue.main.async { [weak self] in + self?.bodies[pid] = body + } + case .video: + guard let payload = + VideoPayload(decoding: frame.payload) else { return } + DispatchQueue.main.async { [weak self] in + self?.onVideo?(payload) + } + case .meta: + break + } + } + + private func publishConnected(_ value: Bool) { + DispatchQueue.main.async { [weak self] in + self?.connected = value + } + } +} diff --git a/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/VideoDecoder.swift b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/VideoDecoder.swift new file mode 100644 index 0000000..5e18be8 --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Sources/AVLiveBody/VideoDecoder.swift @@ -0,0 +1,184 @@ +import AVLiveWire +import CoreMedia +import CoreVideo +import Foundation +import VideoToolbox + +/// HEVC decoder. Feed `VideoPayload`s in; receive `CVPixelBuffer`s via +/// `onFrame`. Keyframe payloads must carry the VPS/SPS/PPS parameter +/// sets prepended as 4-byte-length-prefixed NAL units (the layout the +/// iOS `VideoEncoder` emits); the decoder (re)builds its format +/// description from those. +final class VideoDecoder { + var onFrame: ((CVPixelBuffer) -> Void)? + + private var session: VTDecompressionSession? + private var formatDesc: CMVideoFormatDescription? + + /// Decode one access unit. + func decode(_ payload: VideoPayload) { + var au = payload.data + if payload.isKeyframe { + let (params, rest) = Self.splitParameterSets(au) + if !params.isEmpty { + rebuildFormat(params) + } + au = rest + } + guard let fmt = formatDesc, !au.isEmpty else { return } + if session == nil { makeSession(fmt) } + guard let session, let block = Self.blockBuffer(au) else { + return + } + var sample: CMSampleBuffer? + var sampleSize = au.count + guard CMSampleBufferCreateReady( + allocator: kCFAllocatorDefault, dataBuffer: block, + formatDescription: fmt, sampleCount: 1, + sampleTimingEntryCount: 0, sampleTimingArray: nil, + sampleSizeEntryCount: 1, sampleSizeArray: &sampleSize, + sampleBufferOut: &sample) == noErr, let sample else { + return + } + VTDecompressionSessionDecodeFrame( + session, sampleBuffer: sample, flags: [], + infoFlagsOut: nil) { [weak self] status, _, image, _, _ in + guard status == noErr, let image else { return } + self?.onFrame?(image) + } + } + + func stop() { + if let session { VTDecompressionSessionInvalidate(session) } + session = nil + formatDesc = nil + } + + deinit { stop() } + + // MARK: - Helpers + + /// Leading 4-byte-length-prefixed NAL units of HEVC parameter-set + /// type (VPS=32, SPS=33, PPS=34) are split from the frame data. + /// Returns (parameterSetData, frameData). + private static func splitParameterSets(_ data: Data) + -> (Data, Data) { + let bytes = [UInt8](data) + var offset = 0 + var paramEnd = 0 + while offset + 4 <= bytes.count { + let len = (Int(bytes[offset]) << 24) + | (Int(bytes[offset + 1]) << 16) + | (Int(bytes[offset + 2]) << 8) + | Int(bytes[offset + 3]) + let nalStart = offset + 4 + guard len > 0, nalStart + len <= bytes.count else { break } + let nalType = (Int(bytes[nalStart]) >> 1) & 0x3F + if nalType == 32 || nalType == 33 || nalType == 34 { + offset = nalStart + len + paramEnd = offset + } else { + break + } + } + return (data.prefix(paramEnd), + data.suffix(from: data.startIndex + .advanced(by: paramEnd))) + } + + private func rebuildFormat(_ paramData: Data) { + var sets: [[UInt8]] = [] + let bytes = [UInt8](paramData) + var offset = 0 + while offset + 4 <= bytes.count { + let len = (Int(bytes[offset]) << 24) + | (Int(bytes[offset + 1]) << 16) + | (Int(bytes[offset + 2]) << 8) + | Int(bytes[offset + 3]) + let start = offset + 4 + guard len > 0, start + len <= bytes.count else { break } + sets.append(Array(bytes[start..= 3 else { return } + var fmt: CMFormatDescription? + let status = withParameterSetPointers(sets) { pBuf, sBuf in + CMVideoFormatDescriptionCreateFromHEVCParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: sets.count, + parameterSetPointers: pBuf, + parameterSetSizes: sBuf, + nalUnitHeaderLength: 4, extensions: nil, + formatDescriptionOut: &fmt) + } + if status == noErr, let fmt { + formatDesc = fmt + if let session { VTDecompressionSessionInvalidate(session) } + session = nil + } + } + + /// Build the C-style parallel arrays of parameter-set pointers and + /// sizes that `CMVideoFormatDescriptionCreateFromHEVCParameterSets` + /// requires, keeping the backing storage alive for the call. + private func withParameterSetPointers( + _ sets: [[UInt8]], + _ body: (UnsafePointer>, + UnsafePointer) -> OSStatus) -> OSStatus { + func recurse(_ index: Int, + _ ptrs: inout [UnsafePointer], + _ sizes: inout [Int]) -> OSStatus { + if index == sets.count { + return ptrs.withUnsafeBufferPointer { pBuf in + sizes.withUnsafeBufferPointer { sBuf in + body(pBuf.baseAddress!, sBuf.baseAddress!) + } + } + } + return sets[index].withUnsafeBufferPointer { buf in + ptrs.append(buf.baseAddress!) + sizes.append(buf.count) + return recurse(index + 1, &ptrs, &sizes) + } + } + var ptrs: [UnsafePointer] = [] + var sizes: [Int] = [] + ptrs.reserveCapacity(sets.count) + sizes.reserveCapacity(sets.count) + return recurse(0, &ptrs, &sizes) + } + + private func makeSession(_ fmt: CMVideoFormatDescription) { + let attrs: [CFString: Any] = [ + kCVPixelBufferPixelFormatTypeKey: + kCVPixelFormatType_32BGRA, + ] + VTDecompressionSessionCreate( + allocator: kCFAllocatorDefault, formatDescription: fmt, + decoderSpecification: nil, + imageBufferAttributes: attrs as CFDictionary, + outputCallback: nil, decompressionSessionOut: &session) + } + + private static func blockBuffer(_ data: Data) -> CMBlockBuffer? { + var block: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, memoryBlock: nil, + blockLength: data.count, + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, offsetToData: 0, + dataLength: data.count, flags: 0, + blockBufferOut: &block) == noErr, let block else { + return nil + } + var ok = false + data.withUnsafeBytes { raw in + if let base = raw.baseAddress, + CMBlockBufferReplaceDataBytes( + with: base, blockBuffer: block, + offsetIntoDestination: 0, + dataLength: data.count) == noErr { ok = true } + } + return ok ? block : nil + } +} diff --git a/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBClientTests.swift b/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBClientTests.swift new file mode 100644 index 0000000..c663931 --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBClientTests.swift @@ -0,0 +1,49 @@ +import XCTest +@testable import AVLiveBody + +/// In-memory stand-in for the usbmuxd Unix socket. +final class MockMuxTransport: MuxTransport { + var sent: [Data] = [] + var canned: [Data] = [] + func send(_ data: Data) { sent.append(data) } + func receivePacket() -> Data? { + canned.isEmpty ? nil : canned.removeFirst() + } + func close() {} +} + +final class USBClientTests: XCTestCase { + func testListDevicesParsesDeviceIDs() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode(plist: [ + "DeviceList": [ + ["DeviceID": 42, + "Properties": ["ConnectionType": "USB"]], + ]], tag: 0)] + let client = USBClient(transport: mock) + let devices = client.listDevices() + XCTAssertEqual(devices, [42]) + } + + func testConnectSendsConnectRequest() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode( + plist: ["MessageType": "Result", "Number": 0], tag: 0)] + let client = USBClient(transport: mock) + let ok = client.connect(deviceID: 42, port: 7000) + XCTAssertTrue(ok) + let req = USBMuxProtocol.decode(mock.sent.last!) + XCTAssertEqual(req?["MessageType"] as? String, "Connect") + XCTAssertEqual(req?["DeviceID"] as? Int, 42) + XCTAssertEqual(req?["PortNumber"] as? Int, + Int((UInt16(7000) << 8) | (UInt16(7000) >> 8))) + } + + func testConnectFailsOnNonZeroResult() { + let mock = MockMuxTransport() + mock.canned = [USBMuxProtocol.encode( + plist: ["MessageType": "Result", "Number": 3], tag: 0)] + let client = USBClient(transport: mock) + XCTAssertFalse(client.connect(deviceID: 1, port: 7000)) + } +} diff --git a/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBMuxProtocolTests.swift b/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBMuxProtocolTests.swift new file mode 100644 index 0000000..7fbf2c9 --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBMuxProtocolTests.swift @@ -0,0 +1,27 @@ +import XCTest +@testable import AVLiveBody + +final class USBMuxProtocolTests: XCTestCase { + func testEncodeWrapsPlistWith16ByteHeader() { + let body: [String: Any] = ["MessageType": "ListDevices"] + let packet = USBMuxProtocol.encode(plist: body, tag: 3) + XCTAssertGreaterThan(packet.count, 16) + XCTAssertEqual(USBMuxProtocol.readLE32(packet, 0).map(Int.init), + packet.count) + XCTAssertEqual(USBMuxProtocol.readLE32(packet, 4), 1) + XCTAssertEqual(USBMuxProtocol.readLE32(packet, 8), 8) + XCTAssertEqual(USBMuxProtocol.readLE32(packet, 12), 3) + } + + func testDecodeRoundTrip() { + let packet = USBMuxProtocol.encode( + plist: ["MessageType": "Result", "Number": 0], tag: 1) + let decoded = USBMuxProtocol.decode(packet) + XCTAssertEqual(decoded?["MessageType"] as? String, "Result") + XCTAssertEqual(decoded?["Number"] as? Int, 0) + } + + func testDecodeRejectsShortPacket() { + XCTAssertNil(USBMuxProtocol.decode(Data([0, 1, 2]))) + } +} diff --git a/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBSkeletonConsumerTests.swift b/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBSkeletonConsumerTests.swift new file mode 100644 index 0000000..d4726cf --- /dev/null +++ b/launcher/_archive-AV-Live-Body/Tests/AVLiveBodyTests/USBSkeletonConsumerTests.swift @@ -0,0 +1,21 @@ +import XCTest +import AVLiveWire +@testable import AVLiveBody + +final class USBSkeletonConsumerTests: XCTestCase { + func testSkeletonPayloadMapsToBodyFrame() { + var p = SkeletonPayload() + p.joints[0] = SIMD3(1, 2, 3) + p.valid[0] = true + p.joints[90] = SIMD3(-4, 5, -6) + p.valid[90] = true + let frame = USBSkeletonConsumer.bodyFrame(pid: 7, from: p) + XCTAssertEqual(frame.pid, 7) + XCTAssertEqual(frame.joints.count, 91) + XCTAssertEqual(frame.hasJoint.count, 91) + XCTAssertEqual(frame.joints[0], SIMD3(1, 2, 3)) + XCTAssertTrue(frame.hasJoint[0]) + XCTAssertEqual(frame.joints[90], SIMD3(-4, 5, -6)) + XCTAssertFalse(frame.hasJoint[1]) + } +} diff --git a/shared/AVLiveWire/Package.swift b/shared/AVLiveWire/Package.swift new file mode 100644 index 0000000..762034b --- /dev/null +++ b/shared/AVLiveWire/Package.swift @@ -0,0 +1,14 @@ +// swift-tools-version:5.9 +import PackageDescription + +let package = Package( + name: "AVLiveWire", + platforms: [.macOS(.v13), .iOS(.v17)], + products: [ + .library(name: "AVLiveWire", targets: ["AVLiveWire"]), + ], + targets: [ + .target(name: "AVLiveWire"), + .testTarget(name: "AVLiveWireTests", dependencies: ["AVLiveWire"]), + ] +) diff --git a/shared/AVLiveWire/Sources/AVLiveWire/AVLiveWire.swift b/shared/AVLiveWire/Sources/AVLiveWire/AVLiveWire.swift new file mode 100644 index 0000000..82dc8d2 --- /dev/null +++ b/shared/AVLiveWire/Sources/AVLiveWire/AVLiveWire.swift @@ -0,0 +1,5 @@ +/// AVLiveWire — binary frame format shared by ARBodyTracker (iOS) +/// and AVLiveBody (macOS) over the USB transport. +public enum AVLiveWire { + public static let protocolVersion: UInt8 = 1 +} diff --git a/shared/AVLiveWire/Sources/AVLiveWire/FrameHeader.swift b/shared/AVLiveWire/Sources/AVLiveWire/FrameHeader.swift new file mode 100644 index 0000000..2958741 --- /dev/null +++ b/shared/AVLiveWire/Sources/AVLiveWire/FrameHeader.swift @@ -0,0 +1,73 @@ +import Foundation + +public enum FrameTag: UInt8 { + case skeleton = 1 + case video = 2 + case meta = 3 +} + +/// Fixed-size frame header. Layout (big-endian): +/// magic[4]=`AVL1` | tag u8 | pid i16 | timestamp f64 | length u32 +public struct FrameHeader: Equatable { + public static let magic: [UInt8] = [0x41, 0x56, 0x4C, 0x31] + public static let byteCount = 19 + + public var tag: FrameTag + public var pid: Int16 + public var timestamp: Double + public var length: UInt32 + + public init(tag: FrameTag, pid: Int16, + timestamp: Double, length: UInt32) { + self.tag = tag; self.pid = pid + self.timestamp = timestamp; self.length = length + } + + public func encoded() -> Data { + var d = Data(Self.magic) + d.append(tag.rawValue) + d.appendBE(UInt16(bitPattern: pid)) + d.appendBE(timestamp.bitPattern) + d.appendBE(length) + return d + } + + public init?(decoding data: Data) { + guard data.count >= Self.byteCount else { return nil } + let b = [UInt8](data.prefix(Self.byteCount)) + guard Array(b[0..<4]) == Self.magic, + let t = FrameTag(rawValue: b[4]) else { return nil } + tag = t + pid = Int16(bitPattern: UInt16(bigEndianBytes: b[5...6])) + timestamp = Double(bitPattern: UInt64(bigEndianBytes: b[7...14])) + length = UInt32(bigEndianBytes: b[15...18]) + } +} + +extension Data { + mutating func appendBE(_ v: UInt16) { + appendBE(UInt64(v), width: 2) } + mutating func appendBE(_ v: UInt32) { + appendBE(UInt64(v), width: 4) } + mutating func appendBE(_ v: UInt64, width: Int = 8) { + for i in stride(from: width - 1, through: 0, by: -1) { + append(UInt8((v >> (8 * i)) & 0xFF)) + } + } +} + +extension UInt16 { + init(bigEndianBytes s: S) where S.Element == UInt8 { + self = s.reduce(0) { ($0 << 8) | UInt16($1) } + } +} +extension UInt32 { + init(bigEndianBytes s: S) where S.Element == UInt8 { + self = s.reduce(0) { ($0 << 8) | UInt32($1) } + } +} +extension UInt64 { + init(bigEndianBytes s: S) where S.Element == UInt8 { + self = s.reduce(0) { ($0 << 8) | UInt64($1) } + } +} diff --git a/shared/AVLiveWire/Sources/AVLiveWire/StreamDemuxer.swift b/shared/AVLiveWire/Sources/AVLiveWire/StreamDemuxer.swift new file mode 100644 index 0000000..2a263bd --- /dev/null +++ b/shared/AVLiveWire/Sources/AVLiveWire/StreamDemuxer.swift @@ -0,0 +1,59 @@ +import Foundation + +public struct StreamDemuxer { + public struct Frame: Equatable { + public let header: FrameHeader + public let payload: Data + } + + /// Largest plausible payload (8 MB) — comfortably covers any HEVC + /// access unit. A header claiming more is treated as corrupt. + public static let maxPayloadLength: UInt32 = 8 * 1024 * 1024 + + private var buffer = Data() + public init() {} + + /// Append bytes; return every complete frame now available. + public mutating func feed(_ chunk: Data) -> [Frame] { + buffer.append(chunk) + var out: [Frame] = [] + while true { + guard let start = findMagic() else { + // keep at most 3 trailing bytes (partial magic) + if buffer.count > 3 { + buffer = buffer.suffix(3) + } + break + } + if start > 0 { buffer.removeFirst(start) } + guard buffer.count >= FrameHeader.byteCount, + let h = FrameHeader(decoding: buffer) else { break } + if h.length > Self.maxPayloadLength { + // Implausible length — corrupt header; skip the magic + // and resync on the next one. + buffer.removeFirst(FrameHeader.magic.count) + continue + } + let total = FrameHeader.byteCount + Int(h.length) + guard buffer.count >= total else { break } + let payloadStart = buffer.index( + buffer.startIndex, offsetBy: FrameHeader.byteCount) + let payloadEnd = buffer.index( + buffer.startIndex, offsetBy: total) + out.append(Frame(header: h, + payload: Data(buffer[payloadStart.. Int? { + let m = FrameHeader.magic + let bytes = [UInt8](buffer) + guard bytes.count >= m.count else { return nil } + for i in 0...(bytes.count - m.count) { + if Array(bytes[i..] + public var valid: [Bool] + + public init() { + joints = Array(repeating: .zero, count: Self.jointCount) + valid = Array(repeating: false, count: Self.jointCount) + } + + public func encoded() -> Data { + var d = Data(capacity: Self.byteCount) + for j in joints { + d.appendBE(j.x.bitPattern); d.appendBE(j.y.bitPattern) + d.appendBE(j.z.bitPattern) + } + for v in valid { d.append(v ? 1 : 0) } + return d + } + + public init?(decoding data: Data) { + guard data.count == Self.byteCount else { return nil } + let b = [UInt8](data) + self.init() + var o = 0 + for i in 0.. Float { + let v = Float(bitPattern: + UInt32(bigEndianBytes: b[o.. Data { + var d = Data([isKeyframe ? 1 : 0]) + d.append(data) + return d + } + + public init?(decoding data: Data) { + guard let first = data.first else { return nil } + isKeyframe = first != 0 + self.data = data.dropFirst() + } +} diff --git a/shared/AVLiveWire/Tests/AVLiveWireTests/FrameHeaderTests.swift b/shared/AVLiveWire/Tests/AVLiveWireTests/FrameHeaderTests.swift new file mode 100644 index 0000000..98ccb82 --- /dev/null +++ b/shared/AVLiveWire/Tests/AVLiveWireTests/FrameHeaderTests.swift @@ -0,0 +1,24 @@ +import XCTest +@testable import AVLiveWire + +final class FrameHeaderTests: XCTestCase { + func testRoundTrip() { + let h = FrameHeader(tag: .skeleton, pid: 7, + timestamp: 12.5, length: 1092) + let bytes = h.encoded() + XCTAssertEqual(bytes.count, FrameHeader.byteCount) + let decoded = FrameHeader(decoding: bytes) + XCTAssertEqual(decoded, h) + } + + func testRejectsBadMagic() { + var bytes = FrameHeader(tag: .video, pid: -1, + timestamp: 0, length: 0).encoded() + bytes[0] = 0x00 + XCTAssertNil(FrameHeader(decoding: bytes)) + } + + func testRejectsShortBuffer() { + XCTAssertNil(FrameHeader(decoding: Data([0x41, 0x56]))) + } +} diff --git a/shared/AVLiveWire/Tests/AVLiveWireTests/LoopbackTests.swift b/shared/AVLiveWire/Tests/AVLiveWireTests/LoopbackTests.swift new file mode 100644 index 0000000..5481e5f --- /dev/null +++ b/shared/AVLiveWire/Tests/AVLiveWireTests/LoopbackTests.swift @@ -0,0 +1,37 @@ +import XCTest +@testable import AVLiveWire + +/// Feeds an encoded frame stream through the demuxer in 7-byte +/// chunks — the worst-case fragmentation a TCP tunnel can produce. +final class LoopbackTests: XCTestCase { + func testManyFramesChunked() { + var skel = SkeletonPayload() + skel.valid[0] = true + skel.joints[0] = SIMD3(1, 1, 1) + + var stream = Data() + for i in 0..<20 { + let payload = skel.encoded() + let h = FrameHeader(tag: .skeleton, pid: Int16(i), + timestamp: Double(i), + length: UInt32(payload.count)) + stream += h.encoded() + payload + } + + var demux = StreamDemuxer() + var frames: [StreamDemuxer.Frame] = [] + var offset = 0 + while offset < stream.count { + let end = min(offset + 7, stream.count) + let lo = stream.index(stream.startIndex, offsetBy: offset) + let hi = stream.index(stream.startIndex, offsetBy: end) + frames += demux.feed(Data(stream[lo.. Data { + let h = FrameHeader(tag: tag, pid: 0, timestamp: 1, + length: UInt32(payload.count)) + return h.encoded() + payload + } + + func testSingleFrame() { + var d = StreamDemuxer() + let out = d.feed(frame(.video, Data([1, 2, 3]))) + XCTAssertEqual(out.count, 1) + XCTAssertEqual(out[0].payload, Data([1, 2, 3])) + } + + func testSplitAcrossChunks() { + var d = StreamDemuxer() + let whole = frame(.video, Data([9, 9, 9, 9, 9])) + XCTAssertTrue(d.feed(whole.prefix(10)).isEmpty) + let out = d.feed(whole.dropFirst(10)) + XCTAssertEqual(out.count, 1) + XCTAssertEqual(out[0].payload, Data([9, 9, 9, 9, 9])) + } + + func testTwoFramesOneChunk() { + var d = StreamDemuxer() + let out = d.feed(frame(.meta, Data([1])) + + frame(.video, Data([2, 2]))) + XCTAssertEqual(out.count, 2) + } + + func testResyncAfterGarbage() { + var d = StreamDemuxer() + let out = d.feed(Data([0xDE, 0xAD]) + + frame(.video, Data([7]))) + XCTAssertEqual(out.count, 1) + XCTAssertEqual(out[0].payload, Data([7])) + } + + func testPartialMagicAtBoundary() { + var d = StreamDemuxer() + XCTAssertTrue(d.feed(Data([0x41, 0x56, 0x4C])).isEmpty) + let payload = Data([42]) + let h = FrameHeader(tag: .meta, pid: 0, timestamp: 0, + length: UInt32(payload.count)) + var rest = Data([0x31]) + rest.append(h.encoded().dropFirst(4)) + rest.append(payload) + let out = d.feed(rest) + XCTAssertEqual(out.count, 1) + XCTAssertEqual(out[0].payload, payload) + } + + func testSkipsCorruptOversizedLength() { + var d = StreamDemuxer() + // Header claiming a 4 GB payload, then a valid frame after. + let bad = FrameHeader(tag: .video, pid: 0, timestamp: 0, + length: UInt32.max).encoded() + let good = frame(.video, Data([5, 5])) + let out = d.feed(bad + good) + XCTAssertEqual(out.count, 1) + XCTAssertEqual(out[0].payload, Data([5, 5])) + } +} diff --git a/shared/AVLiveWire/Tests/AVLiveWireTests/WirePayloadsTests.swift b/shared/AVLiveWire/Tests/AVLiveWireTests/WirePayloadsTests.swift new file mode 100644 index 0000000..2f35912 --- /dev/null +++ b/shared/AVLiveWire/Tests/AVLiveWireTests/WirePayloadsTests.swift @@ -0,0 +1,25 @@ +import XCTest +@testable import AVLiveWire + +final class WirePayloadsTests: XCTestCase { + func testSkeletonRoundTrip() { + var f = SkeletonPayload() + f.joints[0] = SIMD3(1, 2, 3) + f.valid[0] = true + f.joints[90] = SIMD3(-4, 5, -6) + f.valid[90] = true + let decoded = SkeletonPayload(decoding: f.encoded()) + XCTAssertEqual(decoded, f) + } + + func testSkeletonRejectsWrongSize() { + XCTAssertNil(SkeletonPayload(decoding: Data([0, 1, 2]))) + } + + func testVideoPayloadRoundTrip() { + let p = VideoPayload(isKeyframe: true, + data: Data([9, 8, 7, 6])) + let decoded = VideoPayload(decoding: p.encoded()) + XCTAssertEqual(decoded, p) + } +}