From 8099b79ec8b9b7eded1ce4eb4fdad3cf74cedc75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=27=C3=A9lectron=20rare?=
 <108685187+electron-rare@users.noreply.github.com>
Date: Mon, 18 May 2026 17:49:22 +0200
Subject: [PATCH] feat(av-live-body): render 91-joint USB skeleton

Complete the long-standing TODO: draw the 91 ARKit/USB
skeleton joints as yellow markers, fed from lastArkit. Spawn
entity trees for ARKit-only pids so the USB skeleton shows
without a MediaPipe pose.
---
 .../AVLiveBody/Skeleton3DRenderer.swift       | 56 +++++++++++++++++--
 1 file changed, 52 insertions(+), 4 deletions(-)

diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift
index 7c64684..f255906 100644
--- a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift
+++ b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift
@@ -75,6 +75,7 @@ final class Skeleton3DRenderer: ObservableObject {
         var faceJoints: [ModelEntity]      // 68 dlib face landmarks
         var leftHandJoints: [ModelEntity]  // 21 cyan
         var rightHandJoints: [ModelEntity] // 21 magenta
+        var arkitMarkers: [ModelEntity]    // 91 yellow ARKit/USB joints
     }
 
     private var persons: [Int: PersonEntities] = [:]
@@ -135,7 +136,6 @@ final class Skeleton3DRenderer: ObservableObject {
                     Task { @MainActor in self?.lastArkit = frames }
                 }
         }
-        // TODO: render yellow ARKit markers from lastArkit in update()
     }
 
     func detach() {
@@ -163,8 +163,9 @@ final class Skeleton3DRenderer: ObservableObject {
 
         guard let anchor = rootAnchor else { return }
 
-        // Mark fresh pids
+        // Mark fresh pids (MediaPipe pose + ARKit/USB skeleton)
         for pid in frames.keys { lastSeenAt[pid] = now }
+        for pid in lastArkit.keys { lastSeenAt[pid] = now }
         // GC stale persons
         let cutoff = now - Self.retainSec
         for (pid, p) in persons where (lastSeenAt[pid] ?? 0) < cutoff {
@@ -178,6 +179,41 @@ final class Skeleton3DRenderer: ObservableObject {
             persons[pid] = entities
             apply(frame: frame, pid: pid, to: entities)
         }
+
+        // Ensure entity trees exist for ARKit/USB-only pids, then
+        // draw their 91 joint markers.
+        for pid in lastArkit.keys where persons[pid] == nil {
+            persons[pid] = makePerson(pid: pid, parent: anchor)
+        }
+        applyArkit()
+    }
+
+    /// Draw the 91-joint ARKit/USB skeletons as yellow joint markers.
+    /// ARKit joints are world-space metric; convert to RealityKit
+    /// space (x, y, z) -> (x, -y, -z) like the MediaPipe path.
+    private func applyArkit() {
+        for (pid, entities) in persons {
+            guard let frame = lastArkit[pid] else {
+                for m in entities.arkitMarkers { m.isEnabled = false }
+                continue
+            }
+            let n = min(entities.arkitMarkers.count,
+                        frame.joints.count, frame.hasJoint.count)
+            for i in 0..<n {
+                let marker = entities.arkitMarkers[i]
+                if frame.hasJoint[i] {
+                    let j = frame.joints[i]
+                    marker.transform.translation =
+                        SIMD3<Float>(j.x, -j.y, -j.z)
+                    marker.isEnabled = true
+                } else {
+                    marker.isEnabled = false
+                }
+            }
+            for i in n..<entities.arkitMarkers.count {
+                entities.arkitMarkers[i].isEnabled = false
+            }
+        }
     }
 
     private func apply(frame: PoseOSCListener.Pose3DFrame,
@@ -415,13 +451,25 @@ final class Skeleton3DRenderer: ObservableObject {
             root.addChild(er)
             rightHand.append(er)
         }
-        NSLog("Skeleton3DRenderer: spawned pid=%d (33 joints, %d bones, 68 face, 21+21 hands)",
+        // ARKit/USB skeleton: 91 yellow joint markers.
+        let arkitMat = SimpleMaterial(
+            color: .systemYellow, roughness: 0.6, isMetallic: false)
+        var arkitMarkers: [ModelEntity] = []
+        arkitMarkers.reserveCapacity(91)
+        for _ in 0..<91 {
+            let e = ModelEntity(mesh: sphereMesh, materials: [arkitMat])
+            e.isEnabled = false
+            root.addChild(e)
+            arkitMarkers.append(e)
+        }
+        NSLog("Skeleton3DRenderer: spawned pid=%d (%d bones, +91 arkit)",
               pid, bones.count)
         return PersonEntities(root: root,
                               joints: joints,
                               bones: bones,
                               faceJoints: faceJoints,
                               leftHandJoints: leftHand,
-                              rightHandJoints: rightHand)
+                              rightHandJoints: rightHand,
+                              arkitMarkers: arkitMarkers)
     }
 }