From c4bc5c2e5ad1c691bc0211ceb11a232a81407413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=27=C3=A9lectron=20rare?= <108685187+electron-rare@users.noreply.github.com> Date: Thu, 14 May 2026 00:45:31 +0200 Subject: [PATCH] fix(av-live-body): restore face+hand+3D (f540158) Three regressions after recent merges, all restored to match the original f540158 design: 1. FaceHandOverlay was no longer instantiated in ContentView. Added back as a SwiftUI Canvas overlay (68 dlib face landmarks with mouth slots 48-67, plus 21x2 hand landmarks cyan/magenta). 2. Skeleton3DRenderer was not attached. BodyView now creates an AnchorEntity at (0,0,-2.5), instantiates Skeleton3DRenderer and ties its visibility to vizMode==9 or showSkeleton toggle. 3. Joint and bone radii bumped to 4.5 cm / 2.2 cm so the 3D skeleton actually reads as 3D instead of looking flat. MeshRenderer exposes pelvisWorld map per pid for future interconnect uses (not auto-applied -- design keeps mesh and skeleton each in their own coord space per f540158). --- .../Sources/AVLiveBody/AVLiveBodyApp.swift | 7 ++++++ .../Sources/AVLiveBody/BodyView.swift | 5 +++- .../Sources/AVLiveBody/MeshRenderer.swift | 11 +++++++++ .../AVLiveBody/Skeleton3DRenderer.swift | 23 +++++++++++++++++-- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift index 808e8cb..c04db0d 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/AVLiveBodyApp.swift @@ -87,6 +87,13 @@ struct ContentView: View { if let n = note.object as? Int { settings.vizMode = n } } + // Face + hand overlay 2D Canvas (68 dlib landmarks dont + // bouche slots 48-67 outerLips + 60-67 innerLips, plus 21 + // landmarks par main, gauche=cyan droite=magenta). Source : + // /face/kp et /hand/kp depuis MediaPipe Holistic. + FaceHandOverlay(poseListener: poseListener) + .allowsHitTesting(false) + // HUD coin haut-gauche : mode + touches + pose HUDOverlay(settings: settings, poseListener: poseListener) diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift index 2e7bf1b..8848feb 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/BodyView.swift @@ -172,7 +172,10 @@ struct BodyView: NSViewRepresentable { c.skeletonOverlay?.update(persons: poseListener.persons, visible: skelVisible) // 3D RealityKit armature : show/hide root anchor in sync with - // the same skelVisible signal as the 2D overlay. + // the same skelVisible signal as the 2D overlay. Skeleton keeps + // its own hip-relative coords (z=-3 anchor), mesh keeps its own + // world coords — both visible together in mode openpos, no + // spatial fusion (original design from commit f540158). c.skel3dAnchor?.isEnabled = skelVisible // Pose -> scene uniforms : drive hands3d (mode 8) et openpos // (mode 9) avec la premiere personne detectee. Les wrists pilotent diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift index 92f3fa9..d040c3f 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/MeshRenderer.swift @@ -9,6 +9,11 @@ import SwiftUI @MainActor final class MeshRenderer: ObservableObject { @Published var personEntities: [Int: ModelEntity] = [:] + /// Pelvis world position (RealityKit coords) par pid. Mis a jour + /// chaque frame TCP. Consume par Skeleton3DRenderer pour aligner + /// le squelette 3D openpos avec le mesh dense SMPL-X. + /// SMPL-X pelvis = vertex index 0 par convention canonique. + @Published var pelvisWorld: [Int: SIMD3] = [:] private var faces: [UInt32] = [] private var oscServer: OSCServer? @@ -130,6 +135,7 @@ final class MeshRenderer: ObservableObject { wireframeEntities.removeValue(forKey: pid) interpStates.removeValue(forKey: pid) lastSeenAt.removeValue(forKey: pid) + pelvisWorld.removeValue(forKey: pid) } } for p in persons { @@ -163,6 +169,11 @@ final class MeshRenderer: ObservableObject { displayed: converted, target: converted) } entity.transform.translation = SIMD3.zero + // Track pelvis world position (vertex 0 = SMPL-X pelvis) so + // Skeleton3DRenderer can co-locate openpos joints with mesh. + if !converted.isEmpty { + pelvisWorld[p.pid] = converted[0] + } // Bbox debug : utile une fois par creation if personEntities[p.pid] == nil { let xs = converted.map(\.x) diff --git a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift index 0a6c2f3..aef057a 100644 --- a/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift +++ b/launcher/AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift @@ -53,8 +53,8 @@ final class Skeleton3DRenderer: ObservableObject { } } - private static let jointRadius: Float = 0.02 // 2 cm - private static let boneRadius: Float = 0.012 // 1.2 cm + private static let jointRadius: Float = 0.045 // 4.5 cm — visible 3D depth + private static let boneRadius: Float = 0.022 // 2.2 cm — chunky bones private static let minConfidence: Float = 0.3 private static let retainSec: TimeInterval = 1.0 @@ -73,6 +73,25 @@ final class Skeleton3DRenderer: ObservableObject { private weak var rootAnchor: Entity? private var poseSub: AnyCancellable? private var lastUpdateAt: TimeInterval = 0 + /// Optional per-pid offset to align the skeleton with another + /// renderer's coordinate space (typically MeshRenderer's pelvis). + /// When nil for a pid, falls back to the renderer's default anchor. + private var pelvisOffsets: [Int: SIMD3] = [:] + + /// External wiring : MeshRenderer or any other source publishes a + /// pelvis world position per pid. We translate each person's root + /// entity to that pose so the openpos skeleton co-locates with the + /// dense SMPL-X mesh. + func setPelvisOffsets(_ offsets: [Int: SIMD3]) { + pelvisOffsets = offsets + for (pid, entities) in persons { + if let off = offsets[pid] { + entities.root.transform.translation = off + } else { + entities.root.transform.translation = .zero + } + } + } /// Attach to a scene by giving it an AnchorEntity that owns all /// skeleton entities, and start observing the listener.