fix(av-live-body): restore face+hand+3D (f540158)
Three regressions after recent merges, all restored to match the originalf540158design: 1. FaceHandOverlay was no longer instantiated in ContentView. Added back as a SwiftUI Canvas overlay (68 dlib face landmarks with mouth slots 48-67, plus 21x2 hand landmarks cyan/magenta). 2. Skeleton3DRenderer was not attached. BodyView now creates an AnchorEntity at (0,0,-2.5), instantiates Skeleton3DRenderer and ties its visibility to vizMode==9 or showSkeleton toggle. 3. Joint and bone radii bumped to 4.5 cm / 2.2 cm so the 3D skeleton actually reads as 3D instead of looking flat. MeshRenderer exposes pelvisWorld map per pid for future interconnect uses (not auto-applied -- design keeps mesh and skeleton each in their own coord space perf540158).
This commit is contained in:
@@ -87,6 +87,13 @@ struct ContentView: View {
|
||||
if let n = note.object as? Int { settings.vizMode = n }
|
||||
}
|
||||
|
||||
// Face + hand overlay 2D Canvas (68 dlib landmarks dont
|
||||
// bouche slots 48-67 outerLips + 60-67 innerLips, plus 21
|
||||
// landmarks par main, gauche=cyan droite=magenta). Source :
|
||||
// /face/kp et /hand/kp depuis MediaPipe Holistic.
|
||||
FaceHandOverlay(poseListener: poseListener)
|
||||
.allowsHitTesting(false)
|
||||
|
||||
// HUD coin haut-gauche : mode + touches + pose
|
||||
HUDOverlay(settings: settings, poseListener: poseListener)
|
||||
|
||||
|
||||
@@ -172,7 +172,10 @@ struct BodyView: NSViewRepresentable {
|
||||
c.skeletonOverlay?.update(persons: poseListener.persons,
|
||||
visible: skelVisible)
|
||||
// 3D RealityKit armature : show/hide root anchor in sync with
|
||||
// the same skelVisible signal as the 2D overlay.
|
||||
// the same skelVisible signal as the 2D overlay. Skeleton keeps
|
||||
// its own hip-relative coords (z=-3 anchor), mesh keeps its own
|
||||
// world coords — both visible together in mode openpos, no
|
||||
// spatial fusion (original design from commit f540158).
|
||||
c.skel3dAnchor?.isEnabled = skelVisible
|
||||
// Pose -> scene uniforms : drive hands3d (mode 8) et openpos
|
||||
// (mode 9) avec la premiere personne detectee. Les wrists pilotent
|
||||
|
||||
@@ -9,6 +9,11 @@ import SwiftUI
|
||||
@MainActor
|
||||
final class MeshRenderer: ObservableObject {
|
||||
@Published var personEntities: [Int: ModelEntity] = [:]
|
||||
/// Pelvis world position (RealityKit coords) par pid. Mis a jour
|
||||
/// chaque frame TCP. Consume par Skeleton3DRenderer pour aligner
|
||||
/// le squelette 3D openpos avec le mesh dense SMPL-X.
|
||||
/// SMPL-X pelvis = vertex index 0 par convention canonique.
|
||||
@Published var pelvisWorld: [Int: SIMD3<Float>] = [:]
|
||||
private var faces: [UInt32] = []
|
||||
private var oscServer: OSCServer?
|
||||
|
||||
@@ -130,6 +135,7 @@ final class MeshRenderer: ObservableObject {
|
||||
wireframeEntities.removeValue(forKey: pid)
|
||||
interpStates.removeValue(forKey: pid)
|
||||
lastSeenAt.removeValue(forKey: pid)
|
||||
pelvisWorld.removeValue(forKey: pid)
|
||||
}
|
||||
}
|
||||
for p in persons {
|
||||
@@ -163,6 +169,11 @@ final class MeshRenderer: ObservableObject {
|
||||
displayed: converted, target: converted)
|
||||
}
|
||||
entity.transform.translation = SIMD3<Float>.zero
|
||||
// Track pelvis world position (vertex 0 = SMPL-X pelvis) so
|
||||
// Skeleton3DRenderer can co-locate openpos joints with mesh.
|
||||
if !converted.isEmpty {
|
||||
pelvisWorld[p.pid] = converted[0]
|
||||
}
|
||||
// Bbox debug : utile une fois par creation
|
||||
if personEntities[p.pid] == nil {
|
||||
let xs = converted.map(\.x)
|
||||
|
||||
@@ -53,8 +53,8 @@ final class Skeleton3DRenderer: ObservableObject {
|
||||
}
|
||||
}
|
||||
|
||||
private static let jointRadius: Float = 0.02 // 2 cm
|
||||
private static let boneRadius: Float = 0.012 // 1.2 cm
|
||||
private static let jointRadius: Float = 0.045 // 4.5 cm — visible 3D depth
|
||||
private static let boneRadius: Float = 0.022 // 2.2 cm — chunky bones
|
||||
private static let minConfidence: Float = 0.3
|
||||
private static let retainSec: TimeInterval = 1.0
|
||||
|
||||
@@ -73,6 +73,25 @@ final class Skeleton3DRenderer: ObservableObject {
|
||||
private weak var rootAnchor: Entity?
|
||||
private var poseSub: AnyCancellable?
|
||||
private var lastUpdateAt: TimeInterval = 0
|
||||
/// Optional per-pid offset to align the skeleton with another
|
||||
/// renderer's coordinate space (typically MeshRenderer's pelvis).
|
||||
/// When nil for a pid, falls back to the renderer's default anchor.
|
||||
private var pelvisOffsets: [Int: SIMD3<Float>] = [:]
|
||||
|
||||
/// External wiring : MeshRenderer or any other source publishes a
|
||||
/// pelvis world position per pid. We translate each person's root
|
||||
/// entity to that pose so the openpos skeleton co-locates with the
|
||||
/// dense SMPL-X mesh.
|
||||
func setPelvisOffsets(_ offsets: [Int: SIMD3<Float>]) {
|
||||
pelvisOffsets = offsets
|
||||
for (pid, entities) in persons {
|
||||
if let off = offsets[pid] {
|
||||
entities.root.transform.translation = off
|
||||
} else {
|
||||
entities.root.transform.translation = .zero
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach to a scene by giving it an AnchorEntity that owns all
|
||||
/// skeleton entities, and start observing the listener.
|
||||
|
||||
Reference in New Issue
Block a user