feat(av-live): openpos 3D + DINO reid + filter
Three improvements wired end-to-end: 1. Openpos 3D skeleton visible: Skeleton3DRenderer attached to a RealityKit AnchorEntity in BodyView, toggled by showSkeleton or vizMode==9. PoseOSCListener now parses /pose3d/count and /pose3d/kp (plus restored /face/* and /hand/* paths). 2. DINO re-id (dinov2_vits14, ~9 ms ANE forward): MeshRigger combines Hungarian IoU with cosine similarity over a per-pid embedding history (deque maxlen=10), weighted by MULTIHMR_REID_ALPHA (default 0.5). Falls back to pure IoU if DINO mlpackage absent or scipy missing. state.last_frame_rgb buffer added so the rigger can crop bbox regions for embedding. 3. PoseFilterChain on pose_world_landmarks: median (anti-spike) -> Kalman constant-velocity -> 50 ms lookahead -> IK elbow/knee/ankle clamp. Configurable via POSE_FILTER env (default median+kalman+lookahead+ik). <2 ms per frame for typical 1-2 persons. Tests: 5 new in test_dino_reid.py + 6 new in test_pose_filter.py, all green. Live validated by user: skeleton spawns, mesh stays stable.
This commit is contained in:
@@ -101,6 +101,15 @@ struct BodyView: NSViewRepresentable {
|
||||
context.coordinator.sceneRenderer = scene
|
||||
context.coordinator.mtkView = mtkView
|
||||
context.coordinator.skeletonOverlay = SkeletonOverlay(parent: bodyAnchor)
|
||||
// Skeleton 3D RealityKit armature (33 spheres + 32 cylinders bones)
|
||||
// driven by /pose3d/* OSC from MediaPipe pose_world_landmarks.
|
||||
// Visible quand toggle showSkeleton ou vizMode==9 (openpos).
|
||||
let skel3dAnchor = AnchorEntity(world: SIMD3<Float>(0, 0, -2.5))
|
||||
arView.scene.addAnchor(skel3dAnchor)
|
||||
let skel3d = Skeleton3DRenderer()
|
||||
skel3d.attach(to: skel3dAnchor, listener: poseListener)
|
||||
context.coordinator.skel3dAnchor = skel3dAnchor
|
||||
context.coordinator.skel3d = skel3d
|
||||
context.coordinator.keyLight = key
|
||||
context.coordinator.fillLight = fill
|
||||
context.coordinator.rimLight = rim
|
||||
@@ -162,6 +171,9 @@ struct BodyView: NSViewRepresentable {
|
||||
let skelVisible = settings.vizMode == 9 || settings.showSkeleton
|
||||
c.skeletonOverlay?.update(persons: poseListener.persons,
|
||||
visible: skelVisible)
|
||||
// 3D RealityKit armature : show/hide root anchor in sync with
|
||||
// the same skelVisible signal as the 2D overlay.
|
||||
c.skel3dAnchor?.isEnabled = skelVisible
|
||||
// Pose -> scene uniforms : drive hands3d (mode 8) et openpos
|
||||
// (mode 9) avec la premiere personne detectee. Les wrists pilotent
|
||||
// hand_l/r ; pose_count alimente bg_fragment.
|
||||
@@ -211,6 +223,8 @@ struct BodyView: NSViewRepresentable {
|
||||
var sceneRenderer: SceneRenderer?
|
||||
var mtkView: MTKView?
|
||||
var skeletonOverlay: SkeletonOverlay?
|
||||
var skel3dAnchor: AnchorEntity?
|
||||
var skel3d: Skeleton3DRenderer?
|
||||
var kbMonitor: Any?
|
||||
|
||||
deinit {
|
||||
|
||||
@@ -26,8 +26,39 @@ final class PoseOSCListener: ObservableObject {
|
||||
var seenAt: TimeInterval = 0
|
||||
}
|
||||
|
||||
/// MediaPipe pose_world_landmarks : 33 keypoints in meters, hip-rel.
|
||||
/// MediaPipe convention : x=right, y=down, z=forward (away from cam).
|
||||
struct Pose3DFrame: Equatable {
|
||||
var pid: Int = -1
|
||||
var kps: [SIMD4<Float>] = Array(repeating: .zero, count: 33)
|
||||
var hasPoint: [Bool] = Array(repeating: false, count: 33)
|
||||
var seenAt: TimeInterval = 0
|
||||
}
|
||||
|
||||
/// 68 dlib-style facial landmarks (x,y normalises 0..1).
|
||||
struct FaceFrame: Equatable {
|
||||
var points: [SIMD2<Float>] = Array(repeating: .zero, count: 68)
|
||||
var hasPoint: [Bool] = Array(repeating: false, count: 68)
|
||||
var seenAt: TimeInterval = 0
|
||||
}
|
||||
|
||||
/// 21 MediaPipe hand landmarks per detected hand.
|
||||
struct HandFrame: Equatable {
|
||||
var side: Int = 0
|
||||
var points: [SIMD2<Float>] = Array(repeating: .zero, count: 21)
|
||||
var hasPoint: [Bool] = Array(repeating: false, count: 21)
|
||||
var seenAt: TimeInterval = 0
|
||||
}
|
||||
|
||||
@Published var persons: [Int: PoseFrame] = [:]
|
||||
@Published var count: Int = 0
|
||||
@Published var body3d: [Int: Pose3DFrame] = [:]
|
||||
@Published var body3dCount: Int = 0
|
||||
@Published var faces: [Int: FaceFrame] = [:]
|
||||
@Published var hands: [Int: HandFrame] = [:]
|
||||
@Published var faceCount: Int = 0
|
||||
@Published var handCountLeft: Int = 0
|
||||
@Published var handCountRight: Int = 0
|
||||
|
||||
private var listener: NWListener?
|
||||
|
||||
@@ -148,13 +179,77 @@ final class PoseOSCListener: ObservableObject {
|
||||
p.skeleton = skel
|
||||
p.seenAt = CFAbsoluteTimeGetCurrent()
|
||||
persons[Int(pid)] = p
|
||||
case "/face/count":
|
||||
if let n = args.first as? Int32 { faceCount = Int(n) }
|
||||
if faceCount == 0 { faces.removeAll(keepingCapacity: true) }
|
||||
case "/face/kp":
|
||||
guard args.count >= 6,
|
||||
let pid = args[0] as? Int32,
|
||||
let slot = args[1] as? Int32,
|
||||
let x = args[2] as? Float,
|
||||
let y = args[3] as? Float else { return }
|
||||
let s = Int(slot)
|
||||
guard s >= 0 && s < 68 else { return }
|
||||
var f = faces[Int(pid)] ?? FaceFrame()
|
||||
f.points[s] = SIMD2(x, y)
|
||||
f.hasPoint[s] = true
|
||||
f.seenAt = CFAbsoluteTimeGetCurrent()
|
||||
faces[Int(pid)] = f
|
||||
case "/hand/count":
|
||||
if args.count >= 2,
|
||||
let l = args[0] as? Int32, let r = args[1] as? Int32 {
|
||||
handCountLeft = Int(l)
|
||||
handCountRight = Int(r)
|
||||
if handCountLeft + handCountRight == 0 {
|
||||
hands.removeAll(keepingCapacity: true)
|
||||
}
|
||||
}
|
||||
case "/hand/kp":
|
||||
guard args.count >= 7,
|
||||
let pid = args[0] as? Int32,
|
||||
let side = args[1] as? Int32,
|
||||
let idx = args[2] as? Int32,
|
||||
let x = args[3] as? Float,
|
||||
let y = args[4] as? Float else { return }
|
||||
let i = Int(idx)
|
||||
guard i >= 0 && i < 21 else { return }
|
||||
var h = hands[Int(pid)] ?? HandFrame()
|
||||
h.side = Int(side)
|
||||
h.points[i] = SIMD2(x, y)
|
||||
h.hasPoint[i] = true
|
||||
h.seenAt = CFAbsoluteTimeGetCurrent()
|
||||
hands[Int(pid)] = h
|
||||
case "/pose3d/count":
|
||||
if let n = args.first as? Int32 {
|
||||
body3dCount = Int(n)
|
||||
if body3dCount == 0 {
|
||||
body3d.removeAll(keepingCapacity: true)
|
||||
}
|
||||
}
|
||||
case "/pose3d/kp":
|
||||
guard args.count >= 6,
|
||||
let pid = args[0] as? Int32,
|
||||
let idx = args[1] as? Int32,
|
||||
let x = args[2] as? Float,
|
||||
let y = args[3] as? Float,
|
||||
let z = args[4] as? Float,
|
||||
let c = args[5] as? Float else { return }
|
||||
let i = Int(idx)
|
||||
guard i >= 0 && i < 33 else { return }
|
||||
var p = body3d[Int(pid)] ?? Pose3DFrame(pid: Int(pid))
|
||||
p.pid = Int(pid)
|
||||
p.kps[i] = SIMD4<Float>(x, y, z, c)
|
||||
p.hasPoint[i] = true
|
||||
p.seenAt = CFAbsoluteTimeGetCurrent()
|
||||
body3d[Int(pid)] = p
|
||||
default:
|
||||
break
|
||||
}
|
||||
// Garbage-collect persons non vues depuis > 2 s
|
||||
// Garbage-collect persons + body3d non vus depuis > 2 s
|
||||
let now = CFAbsoluteTimeGetCurrent()
|
||||
persons = persons.filter { $0.value.seenAt == 0
|
||||
|| now - $0.value.seenAt < 2.0 }
|
||||
body3d = body3d.filter { now - $0.value.seenAt < 2.0 }
|
||||
}
|
||||
|
||||
// MARK: - Minimal OSC parser
|
||||
|
||||
Reference in New Issue
Block a user