feat(av-live): openpos 3D + DINO reid + filter

Three improvements wired end-to-end:

1. Openpos 3D skeleton visible: Skeleton3DRenderer attached to a
   RealityKit AnchorEntity in BodyView, toggled by showSkeleton
   or vizMode==9. PoseOSCListener now parses /pose3d/count and
   /pose3d/kp (plus restored /face/* and /hand/* paths).

2. DINO re-id (dinov2_vits14, ~9 ms ANE forward):
   MeshRigger combines Hungarian IoU with cosine similarity over
   a per-pid embedding history (deque maxlen=10), weighted by
   MULTIHMR_REID_ALPHA (default 0.5). Falls back to pure IoU if
   DINO mlpackage absent or scipy missing. state.last_frame_rgb
   buffer added so the rigger can crop bbox regions for embedding.

3. PoseFilterChain on pose_world_landmarks:
   median (anti-spike) -> Kalman constant-velocity ->
   50 ms lookahead -> IK elbow/knee/ankle clamp. Configurable
   via POSE_FILTER env (default median+kalman+lookahead+ik).
   <2 ms per frame for typical 1-2 persons.

Tests: 5 new in test_dino_reid.py + 6 new in test_pose_filter.py,
all green. Live validated by user: skeleton spawns, mesh stays
stable.
This commit is contained in:
L'électron rare
2026-05-14 00:30:42 +02:00
parent aad56e5bf9
commit 7ed2e2764a
12 changed files with 1511 additions and 3 deletions
@@ -101,6 +101,15 @@ struct BodyView: NSViewRepresentable {
context.coordinator.sceneRenderer = scene
context.coordinator.mtkView = mtkView
context.coordinator.skeletonOverlay = SkeletonOverlay(parent: bodyAnchor)
// Skeleton 3D RealityKit armature (33 spheres + 32 cylinders bones)
// driven by /pose3d/* OSC from MediaPipe pose_world_landmarks.
// Visible quand toggle showSkeleton ou vizMode==9 (openpos).
let skel3dAnchor = AnchorEntity(world: SIMD3<Float>(0, 0, -2.5))
arView.scene.addAnchor(skel3dAnchor)
let skel3d = Skeleton3DRenderer()
skel3d.attach(to: skel3dAnchor, listener: poseListener)
context.coordinator.skel3dAnchor = skel3dAnchor
context.coordinator.skel3d = skel3d
context.coordinator.keyLight = key
context.coordinator.fillLight = fill
context.coordinator.rimLight = rim
@@ -162,6 +171,9 @@ struct BodyView: NSViewRepresentable {
let skelVisible = settings.vizMode == 9 || settings.showSkeleton
c.skeletonOverlay?.update(persons: poseListener.persons,
visible: skelVisible)
// 3D RealityKit armature : show/hide root anchor in sync with
// the same skelVisible signal as the 2D overlay.
c.skel3dAnchor?.isEnabled = skelVisible
// Pose -> scene uniforms : drive hands3d (mode 8) et openpos
// (mode 9) avec la premiere personne detectee. Les wrists pilotent
// hand_l/r ; pose_count alimente bg_fragment.
@@ -211,6 +223,8 @@ struct BodyView: NSViewRepresentable {
var sceneRenderer: SceneRenderer?
var mtkView: MTKView?
var skeletonOverlay: SkeletonOverlay?
var skel3dAnchor: AnchorEntity?
var skel3d: Skeleton3DRenderer?
var kbMonitor: Any?
deinit {
@@ -26,8 +26,39 @@ final class PoseOSCListener: ObservableObject {
var seenAt: TimeInterval = 0
}
/// MediaPipe pose_world_landmarks : 33 keypoints in meters, hip-rel.
/// MediaPipe convention : x=right, y=down, z=forward (away from cam).
struct Pose3DFrame: Equatable {
var pid: Int = -1
var kps: [SIMD4<Float>] = Array(repeating: .zero, count: 33)
var hasPoint: [Bool] = Array(repeating: false, count: 33)
var seenAt: TimeInterval = 0
}
/// 68 dlib-style facial landmarks (x,y normalises 0..1).
struct FaceFrame: Equatable {
var points: [SIMD2<Float>] = Array(repeating: .zero, count: 68)
var hasPoint: [Bool] = Array(repeating: false, count: 68)
var seenAt: TimeInterval = 0
}
/// 21 MediaPipe hand landmarks per detected hand.
struct HandFrame: Equatable {
var side: Int = 0
var points: [SIMD2<Float>] = Array(repeating: .zero, count: 21)
var hasPoint: [Bool] = Array(repeating: false, count: 21)
var seenAt: TimeInterval = 0
}
@Published var persons: [Int: PoseFrame] = [:]
@Published var count: Int = 0
@Published var body3d: [Int: Pose3DFrame] = [:]
@Published var body3dCount: Int = 0
@Published var faces: [Int: FaceFrame] = [:]
@Published var hands: [Int: HandFrame] = [:]
@Published var faceCount: Int = 0
@Published var handCountLeft: Int = 0
@Published var handCountRight: Int = 0
private var listener: NWListener?
@@ -148,13 +179,77 @@ final class PoseOSCListener: ObservableObject {
p.skeleton = skel
p.seenAt = CFAbsoluteTimeGetCurrent()
persons[Int(pid)] = p
case "/face/count":
if let n = args.first as? Int32 { faceCount = Int(n) }
if faceCount == 0 { faces.removeAll(keepingCapacity: true) }
case "/face/kp":
guard args.count >= 6,
let pid = args[0] as? Int32,
let slot = args[1] as? Int32,
let x = args[2] as? Float,
let y = args[3] as? Float else { return }
let s = Int(slot)
guard s >= 0 && s < 68 else { return }
var f = faces[Int(pid)] ?? FaceFrame()
f.points[s] = SIMD2(x, y)
f.hasPoint[s] = true
f.seenAt = CFAbsoluteTimeGetCurrent()
faces[Int(pid)] = f
case "/hand/count":
if args.count >= 2,
let l = args[0] as? Int32, let r = args[1] as? Int32 {
handCountLeft = Int(l)
handCountRight = Int(r)
if handCountLeft + handCountRight == 0 {
hands.removeAll(keepingCapacity: true)
}
}
case "/hand/kp":
guard args.count >= 7,
let pid = args[0] as? Int32,
let side = args[1] as? Int32,
let idx = args[2] as? Int32,
let x = args[3] as? Float,
let y = args[4] as? Float else { return }
let i = Int(idx)
guard i >= 0 && i < 21 else { return }
var h = hands[Int(pid)] ?? HandFrame()
h.side = Int(side)
h.points[i] = SIMD2(x, y)
h.hasPoint[i] = true
h.seenAt = CFAbsoluteTimeGetCurrent()
hands[Int(pid)] = h
case "/pose3d/count":
if let n = args.first as? Int32 {
body3dCount = Int(n)
if body3dCount == 0 {
body3d.removeAll(keepingCapacity: true)
}
}
case "/pose3d/kp":
guard args.count >= 6,
let pid = args[0] as? Int32,
let idx = args[1] as? Int32,
let x = args[2] as? Float,
let y = args[3] as? Float,
let z = args[4] as? Float,
let c = args[5] as? Float else { return }
let i = Int(idx)
guard i >= 0 && i < 33 else { return }
var p = body3d[Int(pid)] ?? Pose3DFrame(pid: Int(pid))
p.pid = Int(pid)
p.kps[i] = SIMD4<Float>(x, y, z, c)
p.hasPoint[i] = true
p.seenAt = CFAbsoluteTimeGetCurrent()
body3d[Int(pid)] = p
default:
break
}
// Garbage-collect persons non vues depuis > 2 s
// Garbage-collect persons + body3d non vus depuis > 2 s
let now = CFAbsoluteTimeGetCurrent()
persons = persons.filter { $0.value.seenAt == 0
|| now - $0.value.seenAt < 2.0 }
body3d = body3d.filter { now - $0.value.seenAt < 2.0 }
}
// MARK: - Minimal OSC parser