fix(avlivebody): run Multi-HMR off the main thread
CI build oscope-of / build-check (push) Has been cancelled

inferAsync runs the CoreML prediction on a background queue with drop-if-busy; wire() snapshots skeletons on main, infers off-main, then fuses + updates mesh on main. Makes AVBODY_MULTIHMR=1 safe (no run-loop saturation). Default stays skeleton-only.
This commit is contained in:
L'électron rare
2026-06-26 11:09:59 +02:00
parent a8c803403c
commit 5ed6aeb388
2 changed files with 43 additions and 8 deletions
@@ -60,14 +60,20 @@ struct ContentView: View {
consumer.onVideoFrame = { [weak consumer] pixelBuffer in
MainActor.assumeIsolated {
controller.updateVideo(pixelBuffer)
guard let consumer else { return }
if let hmr = multiHMR {
let raw = hmr.infer(
pixelBuffer, cameraK: cameraK)
let fused = BodyFusion.fuse(
persons: raw,
skeletons: consumer.skeletons)
controller.updateMesh(fused)
guard let consumer, let hmr = multiHMR else { return }
// Snapshot skeletons on main; run the heavy CoreML
// prediction off-main (drops frames if busy), then hop
// back to main for the light fuse + mesh UI update.
let skeletons = consumer.skeletons
hmr.inferAsync(pixelBuffer, cameraK: cameraK) { raw in
DispatchQueue.main.async {
MainActor.assumeIsolated {
let fused = BodyFusion.fuse(
persons: raw,
skeletons: skeletons)
controller.updateMesh(fused)
}
}
}
}
}
@@ -24,6 +24,13 @@ final class MultiHMRCoreML {
private let model: MLModel
private let ciContext = CIContext()
/// Serial background queue + busy flag for `inferAsync`: keep the
/// ~150-300 ms CoreML prediction off the main thread and drop frames
/// that arrive while one is in flight.
private let inferQueue = DispatchQueue(
label: "cc.saillant.avlivebody.multihmr", qos: .userInitiated)
private let inferLock = NSLock()
private var inferBusy = false
/// Loads the bundled model. Returns nil if the resource or load
/// fails callers fall back to skeleton-only rendering.
@@ -45,6 +52,28 @@ final class MultiHMRCoreML {
}
}
/// Async inference: runs `infer` on a background queue, DROPS the
/// frame if a prior prediction is still in flight, and calls
/// `completion` with the result ON THE BACKGROUND QUEUE (the caller
/// hops back to the main thread for UI). Prevents the per-frame
/// CoreML prediction from saturating the run loop.
func inferAsync(_ pixelBuffer: CVPixelBuffer,
cameraK: [Float],
completion: @escaping ([MultiHMRPerson]) -> Void) {
inferLock.lock()
if inferBusy { inferLock.unlock(); return }
inferBusy = true
inferLock.unlock()
inferQueue.async { [weak self] in
guard let self else { return }
let result = self.infer(pixelBuffer, cameraK: cameraK)
self.inferLock.lock()
self.inferBusy = false
self.inferLock.unlock()
completion(result)
}
}
/// Run inference on one camera frame. `cameraK` is the 3x3 camera
/// intrinsics row-major.
func infer(_ pixelBuffer: CVPixelBuffer,