fix(avlivebody): run Multi-HMR off the main thread
CI build oscope-of / build-check (push) Has been cancelled
CI build oscope-of / build-check (push) Has been cancelled
inferAsync runs the CoreML prediction on a background queue with drop-if-busy; wire() snapshots skeletons on main, infers off-main, then fuses + updates mesh on main. Makes AVBODY_MULTIHMR=1 safe (no run-loop saturation). Default stays skeleton-only.
This commit is contained in:
@@ -60,14 +60,20 @@ struct ContentView: View {
|
||||
consumer.onVideoFrame = { [weak consumer] pixelBuffer in
|
||||
MainActor.assumeIsolated {
|
||||
controller.updateVideo(pixelBuffer)
|
||||
guard let consumer else { return }
|
||||
if let hmr = multiHMR {
|
||||
let raw = hmr.infer(
|
||||
pixelBuffer, cameraK: cameraK)
|
||||
let fused = BodyFusion.fuse(
|
||||
persons: raw,
|
||||
skeletons: consumer.skeletons)
|
||||
controller.updateMesh(fused)
|
||||
guard let consumer, let hmr = multiHMR else { return }
|
||||
// Snapshot skeletons on main; run the heavy CoreML
|
||||
// prediction off-main (drops frames if busy), then hop
|
||||
// back to main for the light fuse + mesh UI update.
|
||||
let skeletons = consumer.skeletons
|
||||
hmr.inferAsync(pixelBuffer, cameraK: cameraK) { raw in
|
||||
DispatchQueue.main.async {
|
||||
MainActor.assumeIsolated {
|
||||
let fused = BodyFusion.fuse(
|
||||
persons: raw,
|
||||
skeletons: skeletons)
|
||||
controller.updateMesh(fused)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,13 @@ final class MultiHMRCoreML {
|
||||
|
||||
private let model: MLModel
|
||||
private let ciContext = CIContext()
|
||||
/// Serial background queue + busy flag for `inferAsync`: keep the
|
||||
/// ~150-300 ms CoreML prediction off the main thread and drop frames
|
||||
/// that arrive while one is in flight.
|
||||
private let inferQueue = DispatchQueue(
|
||||
label: "cc.saillant.avlivebody.multihmr", qos: .userInitiated)
|
||||
private let inferLock = NSLock()
|
||||
private var inferBusy = false
|
||||
|
||||
/// Loads the bundled model. Returns nil if the resource or load
|
||||
/// fails — callers fall back to skeleton-only rendering.
|
||||
@@ -45,6 +52,28 @@ final class MultiHMRCoreML {
|
||||
}
|
||||
}
|
||||
|
||||
/// Async inference: runs `infer` on a background queue, DROPS the
|
||||
/// frame if a prior prediction is still in flight, and calls
|
||||
/// `completion` with the result ON THE BACKGROUND QUEUE (the caller
|
||||
/// hops back to the main thread for UI). Prevents the per-frame
|
||||
/// CoreML prediction from saturating the run loop.
|
||||
func inferAsync(_ pixelBuffer: CVPixelBuffer,
|
||||
cameraK: [Float],
|
||||
completion: @escaping ([MultiHMRPerson]) -> Void) {
|
||||
inferLock.lock()
|
||||
if inferBusy { inferLock.unlock(); return }
|
||||
inferBusy = true
|
||||
inferLock.unlock()
|
||||
inferQueue.async { [weak self] in
|
||||
guard let self else { return }
|
||||
let result = self.infer(pixelBuffer, cameraK: cameraK)
|
||||
self.inferLock.lock()
|
||||
self.inferBusy = false
|
||||
self.inferLock.unlock()
|
||||
completion(result)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run inference on one camera frame. `cameraK` is the 3x3 camera
|
||||
/// intrinsics row-major.
|
||||
func infer(_ pixelBuffer: CVPixelBuffer,
|
||||
|
||||
Reference in New Issue
Block a user