feat(av-live-body): webcam preview behind mesh

Adds CameraPreviewLayer that runs its own AVCaptureSession on the
built-in webcam and exposes the AVCaptureVideoPreviewLayer.

BodyView now layers : an NSView with the preview layer as backing
+ an ARView on top with transparent background. The result is the
live webcam visible behind the SMPL-X meshes — useful for debug
and for the performative angle the user wants.

macOS lets multiple processes open the same camera simultaneously
(each with its own AVCaptureSession), so this coexists fine with
the Python Multi-HMR worker that consumes the camera too.
This commit is contained in:
L'électron rare
2026-05-13 11:57:59 +02:00
parent 146fc335cf
commit 5eefe7baa3
2 changed files with 82 additions and 8 deletions
@@ -1,26 +1,48 @@
import AVFoundation
import RealityKit
import SwiftUI
/// Wrapper SwiftUI autour de ARView contenant les meshes SMPL-X.
/// Wrapper SwiftUI : NSView container avec AVCaptureVideoPreviewLayer
/// en backing (webcam) et ARView en surcouche pour les meshes SMPL-X.
/// Le background de l'ARView est transparent pour laisser passer la cam.
struct BodyView: NSViewRepresentable {
@ObservedObject var renderer: MeshRenderer
func makeNSView(context: Context) -> ARView {
let view = ARView(frame: .zero)
view.environment.background = .color(.black)
func makeNSView(context: Context) -> NSView {
let container = NSView(frame: .zero)
container.wantsLayer = true
container.layer = CALayer()
container.layer?.backgroundColor = NSColor.black.cgColor
// 1. Camera preview layer en backing
let camera = CameraPreviewLayer()
_ = camera.start()
let preview = camera.previewLayer
preview.frame = container.bounds
preview.autoresizingMask = [.layerWidthSizable, .layerHeightSizable]
container.layer?.addSublayer(preview)
context.coordinator.camera = camera
// 2. ARView par-dessus, background clear pour voir la cam
let arView = ARView(frame: container.bounds)
arView.environment.background = .color(.clear)
arView.autoresizingMask = [.width, .height]
let cam = PerspectiveCamera()
cam.camera.fieldOfViewInDegrees = 60
let camAnchor = AnchorEntity(world: SIMD3(0, 0, 2))
camAnchor.addChild(cam)
view.scene.addAnchor(camAnchor)
arView.scene.addAnchor(camAnchor)
let bodyAnchor = AnchorEntity(world: .zero)
view.scene.addAnchor(bodyAnchor)
arView.scene.addAnchor(bodyAnchor)
container.addSubview(arView)
context.coordinator.bodyAnchor = bodyAnchor
context.coordinator.arView = arView
context.coordinator.renderer = renderer
return view
return container
}
func updateNSView(_ view: ARView, context: Context) {
func updateNSView(_ view: NSView, context: Context) {
guard let anchor = context.coordinator.bodyAnchor else { return }
anchor.children.removeAll()
for entity in renderer.personEntities.values {
@@ -32,6 +54,8 @@ struct BodyView: NSViewRepresentable {
final class Coordinator {
var bodyAnchor: AnchorEntity?
var arView: ARView?
var renderer: MeshRenderer?
var camera: CameraPreviewLayer?
}
}
@@ -0,0 +1,50 @@
import AVFoundation
import Cocoa
/// Capture la webcam macOS built-in (AVCaptureDeviceTypeBuiltInWideAngleCamera)
/// et expose une AVCaptureVideoPreviewLayer attachee a une NSView.
///
/// macOS autorise plusieurs processus a ouvrir la meme camera simultanement
/// (chaque process a sa propre AVCaptureSession), donc cohabiter avec le
/// worker Python Multi-HMR ne pose pas de probleme TCC.
final class CameraPreviewLayer {
private let session = AVCaptureSession()
let previewLayer: AVCaptureVideoPreviewLayer
init() {
self.previewLayer = AVCaptureVideoPreviewLayer(session: session)
self.previewLayer.videoGravity = .resizeAspectFill
}
func start() -> Bool {
let discovery = AVCaptureDevice.DiscoverySession(
deviceTypes: [.builtInWideAngleCamera],
mediaType: .video,
position: .unspecified)
guard let device = discovery.devices.first else {
print("No built-in camera found")
return false
}
do {
let input = try AVCaptureDeviceInput(device: device)
guard session.canAddInput(input) else {
print("session refused input")
return false
}
session.addInput(input)
} catch {
print("AVCaptureDeviceInput error: \(error)")
return false
}
session.sessionPreset = .high
DispatchQueue.global(qos: .userInitiated).async { [weak self] in
self?.session.startRunning()
}
print("CameraPreviewLayer running on '\(device.localizedName)'")
return true
}
func stop() {
session.stopRunning()
}
}