feat(av-live-body): webcam preview behind mesh
Adds CameraPreviewLayer that runs its own AVCaptureSession on the built-in webcam and exposes the AVCaptureVideoPreviewLayer. BodyView now layers : an NSView with the preview layer as backing + an ARView on top with transparent background. The result is the live webcam visible behind the SMPL-X meshes — useful for debug and for the performative angle the user wants. macOS lets multiple processes open the same camera simultaneously (each with its own AVCaptureSession), so this coexists fine with the Python Multi-HMR worker that consumes the camera too.
This commit is contained in:
@@ -1,26 +1,48 @@
|
||||
import AVFoundation
|
||||
import RealityKit
|
||||
import SwiftUI
|
||||
|
||||
/// Wrapper SwiftUI autour de ARView contenant les meshes SMPL-X.
|
||||
/// Wrapper SwiftUI : NSView container avec AVCaptureVideoPreviewLayer
|
||||
/// en backing (webcam) et ARView en surcouche pour les meshes SMPL-X.
|
||||
/// Le background de l'ARView est transparent pour laisser passer la cam.
|
||||
struct BodyView: NSViewRepresentable {
|
||||
@ObservedObject var renderer: MeshRenderer
|
||||
|
||||
func makeNSView(context: Context) -> ARView {
|
||||
let view = ARView(frame: .zero)
|
||||
view.environment.background = .color(.black)
|
||||
func makeNSView(context: Context) -> NSView {
|
||||
let container = NSView(frame: .zero)
|
||||
container.wantsLayer = true
|
||||
container.layer = CALayer()
|
||||
container.layer?.backgroundColor = NSColor.black.cgColor
|
||||
|
||||
// 1. Camera preview layer en backing
|
||||
let camera = CameraPreviewLayer()
|
||||
_ = camera.start()
|
||||
let preview = camera.previewLayer
|
||||
preview.frame = container.bounds
|
||||
preview.autoresizingMask = [.layerWidthSizable, .layerHeightSizable]
|
||||
container.layer?.addSublayer(preview)
|
||||
context.coordinator.camera = camera
|
||||
|
||||
// 2. ARView par-dessus, background clear pour voir la cam
|
||||
let arView = ARView(frame: container.bounds)
|
||||
arView.environment.background = .color(.clear)
|
||||
arView.autoresizingMask = [.width, .height]
|
||||
let cam = PerspectiveCamera()
|
||||
cam.camera.fieldOfViewInDegrees = 60
|
||||
let camAnchor = AnchorEntity(world: SIMD3(0, 0, 2))
|
||||
camAnchor.addChild(cam)
|
||||
view.scene.addAnchor(camAnchor)
|
||||
arView.scene.addAnchor(camAnchor)
|
||||
let bodyAnchor = AnchorEntity(world: .zero)
|
||||
view.scene.addAnchor(bodyAnchor)
|
||||
arView.scene.addAnchor(bodyAnchor)
|
||||
container.addSubview(arView)
|
||||
|
||||
context.coordinator.bodyAnchor = bodyAnchor
|
||||
context.coordinator.arView = arView
|
||||
context.coordinator.renderer = renderer
|
||||
return view
|
||||
return container
|
||||
}
|
||||
|
||||
func updateNSView(_ view: ARView, context: Context) {
|
||||
func updateNSView(_ view: NSView, context: Context) {
|
||||
guard let anchor = context.coordinator.bodyAnchor else { return }
|
||||
anchor.children.removeAll()
|
||||
for entity in renderer.personEntities.values {
|
||||
@@ -32,6 +54,8 @@ struct BodyView: NSViewRepresentable {
|
||||
|
||||
final class Coordinator {
|
||||
var bodyAnchor: AnchorEntity?
|
||||
var arView: ARView?
|
||||
var renderer: MeshRenderer?
|
||||
var camera: CameraPreviewLayer?
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import AVFoundation
|
||||
import Cocoa
|
||||
|
||||
/// Capture la webcam macOS built-in (AVCaptureDeviceTypeBuiltInWideAngleCamera)
|
||||
/// et expose une AVCaptureVideoPreviewLayer attachee a une NSView.
|
||||
///
|
||||
/// macOS autorise plusieurs processus a ouvrir la meme camera simultanement
|
||||
/// (chaque process a sa propre AVCaptureSession), donc cohabiter avec le
|
||||
/// worker Python Multi-HMR ne pose pas de probleme TCC.
|
||||
final class CameraPreviewLayer {
|
||||
private let session = AVCaptureSession()
|
||||
let previewLayer: AVCaptureVideoPreviewLayer
|
||||
|
||||
init() {
|
||||
self.previewLayer = AVCaptureVideoPreviewLayer(session: session)
|
||||
self.previewLayer.videoGravity = .resizeAspectFill
|
||||
}
|
||||
|
||||
func start() -> Bool {
|
||||
let discovery = AVCaptureDevice.DiscoverySession(
|
||||
deviceTypes: [.builtInWideAngleCamera],
|
||||
mediaType: .video,
|
||||
position: .unspecified)
|
||||
guard let device = discovery.devices.first else {
|
||||
print("No built-in camera found")
|
||||
return false
|
||||
}
|
||||
do {
|
||||
let input = try AVCaptureDeviceInput(device: device)
|
||||
guard session.canAddInput(input) else {
|
||||
print("session refused input")
|
||||
return false
|
||||
}
|
||||
session.addInput(input)
|
||||
} catch {
|
||||
print("AVCaptureDeviceInput error: \(error)")
|
||||
return false
|
||||
}
|
||||
session.sessionPreset = .high
|
||||
DispatchQueue.global(qos: .userInitiated).async { [weak self] in
|
||||
self?.session.startRunning()
|
||||
}
|
||||
print("CameraPreviewLayer running on '\(device.localizedName)'")
|
||||
return true
|
||||
}
|
||||
|
||||
func stop() {
|
||||
session.stopRunning()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user