electron-rare/AV-Live
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/action-head'

Browse Source 
# Conflicts:
#	data_only_viz/multihmr_coreml.py
#	launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Skeleton3DRenderer.swift
This commit is contained in:
L'électron rare
2026-06-23 23:01:07 +02:00
parent df2b068c2b b6cd8cf4c7
commit c5941b9625
16 changed files with 1823 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
launcher/_archive-AV-Live-Body/Sources/AVLiveBody/BodyView.swift
+119
View File
@@ -2,6 +2,7 @@ import AppKit
import AVFoundation
import MetalKit
import RealityKit
import simd
import SwiftUI
/// Wrapper SwiftUI : webcam en BACKGROUND LEGER (opacite controlable)
@@ -199,6 +200,119 @@ struct BodyView: NSViewRepresentable {
c.sceneRenderer?.uniforms.hand_r_x = 0
c.sceneRenderer?.uniforms.hand_r_y = 0
}
// ---- 2026-05-14 face/hand/body3d derivatives ----
// Simple EMA smoother to limit jitter from raw detections.
func ema(_ key: String, _ raw: Float, alpha: Float = 0.7) -> Float {
let prev = c.smoothedUniforms[key] ?? raw
let next = alpha * prev + (1.0 - alpha) * raw
c.smoothedUniforms[key] = next
return next
}
// --- Face : mouth_open, eye_open_l/r, head_tilt, head_yaw ---
var rawMouth: Float = 0
var rawEyeL: Float = 0
var rawEyeR: Float = 0
var rawTilt: Float = 0
var rawYaw: Float = 0
if let face = poseListener.faces.values.first {
let pts = face.points
let has = face.hasPoint
// dlib 68 : upper inner lip 51, lower inner lip 57
if has[51] && has[57] {
rawMouth = abs(pts[51].y - pts[57].y)
}
// Eye L : 36..41, Eye R : 42..47
func eyeRatio(_ a: Int, _ b: Int) -> Float {
var xs: [Float] = []
var ys: [Float] = []
for i in a...b where has[i] {
xs.append(pts[i].x)
ys.append(pts[i].y)
}
guard let mxX = xs.max(), let mnX = xs.min(),
let mxY = ys.max(), let mnY = ys.min() else {
return 0
}
let w = max(mxX - mnX, 1e-4)
let h = mxY - mnY
return h / w
}
rawEyeL = eyeRatio(36, 41)
rawEyeR = eyeRatio(42, 47)
// head_tilt : line eyeL_center -> eyeR_center
if has[36] && has[45] {
let dy = pts[45].y - pts[36].y
let dx = pts[45].x - pts[36].x
rawTilt = atan2(dy, dx)
}
// head_yaw proxy : nose(30) vs eye midpoint y
if has[30] && has[36] && has[45] {
let midY = (pts[36].y + pts[45].y) * 0.5
rawYaw = (pts[30].y - midY) * 4.0 // amplify
}
}
c.sceneRenderer?.uniforms.mouth_open = ema("mouth", rawMouth)
c.sceneRenderer?.uniforms.eye_open_l = ema("eyeL", rawEyeL)
c.sceneRenderer?.uniforms.eye_open_r = ema("eyeR", rawEyeR)
c.sceneRenderer?.uniforms.head_tilt = ema("tilt", rawTilt)
c.sceneRenderer?.uniforms.head_yaw = ema("yaw", rawYaw)
// --- Hands : finger_pinch_l, finger_pinch_r ---
// MediaPipe : 4 = thumb_tip, 8 = index_tip ; side : 0=L, 1=R.
var rawPinchL: Float = 0
var rawPinchR: Float = 0
for h in poseListener.hands.values {
guard h.hasPoint[4] && h.hasPoint[8] else { continue }
let d = simd_distance(h.points[4], h.points[8])
if h.side == 0 { rawPinchL = d } else { rawPinchR = d }
}
c.sceneRenderer?.uniforms.finger_pinch_l = ema("pinchL", rawPinchL)
c.sceneRenderer?.uniforms.finger_pinch_r = ema("pinchR", rawPinchR)
// --- Body3D : pelvis pos, body_height, arm_spread, velocity ---
// MediaPipe : 23 left_hip, 24 right_hip ; 0 nose ;
// 15 left_wrist, 16 right_wrist.
var rawBX: Float = 0, rawBY: Float = 0, rawBZ: Float = 0
var rawHeight: Float = 0, rawSpread: Float = 0
var rawVel: Float = 0
if let body = poseListener.body3d.values.first {
let kp = body.kps
let has = body.hasPoint
if has[23] && has[24] {
let pelvis = SIMD3<Float>(
(kp[23].x + kp[24].x) * 0.5,
(kp[23].y + kp[24].y) * 0.5,
(kp[23].z + kp[24].z) * 0.5)
rawBX = pelvis.x
rawBY = pelvis.y
rawBZ = pelvis.z
// Velocity : EMA of delta magnitude (alpha=0.3)
if let last = c.lastPelvis {
let dist = simd_distance(pelvis, last)
c.poseVelocityEMA = 0.3 * c.poseVelocityEMA + 0.7 * dist
}
c.lastPelvis = pelvis
rawVel = c.poseVelocityEMA
if has[0] {
// body_height : pelvis.y - head.y (MP y points down so
// head < pelvis ; take abs).
rawHeight = abs(pelvis.y - kp[0].y)
}
}
if has[15] && has[16] {
rawSpread = abs(kp[15].x - kp[16].x)
}
} else {
c.lastPelvis = nil
c.poseVelocityEMA *= 0.9 // decay
}
c.sceneRenderer?.uniforms.body_x = ema("bx", rawBX)
c.sceneRenderer?.uniforms.body_y = ema("by", rawBY)
c.sceneRenderer?.uniforms.body_z = ema("bz", rawBZ)
c.sceneRenderer?.uniforms.body_height = ema("bh", rawHeight)
c.sceneRenderer?.uniforms.arm_spread = ema("spread", rawSpread)
c.sceneRenderer?.uniforms.pose_velocity = ema("vel", rawVel, alpha: 0.5)
c.container?.layer?.backgroundColor = NSColor(
white: CGFloat(settings.bgBrightness), alpha: 1.0).cgColor
c.cameraEntity?.camera.fieldOfViewInDegrees =
@@ -233,6 +347,11 @@ struct BodyView: NSViewRepresentable {
var skel3dAnchor: AnchorEntity?
var skel3d: Skeleton3DRenderer?
var kbMonitor: Any?
// ---- Pose-derived smoothing state (alpha-beta EMA) ----
// EMA alpha=0.7, beta=0.05 (velocity correction).
var smoothedUniforms: [String: Float] = [:]
var lastPelvis: SIMD3<Float>?
var poseVelocityEMA: Float = 0
deinit {
if let m = kbMonitor {
launcher/_archive-AV-Live-Body/Sources/AVLiveBody/Resources/scene.metal
+56 -13
View File
@@ -34,8 +34,34 @@ struct SceneUniforms {
float hand_l_y;
float hand_r_x;
float hand_r_y;
// ---- 2026-05-14 face / hand / body3d derivatives ----
// mouth_open : |lip51.y - lip57.y| normalized 0..~0.15
// eye_open_l/r : H/W ratio of eye bbox, ~0.15 (closed) .. 0.45 (open)
// head_tilt : atan2(eye_R.y-eye_L.y, eye_R.x-eye_L.x) rad
// head_yaw : nose.y - eyes_mid.y (proxy pitch, normalized)
// finger_pinch_l/r: simd_distance(thumb_tip[4], index_tip[8]) px-norm
// body_x/y/z : pelvis world meters (MediaPipe hip-relative)
// body_height : |hip.y - head.y| world meters
// arm_spread : |wristL.x - wristR.x| world meters
// pose_velocity : EMA(|pelvis(t) - pelvis(t-1)|), alpha=0.3
float mouth_open;
float eye_open_l;
float eye_open_r;
float head_tilt;
float head_yaw;
float finger_pinch_l;
float finger_pinch_r;
float body_x;
float body_y;
float body_z;
float body_height;
float arm_spread;
float pose_velocity;
float _pad0;
float _pad1;
float _pad2;
float _pad3;
float _pad4;
};
struct VsOut {
@@ -120,21 +146,25 @@ float vignette(float2 p) {
// ===== Modes =======================================================
// ---- 0 storm : tissu fbm reactif + bloom-fake ----
// Pose mods : pose_velocity boost intensity, head_tilt shifts hue.
float3 mode_storm(float2 p, constant SceneUniforms& U) {
float storm = saturate(U.kp_norm * 1.0 + max(-U.bz_norm, 0.0) * 0.5);
float speed = 0.08 + U.wind_norm * 1.5;
float storm = saturate(U.kp_norm * 1.0 + max(-U.bz_norm, 0.0) * 0.5
+ U.pose_velocity * 2.0);
float speed = 0.08 + U.wind_norm * 1.5 + U.pose_velocity * 3.0;
float zoom = 1.8 - U.rms * 1.2;
float n = fbm(p * zoom + float2(U.time * speed, U.time * speed * 0.7));
n = pow(n, 1.2 - U.rms * 0.5);
float netz = sin(U.time * 50.0 + U.netz_dev * 800.0) * 0.06;
float3 base = palIQ(n + storm * 0.5,
float hue_shift = U.head_tilt * 0.25;
float3 base = palIQ(n + storm * 0.5 + hue_shift,
float3(0.10, 0.05, 0.20),
float3(0.40, 0.30, 0.55),
float3(1.0, 1.0, 1.0),
float3(0.0, 0.33, 0.67));
float bloom = smoothstep(0.7, 1.0, n);
return base * (n * 1.4 + 0.3) + netz + U.rms * 1.2
+ bloom * 0.7
float velocity_boost = 1.0 + U.pose_velocity * 1.5;
return (base * (n * 1.4 + 0.3) + netz + U.rms * 1.2
+ bloom * 0.7) * velocity_boost
+ float3(1.0, 0.55, 0.1) * U.flare * 1.4
+ float3(U.lightning_flash * 0.7);
}
@@ -167,34 +197,40 @@ float3 mode_tunnel(float2 p, constant SceneUniforms& U) {
}
// ---- 2 plasma : volumetric noise palette IQ ----
// Pose mods : mouth_open modulates spatial frequency; head_yaw shifts hue.
float3 mode_plasma(float2 p, constant SceneUniforms& U) {
float t = U.time * (0.5 + U.rms * 1.5);
float freq_boost = 1.0 + U.mouth_open * 12.0;
// 3 octaves de sin/cos en composition
float v = sin(p.x * 4.0 + t)
+ sin(p.y * 5.0 - t * 1.2)
float v = sin(p.x * 4.0 * freq_boost + t)
+ sin(p.y * 5.0 * freq_boost - t * 1.2)
+ sin((p.x + p.y) * 3.5 + t * 0.7)
+ sin(length(p) * (8.0 + U.kp_norm * 4.0) - t * 1.8);
v = v * 0.25 + 0.5;
// Fake volumetric "depth" : repeat layers
float layer2 = sin(p.x * 2.0 - t * 0.5) * sin(p.y * 2.5 + t * 0.7);
v = mix(v, v * 0.5 + 0.5 * (layer2 + 1.0) * 0.5, 0.35);
float3 col = palIQ(v,
float hue_offset = U.head_yaw * 0.5;
float3 col = palIQ(v + hue_offset,
float3(0.5),
float3(0.5),
float3(1.0, 1.0, 1.0),
float3(0.0, 0.33, 0.67));
col *= 0.8 + U.kp_norm * 0.7 + U.social_rate * 0.5;
col *= 0.8 + U.kp_norm * 0.7 + U.social_rate * 0.5
+ U.mouth_open * 1.0;
return col + float3(0.6, 0.3, 1.0) * U.lightning_flash * 0.5;
}
// ---- 3 kaleido : KIFS fractal 6-fold avec rot 3D fake ----
// Pose mods : arm_spread drives segment count (4..16).
float3 mode_kaleido(float2 p, constant SceneUniforms& U) {
float ang = U.time * 0.15 + U.flare * 2.0;
float c = cos(ang), s = sin(ang);
p = float2(c * p.x - s * p.y, s * p.x + c * p.y);
float r = length(p);
float a = atan2(p.y, p.x);
float seg = 6.28318 / 6.0;
float seg_count = clamp(ceil(4.0 + U.arm_spread * 8.0), 3.0, 16.0);
float seg = 6.28318 / seg_count;
a = abs(fmod(a + seg * 0.5, seg) - seg * 0.5);
float2 q = float2(cos(a), sin(a)) * r;
// Iteration KIFS-like
@@ -303,14 +339,18 @@ float3 mode_metaballs(float2 p, constant SceneUniforms& U) {
}
// ---- 6 starfield : galaxy spiral + parallax ----
// Pose mods : finger_pinch L+R drives star density per layer.
float3 mode_starfield(float2 p, constant SceneUniforms& U) {
float warp = U.time * (1.5 + U.wind_norm * 6.0);
float pinch = saturate((U.finger_pinch_l + U.finger_pinch_r) * 2.0);
int stars_per_layer = 30 + int(pinch * 70.0); // 30..100
// 3 layers of stars at different speeds
float3 col = float3(0);
for (int L = 0; L < 3; ++L) {
float speed = (1.0 + float(L) * 0.5);
float scale = 6.0 + float(L) * 4.0;
for (int k = 0; k < 50; ++k) {
for (int k = 0; k < 100; ++k) {
if (k >= stars_per_layer) break;
float fk = float(k + L * 50);
float r0 = hash21(float2(fk, 7.0 + float(L)));
float a0 = hash21(float2(fk, 17.0 + float(L))) * 6.28;
@@ -352,8 +392,11 @@ float3 mode_bars(float2 p, constant SceneUniforms& U) {
// Hauteur barre depend du bin "i" via hash + RMS
float h0 = hash21(float2(float(i), 0.0));
float h = sin(t * (0.5 + h0 * 4.0) + float(i)) * 0.5 + 0.5;
h = h * (0.3 + U.rms * 1.5 + U.social_rate * 0.4);
h = clamp(h, 0.02, 0.85);
// body_height (0..~1.8 m) + eye_open avg modulate bar height
float eyes = (U.eye_open_l + U.eye_open_r) * 0.5;
h = h * (0.3 + U.rms * 1.5 + U.social_rate * 0.4
+ U.body_height * 0.6 + eyes * 0.8);
h = clamp(h, 0.02, 0.95);
float bar_top = y_base + h * perspective * 0.3;
// Largeur = 1 / nbars perspective
float bx = (fi - 0.5) * perspective * 1.5;
launcher/_archive-AV-Live-Body/Sources/AVLiveBody/SceneRenderer.swift
+20 -1
View File
@@ -8,7 +8,9 @@ import MetalKit
/// data_only_viz Python. Sert de couche backing sous l'ARView dans
/// BodyView.
final class SceneRenderer: NSObject, MTKViewDelegate {
// Mirror C struct of scene.metal SceneUniforms (20 floats)
// Mirror C struct of scene.metal SceneUniforms (36 floats, 144 B,
// 16-byte aligned). Append-only : nouveaux uniforms ajoutes apres
// hand_r_y pour preserver la compat des 18 premiers slots.
struct SceneUniforms {
var time: Float = 0
var rms: Float = 0
@@ -28,8 +30,25 @@ final class SceneRenderer: NSObject, MTKViewDelegate {
var hand_l_y: Float = 0
var hand_r_x: Float = 0
var hand_r_y: Float = 0
// ---- Pose 2026-05-14 additions ----
var mouth_open: Float = 0
var eye_open_l: Float = 0
var eye_open_r: Float = 0
var head_tilt: Float = 0
var head_yaw: Float = 0
var finger_pinch_l: Float = 0
var finger_pinch_r: Float = 0
var body_x: Float = 0
var body_y: Float = 0
var body_z: Float = 0
var body_height: Float = 0
var arm_spread: Float = 0
var pose_velocity: Float = 0
var _pad0: Float = 0
var _pad1: Float = 0
var _pad2: Float = 0
var _pad3: Float = 0
var _pad4: Float = 0
}
private let device: MTLDevice