From c1b567f116f408bb9218777b66b9a8c6a63301e6 Mon Sep 17 00:00:00 2001 From: clement Date: Sun, 28 Jun 2026 13:44:09 +0200 Subject: [PATCH] feat: VIDEO_ROTATE to rotate source frame Rotate the iPhone/webcam frame before MediaPipe and display so detection, skeleton overlay and the shown video stay aligned after the phone is physically rotated. ARKit 3D world joints are orientation-invariant and untouched. --- CLAUDE.md | 1 + data_only_viz/multi.py | 26 ++++++++++++ data_only_viz/tests/test_video_rotate.py | 52 ++++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 data_only_viz/tests/test_video_rotate.py diff --git a/CLAUDE.md b/CLAUDE.md index 98e7537..d6894f8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,6 +64,7 @@ Toujours répondre en français à l'utilisateur. Code, commentaires de code, co |-----|---------|--------| | `POSE_FILTER` | `median+kalman+lookahead+ik` | filter chain stages — extra: `one_euro_joints` (joint-space CHI 2012 One Euro, inserted before kalman), `one_euro_bones` (bone-vector One Euro applied after SMPL-X fusion in multi.py), `arkit_fuse` (overrides 14 body slots with ARKit ARSkeleton3D from the iOS app, expects /body3d/kp on :57128) | | `IPHONE_OSC_PORT` | `57128` | UDP port the iPhone ARBodyTracker app pushes /body3d/kp to (always-on listener in data_only_viz) | +| `VIDEO_ROTATE` | `none` | rotate the source video frame before MediaPipe + display: `none`/`ccw`/`cw`/`180`. Applied at multi.py read; pose detection + skeleton overlay stay aligned. ARKit 3D joints (gravity-aligned world) are unaffected | | `ICP_FUSION` | `0` | `1` to enable LiDAR↔SMPL-X ICP fusion (cf. `docs/ICP_FUSION.md`) | | `ICP_LIDAR_HOST` | _(unset)_ | iPhone ARBodyTracker IP when `ICP_FUSION=1` | | `ICP_LIDAR_PORT` | `5500` | iPhone ARMesh TCP port | diff --git a/data_only_viz/multi.py b/data_only_viz/multi.py index b103b34..43bf832 100644 --- a/data_only_viz/multi.py +++ b/data_only_viz/multi.py @@ -19,6 +19,8 @@ import time import urllib.request from pathlib import Path +import numpy as np + from .action_head_pub import ActionHeadPublisher from .euro_filter import SkeletonFilter from .pose_bridge import PoseSoundBridge @@ -29,6 +31,25 @@ from .tracker import IoUTracker LOG = logging.getLogger("multi") +# Rotation cosmetique + detection de la frame video (env VIDEO_ROTATE). +# Appliquee AVANT MediaPipe : la detection tourne sur l'image redressee et +# l'overlay reste aligne. Les joints ARKit (monde 3D, gravity-aligned) sont +# invariants a l'orientation du device, donc non concernes. +_ROTATE_K = {"none": 0, "ccw": 1, "180": 2, "cw": 3} + + +def _apply_video_rotate(frame, mode: str): + """Rotate a BGR frame by mode (none/ccw/cw/180) via numpy rot90. + + Returns a C-contiguous array (MediaPipe / cv2 need contiguous input). + Unknown modes are treated as 'none' (no rotation). + """ + k = _ROTATE_K.get(mode, 0) + if k == 0: + return frame + return np.ascontiguousarray(np.rot90(frame, k)) + + MODELS = { "pose": ( "https://storage.googleapis.com/mediapipe-models/pose_landmarker/" @@ -314,6 +335,8 @@ class MultiWorker: else BaseOptions.Delegate.CPU) LOG.info("MediaPipe delegate = %s (env MEDIAPIPE_DELEGATE)", _deleg.name) + _rot = _os.environ.get("VIDEO_ROTATE", "none").lower() + LOG.info("video rotate = %s (env VIDEO_ROTATE: none/ccw/cw/180)", _rot) pose = PoseLandmarker.create_from_options(PoseLandmarkerOptions( base_options=BaseOptions(model_asset_path=str(pose_p), delegate=_deleg), @@ -367,6 +390,9 @@ class MultiWorker: if not ok or frame_bgr is None: time.sleep(self.period) continue + # Redresse la frame (iPhone tourne physiquement) AVANT MediaPipe + # et l'encodage JPEG : detection + overlay + affichage coherents. + frame_bgr = _apply_video_rotate(frame_bgr, _rot) h, w = frame_bgr.shape[:2] # MediaPipe GPU delegate on macOS uploads via CVPixelBuffer # which only accepts 4-channel formats. SRGB (3ch) crashes diff --git a/data_only_viz/tests/test_video_rotate.py b/data_only_viz/tests/test_video_rotate.py new file mode 100644 index 0000000..9ec6ce6 --- /dev/null +++ b/data_only_viz/tests/test_video_rotate.py @@ -0,0 +1,52 @@ +"""Tests for the VIDEO_ROTATE frame rotation helper in multi.py.""" +from __future__ import annotations + +import numpy as np + +from data_only_viz.multi import _apply_video_rotate + + +def _frame(): + """(2, 3, 3) BGR frame with a marker at the top-right corner (row0, col2).""" + f = np.zeros((2, 3, 3), dtype=np.uint8) + f[0, 2] = (9, 9, 9) + return f + + +def test_none_is_identity(): + f = _frame() + out = _apply_video_rotate(f, "none") + assert out is f # no copy when not rotating + + +def test_unknown_mode_is_identity(): + f = _frame() + out = _apply_video_rotate(f, "garbage") + assert out is f + + +def test_ccw_swaps_dims_and_moves_corner(): + f = _frame() # h=2, w=3 + out = _apply_video_rotate(f, "ccw") + assert out.shape == (3, 2, 3) # w and h swapped + # CCW: top-right corner -> top-left + assert tuple(out[0, 0]) == (9, 9, 9) + assert out.flags["C_CONTIGUOUS"] + + +def test_cw_swaps_dims_and_moves_corner(): + f = _frame() + out = _apply_video_rotate(f, "cw") + assert out.shape == (3, 2, 3) + # CW: top-right corner -> bottom-right + assert tuple(out[2, 1]) == (9, 9, 9) + assert out.flags["C_CONTIGUOUS"] + + +def test_180_moves_corner_opposite(): + f = _frame() + out = _apply_video_rotate(f, "180") + assert out.shape == (2, 3, 3) # dims preserved + # 180: top-right corner -> bottom-left + assert tuple(out[1, 0]) == (9, 9, 9) + assert out.flags["C_CONTIGUOUS"]