electron-rare/Aperant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: symlink Python venvs in worktrees for instant setup

Browse Source 
Switch venv strategy from RECREATE to SYMLINK so worktree creation is
near-instant (matching node_modules behavior). A health check after
symlinking verifies the venv is usable; if it fails, falls back to
recreate with improved robustness:

- Marker-based completion tracking (.setup_complete) detects incomplete venvs
- Popen-based subprocess management with proper terminate/kill on timeout
- Increased pip install timeout from 120s to 300s

Co-Authored-By: Claude Opus 4.6 <[email protected]>
This commit is contained in:
AndyMik90
2026-02-17 15:32:50 +01:00
co-authored by Claude Opus 4.6
parent d98ff7d19c
commit 3ede3ec2a8
4 changed files with 281 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/core/workspace/dependency_strategy.py
+7 -6
View File
@@ -9,9 +9,10 @@ Each dependency ecosystem has different constraints:
- **node_modules**: Safe to symlink. Node's resolution algorithm follows symlinks
correctly, and the directory is self-contained.
- **venv / .venv**: Must be recreated. Python's ``pyvenv.cfg`` discovery walks the
real directory hierarchy without resolving symlinks (CPython bug #106045), so a
symlinked venv resolves paths relative to the *target*, not the worktree.
- **venv / .venv**: Symlinked for fast worktree creation. CPython bug #106045
(pyvenv.cfg symlink resolution) does not affect typical usage (running scripts,
imports, pip). A health check after symlinking verifies usability; if it fails,
the caller falls back to recreating the venv.
- **vendor (PHP)**: Safe to symlink. Composer's autoloader uses ``__DIR__``-relative
paths that resolve correctly through symlinks.
@@ -42,9 +43,9 @@ from .models import DependencyShareConfig, DependencyStrategy
DEFAULT_STRATEGY_MAP: dict[str, DependencyStrategy] = {
# JavaScript / Node.js — symlink is safe and fast
"node_modules": DependencyStrategy.SYMLINK,
# Python — venvs MUST be recreated (pyvenv.cfg symlink bug)
"venv": DependencyStrategy.RECREATE,
".venv": DependencyStrategy.RECREATE,
# Python — symlink for fast worktree creation (health check + fallback to recreate)
"venv": DependencyStrategy.SYMLINK,
".venv": DependencyStrategy.SYMLINK,
# PHP — Composer vendor dir is safe to symlink
"vendor_php": DependencyStrategy.SYMLINK,
# Ruby — Bundler vendor/bundle is safe to symlink
apps/backend/core/workspace/setup.py
+96 -24
View File
@@ -50,6 +50,10 @@ _git_hook_check_done = False
MODULE = "workspace.setup"
# Marker file written inside a recreated venv to indicate setup completed successfully.
# If the marker is absent, the venv is treated as incomplete and will be rebuilt.
VENV_SETUP_COMPLETE_MARKER = ".setup_complete"
def choose_workspace(
project_dir: Path,
@@ -627,6 +631,39 @@ def setup_worktree_dependencies(
performed = True
if config.strategy == DependencyStrategy.SYMLINK:
performed = _apply_symlink_strategy(project_dir, worktree_path, config)
# For venvs, verify the symlink is usable — fall back to recreate
if performed and config.dep_type in ("venv", ".venv"):
venv_path = worktree_path / config.source_rel_path
if is_windows():
python_bin = str(venv_path / "Scripts" / "python.exe")
else:
python_bin = str(venv_path / "bin" / "python")
try:
subprocess.run(
[python_bin, "-c", "import sys; print(sys.prefix)"],
capture_output=True,
text=True,
timeout=10,
check=True,
)
debug(
MODULE,
f"Symlinked venv health check passed: {config.source_rel_path}",
)
except (subprocess.SubprocessError, OSError):
debug_warning(
MODULE,
f"Symlinked venv health check failed, falling back to recreate: {config.source_rel_path}",
)
# Remove the broken symlink and recreate
symlink_path = worktree_path / config.source_rel_path
if symlink_path.is_symlink():
symlink_path.unlink()
elif symlink_path.exists():
shutil.rmtree(symlink_path, ignore_errors=True)
performed = _apply_recreate_strategy(
project_dir, worktree_path, config
)
elif config.strategy == DependencyStrategy.RECREATE:
performed = _apply_recreate_strategy(project_dir, worktree_path, config)
elif config.strategy == DependencyStrategy.COPY:
@@ -707,6 +744,40 @@ def _apply_symlink_strategy(
return False
def _popen_with_cleanup(
cmd: list[str],
timeout: int,
label: str,
) -> tuple[int, str, str]:
"""Run a command via Popen with proper process cleanup on timeout.
On timeout: terminate → wait(10) → kill → wait(5) to ensure file locks
are released before any cleanup (e.g. shutil.rmtree).
Returns (returncode, stdout, stderr).
Raises subprocess.TimeoutExpired if the process could not be stopped.
"""
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
try:
stdout, stderr = proc.communicate(timeout=timeout)
return proc.returncode, stdout, stderr
except subprocess.TimeoutExpired:
debug_warning(MODULE, f"{label} timed out, terminating process")
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
debug_warning(MODULE, f"{label} did not terminate, killing process")
proc.kill()
proc.wait(timeout=5)
raise
def _apply_recreate_strategy(
project_dir: Path,
worktree_path: Path,
@@ -717,10 +788,18 @@ def _apply_recreate_strategy(
Returns True if the venv was successfully created, False if skipped or failed.
"""
venv_path = worktree_path / config.source_rel_path
marker_path = venv_path / VENV_SETUP_COMPLETE_MARKER
if venv_path.exists():
debug(MODULE, f"Skipping recreate {config.source_rel_path} - already exists")
return False
if marker_path.exists():
debug(
MODULE,
f"Skipping recreate {config.source_rel_path} - already complete (marker present)",
)
return False
# Venv exists but marker is missing — incomplete, remove and rebuild
debug(MODULE, f"Removing incomplete venv {config.source_rel_path} (no marker)")
shutil.rmtree(venv_path, ignore_errors=True)
# Detect Python executable from the source venv or fall back to sys.executable
source_venv = project_dir / config.source_rel_path
@@ -737,29 +816,25 @@ def _apply_recreate_strategy(
# Create the venv
try:
debug(MODULE, f"Creating venv at {venv_path}")
result = subprocess.run(
returncode, _, stderr = _popen_with_cleanup(
[python_exec, "-m", "venv", str(venv_path)],
capture_output=True,
text=True,
timeout=120,
label=f"venv creation ({config.source_rel_path})",
)
if result.returncode != 0:
debug_warning(MODULE, f"venv creation failed: {result.stderr}")
if returncode != 0:
debug_warning(MODULE, f"venv creation failed: {stderr}")
print_status(
f"Warning: Could not create venv at {config.source_rel_path}",
"warning",
)
# Clean up partial venv so retries aren't blocked
if venv_path.exists():
shutil.rmtree(venv_path, ignore_errors=True)
return False
except subprocess.TimeoutExpired:
debug_warning(MODULE, f"venv creation timed out for {config.source_rel_path}")
print_status(
f"Warning: venv creation timed out for {config.source_rel_path}",
"warning",
)
# Clean up partial venv so retries aren't blocked
if venv_path.exists():
shutil.rmtree(venv_path, ignore_errors=True)
return False
@@ -800,46 +875,43 @@ def _apply_recreate_strategy(
if install_cmd:
try:
debug(MODULE, f"Installing deps from {req_file}")
pip_result = subprocess.run(
returncode, _, stderr = _popen_with_cleanup(
install_cmd,
capture_output=True,
text=True,
timeout=120,
timeout=300,
label=f"pip install ({req_file})",
)
if pip_result.returncode != 0:
if returncode != 0:
debug_warning(
MODULE,
f"pip install failed (exit {pip_result.returncode}): "
f"{pip_result.stderr}",
f"pip install failed (exit {returncode}): {stderr}",
)
print_status(
f"Warning: Dependency install failed for {req_file}",
"warning",
)
# Clean up broken venv so retries aren't blocked
if venv_path.exists():
shutil.rmtree(venv_path, ignore_errors=True)
return False
except subprocess.TimeoutExpired:
debug_warning(
MODULE,
f"pip install timed out for {req_file}",
)
print_status(
f"Warning: Dependency install timed out for {req_file}",
"warning",
)
# Clean up broken venv so retries aren't blocked
if venv_path.exists():
shutil.rmtree(venv_path, ignore_errors=True)
return False
except OSError as e:
debug_warning(MODULE, f"pip install failed: {e}")
# Clean up broken venv so retries aren't blocked
if venv_path.exists():
shutil.rmtree(venv_path, ignore_errors=True)
return False
# Write completion marker so future runs know this venv is complete
try:
marker_path.touch()
except OSError:
pass # Non-fatal — venv is still usable without the marker
debug(MODULE, f"Recreated venv at {config.source_rel_path}")
return True
apps/frontend/src/main/ipc-handlers/terminal/worktree-handlers.ts
+41 -11
View File
@@ -250,11 +250,12 @@ interface DependencyConfig {
const DEFAULT_STRATEGY_MAP: Record<string, 'symlink' | 'recreate' | 'copy' | 'skip'> = {
// JavaScript / Node.js — symlink is safe and fast
node_modules: 'symlink',
// Python — venvs MUST be recreated, not symlinked.
// CPython bug #106045: pyvenv.cfg discovery does not resolve symlinks,
// so a symlinked venv resolves paths relative to the target, not the worktree.
venv: 'recreate',
'.venv': 'recreate',
// Python — symlink for fast worktree creation.
// CPython bug #106045 (pyvenv.cfg symlink resolution) does not affect
// typical usage (running scripts, imports, pip). If the health check
// after symlinking fails, we fall back to recreate automatically.
venv: 'symlink',
'.venv': 'symlink',
// PHP — Composer vendor dir is safe to symlink
vendor_php: 'symlink',
// Ruby — Bundler vendor/bundle is safe to symlink
@@ -364,6 +365,24 @@ async function setupWorktreeDependencies(projectPath: string, worktreePath: stri
switch (config.strategy) {
case 'symlink':
performed = applySymlinkStrategy(projectPath, worktreePath, config);
// For venvs, verify the symlink is usable — fall back to recreate if not
if (performed && (config.depType === 'venv' || config.depType === '.venv')) {
const venvPath = path.join(worktreePath, config.sourceRelPath);
const pythonBin = isWindows()
? path.join(venvPath, 'Scripts', 'python.exe')
: path.join(venvPath, 'bin', 'python');
try {
await execFileAsync(pythonBin, ['-c', 'import sys; print(sys.prefix)'], {
timeout: 10000,
});
debugLog('[TerminalWorktree] Symlinked venv health check passed:', config.sourceRelPath);
} catch {
debugLog('[TerminalWorktree] Symlinked venv health check failed, falling back to recreate:', config.sourceRelPath);
// Remove the broken symlink and recreate
try { rmSync(path.join(worktreePath, config.sourceRelPath), { force: true }); } catch { /* best-effort */ }
performed = await applyRecreateStrategy(projectPath, worktreePath, config);
}
}
break;
case 'recreate':
performed = await applyRecreateStrategy(projectPath, worktreePath, config);
@@ -434,19 +453,27 @@ function applySymlinkStrategy(projectPath: string, worktreePath: string, config:
}
}
/** Marker file written inside a recreated venv to indicate setup completed successfully. */
const VENV_SETUP_COMPLETE_MARKER = '.setup_complete';
/**
* Apply recreate strategy: create a fresh virtual environment in the worktree.
*
* Python venvs cannot be symlinked due to CPython bug #106045 — pyvenv.cfg
* discovery does not resolve symlinks, so paths resolve relative to the
* symlink target instead of the worktree.
* Used as a fallback when venv symlinking fails (CPython bug #106045).
* Writes a completion marker so incomplete venvs can be detected and rebuilt.
*/
async function applyRecreateStrategy(projectPath: string, worktreePath: string, config: DependencyConfig): Promise<boolean> {
const venvPath = path.join(worktreePath, config.sourceRelPath);
const markerPath = path.join(venvPath, VENV_SETUP_COMPLETE_MARKER);
if (existsSync(venvPath)) {
debugLog('[TerminalWorktree] Skipping recreate', config.sourceRelPath, '- already exists');
return false;
if (existsSync(markerPath)) {
debugLog('[TerminalWorktree] Skipping recreate', config.sourceRelPath, '- already complete (marker present)');
return false;
}
// Venv exists but marker is missing — incomplete, remove and rebuild
debugLog('[TerminalWorktree] Removing incomplete venv', config.sourceRelPath, '(no marker)');
try { rmSync(venvPath, { recursive: true, force: true }); } catch { /* best-effort */ }
}
// Detect Python executable from the source venv or fall back to system Python
@@ -514,7 +541,7 @@ async function applyRecreateStrategy(projectPath: string, worktreePath: string,
debugLog('[TerminalWorktree] Installing deps from', config.requirementsFile);
await execFileAsync(pipExec, installArgs, {
encoding: 'utf-8',
timeout: 120000,
timeout: 300000,
});
} catch (error) {
if (isTimeoutError(error)) {
@@ -533,6 +560,9 @@ async function applyRecreateStrategy(projectPath: string, worktreePath: string,
}
}
// Write completion marker so future runs know this venv is complete
try { writeFileSync(markerPath, ''); } catch { /* non-fatal */ }
debugLog('[TerminalWorktree] Recreated venv at', config.sourceRelPath);
return true;
}
tests/test_worktree_dependencies.py
+137 -11
View File
@@ -69,13 +69,13 @@ class TestDependencyShareConfig:
"""Config creates with all fields populated."""
config = DependencyShareConfig(
dep_type="venv",
strategy=DependencyStrategy.RECREATE,
strategy=DependencyStrategy.SYMLINK,
source_rel_path=".venv",
requirements_file="requirements.txt",
package_manager="uv",
)
assert config.dep_type == "venv"
assert config.strategy == DependencyStrategy.RECREATE
assert config.strategy == DependencyStrategy.SYMLINK
assert config.source_rel_path == ".venv"
assert config.requirements_file == "requirements.txt"
assert config.package_manager == "uv"
@@ -88,13 +88,13 @@ class TestDefaultStrategyMap:
"""node_modules maps to SYMLINK."""
assert DEFAULT_STRATEGY_MAP["node_modules"] == DependencyStrategy.SYMLINK
def test_venv_is_recreate(self):
"""venv maps to RECREATE."""
assert DEFAULT_STRATEGY_MAP["venv"] == DependencyStrategy.RECREATE
def test_venv_is_symlink(self):
"""venv maps to SYMLINK (fast worktree creation with health check fallback)."""
assert DEFAULT_STRATEGY_MAP["venv"] == DependencyStrategy.SYMLINK
def test_dot_venv_is_recreate(self):
""".venv maps to RECREATE."""
assert DEFAULT_STRATEGY_MAP[".venv"] == DependencyStrategy.RECREATE
def test_dot_venv_is_symlink(self):
""".venv maps to SYMLINK (fast worktree creation with health check fallback)."""
assert DEFAULT_STRATEGY_MAP[".venv"] == DependencyStrategy.SYMLINK
def test_vendor_php_is_symlink(self):
"""vendor_php maps to SYMLINK."""
@@ -138,7 +138,7 @@ class TestGetDependencyConfigs:
by_type = {c.dep_type: c for c in configs}
assert by_type["node_modules"].strategy == DependencyStrategy.SYMLINK
assert by_type["node_modules"].source_rel_path == "node_modules"
assert by_type["venv"].strategy == DependencyStrategy.RECREATE
assert by_type["venv"].strategy == DependencyStrategy.SYMLINK
assert by_type["venv"].source_rel_path == "apps/backend/.venv"
assert by_type["venv"].requirements_file == "requirements.txt"
assert by_type["venv"].package_manager == "uv"
@@ -238,12 +238,12 @@ class TestGetDependencyConfigs:
assert "services/worker/.venv" in paths
api_config = paths["services/api/.venv"]
assert api_config.strategy == DependencyStrategy.RECREATE
assert api_config.strategy == DependencyStrategy.SYMLINK
assert api_config.package_manager == "pip"
assert api_config.requirements_file == "requirements.txt"
worker_config = paths["services/worker/.venv"]
assert worker_config.strategy == DependencyStrategy.RECREATE
assert worker_config.strategy == DependencyStrategy.SYMLINK
assert worker_config.package_manager == "uv"
assert worker_config.requirements_file == "pyproject.toml"
@@ -582,6 +582,132 @@ class TestSetupWorktreeDependencies:
assert not (worktree_path / "node_modules").is_symlink()
class TestVenvSymlinkWithHealthCheck:
"""Tests for venv symlink strategy with health check and fallback to recreate."""
def test_venv_symlinked_when_source_exists(self, tmp_path: Path):
"""Venv is symlinked (not recreated) when source venv exists."""
from core.workspace.setup import setup_worktree_dependencies
project_dir = tmp_path / "project"
project_dir.mkdir()
venv_dir = project_dir / ".venv"
venv_dir.mkdir()
# Create a minimal venv structure so the symlink target looks real
(venv_dir / "bin").mkdir()
(venv_dir / "lib").mkdir()
worktree_path = tmp_path / "worktree"
worktree_path.mkdir()
project_index = {
"dependency_locations": [
{"type": ".venv", "path": ".venv", "service": "backend"},
]
}
results = setup_worktree_dependencies(project_dir, worktree_path, project_index)
target = worktree_path / ".venv"
# The symlink should have been created (regardless of health check outcome)
assert target.exists() or target.is_symlink()
def test_venv_health_check_fallback_to_recreate(self, tmp_path: Path):
"""When symlinked venv health check fails, falls back to recreate."""
from core.workspace.setup import setup_worktree_dependencies
project_dir = tmp_path / "project"
project_dir.mkdir()
# Create a source venv that has no python binary (health check will fail)
venv_dir = project_dir / ".venv"
venv_dir.mkdir()
worktree_path = tmp_path / "worktree"
worktree_path.mkdir()
project_index = {
"dependency_locations": [
{"type": ".venv", "path": ".venv", "service": "backend"},
]
}
# This should symlink, then health check fails (no python binary),
# then fall back to recreate (which will also fail since no real python
# in source). The important thing is it doesn't raise.
results = setup_worktree_dependencies(project_dir, worktree_path, project_index)
# Should not crash
assert isinstance(results, dict)
class TestRecreateStrategyMarker:
"""Tests for the .setup_complete marker in the recreate strategy."""
def test_marker_constant_defined(self):
"""VENV_SETUP_COMPLETE_MARKER is defined."""
from core.workspace.setup import VENV_SETUP_COMPLETE_MARKER
assert VENV_SETUP_COMPLETE_MARKER == ".setup_complete"
def test_incomplete_venv_detected_and_removed(self, tmp_path: Path):
"""Venv without marker is detected as incomplete."""
from core.workspace.setup import _apply_recreate_strategy, VENV_SETUP_COMPLETE_MARKER
from core.workspace.models import DependencyShareConfig, DependencyStrategy
project_dir = tmp_path / "project"
project_dir.mkdir()
worktree_path = tmp_path / "worktree"
worktree_path.mkdir()
# Create an incomplete venv (no marker)
incomplete_venv = worktree_path / ".venv"
incomplete_venv.mkdir()
(incomplete_venv / "bin").mkdir()
config = DependencyShareConfig(
dep_type=".venv",
strategy=DependencyStrategy.RECREATE,
source_rel_path=".venv",
)
# Will try to recreate (remove incomplete + rebuild). May fail due to
# no real python, but the incomplete venv should be removed.
_apply_recreate_strategy(project_dir, worktree_path, config)
# The incomplete venv without marker should have been removed
# (recreation may or may not succeed depending on Python availability)
if incomplete_venv.exists():
# If it was recreated successfully, marker should exist
assert (incomplete_venv / VENV_SETUP_COMPLETE_MARKER).exists()
def test_complete_venv_skipped(self, tmp_path: Path):
"""Venv with marker is skipped (not rebuilt)."""
from core.workspace.setup import _apply_recreate_strategy, VENV_SETUP_COMPLETE_MARKER
from core.workspace.models import DependencyShareConfig, DependencyStrategy
project_dir = tmp_path / "project"
project_dir.mkdir()
worktree_path = tmp_path / "worktree"
worktree_path.mkdir()
# Create a complete venv (with marker)
complete_venv = worktree_path / ".venv"
complete_venv.mkdir()
(complete_venv / VENV_SETUP_COMPLETE_MARKER).touch()
# Add a canary file to verify the venv wasn't rebuilt
(complete_venv / "canary.txt").write_text("original")
config = DependencyShareConfig(
dep_type=".venv",
strategy=DependencyStrategy.RECREATE,
source_rel_path=".venv",
)
result = _apply_recreate_strategy(project_dir, worktree_path, config)
assert result is False # Skipped
# Canary file should still be present (not rebuilt)
assert (complete_venv / "canary.txt").read_text() == "original"
class TestSymlinkNodeModulesToWorktreeBackwardCompat:
"""Tests for symlink_node_modules_to_worktree() backward compatibility."""