AndyMik90
588 lines
19 KiB
Python
588 lines
19 KiB
Python
"""
|
|
Smart Rollback and Recovery System
|
|
===================================
|
|
|
|
Automatic recovery from build failures, stuck loops, and broken builds.
|
|
Enables true "walk away" automation by detecting and recovering from common failure modes.
|
|
|
|
Key Features:
|
|
- Automatic rollback to last working state
|
|
- Circular fix detection (prevents infinite loops)
|
|
- Attempt history tracking across sessions
|
|
- Smart retry with different approaches
|
|
- Escalation to human when stuck
|
|
"""
|
|
|
|
import json
|
|
import subprocess
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
|
|
class FailureType(Enum):
|
|
"""Types of failures that can occur during autonomous builds."""
|
|
BROKEN_BUILD = "broken_build" # Code doesn't compile/run
|
|
VERIFICATION_FAILED = "verification_failed" # Chunk verification failed
|
|
CIRCULAR_FIX = "circular_fix" # Same fix attempted multiple times
|
|
CONTEXT_EXHAUSTED = "context_exhausted" # Ran out of context mid-chunk
|
|
UNKNOWN = "unknown"
|
|
|
|
|
|
@dataclass
|
|
class RecoveryAction:
|
|
"""Action to take in response to a failure."""
|
|
action: str # "rollback", "retry", "skip", "escalate"
|
|
target: str # commit hash, chunk id, or message
|
|
reason: str
|
|
|
|
|
|
class RecoveryManager:
|
|
"""
|
|
Manages recovery from build failures.
|
|
|
|
Responsibilities:
|
|
- Track attempt history across sessions
|
|
- Classify failures and determine recovery actions
|
|
- Rollback to working states
|
|
- Detect circular fixes (same approach repeatedly)
|
|
- Escalate stuck chunks for human intervention
|
|
"""
|
|
|
|
def __init__(self, spec_dir: Path, project_dir: Path):
|
|
"""
|
|
Initialize recovery manager.
|
|
|
|
Args:
|
|
spec_dir: Spec directory containing memory/
|
|
project_dir: Root project directory for git operations
|
|
"""
|
|
self.spec_dir = spec_dir
|
|
self.project_dir = project_dir
|
|
self.memory_dir = spec_dir / "memory"
|
|
self.attempt_history_file = self.memory_dir / "attempt_history.json"
|
|
self.build_commits_file = self.memory_dir / "build_commits.json"
|
|
|
|
# Ensure memory directory exists
|
|
self.memory_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Initialize files if they don't exist
|
|
if not self.attempt_history_file.exists():
|
|
self._init_attempt_history()
|
|
|
|
if not self.build_commits_file.exists():
|
|
self._init_build_commits()
|
|
|
|
def _init_attempt_history(self) -> None:
|
|
"""Initialize the attempt history file."""
|
|
initial_data = {
|
|
"chunks": {},
|
|
"stuck_chunks": [],
|
|
"metadata": {
|
|
"created_at": datetime.now().isoformat(),
|
|
"last_updated": datetime.now().isoformat()
|
|
}
|
|
}
|
|
with open(self.attempt_history_file, "w") as f:
|
|
json.dump(initial_data, f, indent=2)
|
|
|
|
def _init_build_commits(self) -> None:
|
|
"""Initialize the build commits tracking file."""
|
|
initial_data = {
|
|
"commits": [],
|
|
"last_good_commit": None,
|
|
"metadata": {
|
|
"created_at": datetime.now().isoformat(),
|
|
"last_updated": datetime.now().isoformat()
|
|
}
|
|
}
|
|
with open(self.build_commits_file, "w") as f:
|
|
json.dump(initial_data, f, indent=2)
|
|
|
|
def _load_attempt_history(self) -> dict:
|
|
"""Load attempt history from JSON file."""
|
|
try:
|
|
with open(self.attempt_history_file, "r") as f:
|
|
return json.load(f)
|
|
except (json.JSONDecodeError, IOError):
|
|
self._init_attempt_history()
|
|
with open(self.attempt_history_file, "r") as f:
|
|
return json.load(f)
|
|
|
|
def _save_attempt_history(self, data: dict) -> None:
|
|
"""Save attempt history to JSON file."""
|
|
data["metadata"]["last_updated"] = datetime.now().isoformat()
|
|
with open(self.attempt_history_file, "w") as f:
|
|
json.dump(data, f, indent=2)
|
|
|
|
def _load_build_commits(self) -> dict:
|
|
"""Load build commits from JSON file."""
|
|
try:
|
|
with open(self.build_commits_file, "r") as f:
|
|
return json.load(f)
|
|
except (json.JSONDecodeError, IOError):
|
|
self._init_build_commits()
|
|
with open(self.build_commits_file, "r") as f:
|
|
return json.load(f)
|
|
|
|
def _save_build_commits(self, data: dict) -> None:
|
|
"""Save build commits to JSON file."""
|
|
data["metadata"]["last_updated"] = datetime.now().isoformat()
|
|
with open(self.build_commits_file, "w") as f:
|
|
json.dump(data, f, indent=2)
|
|
|
|
def classify_failure(self, error: str, chunk_id: str) -> FailureType:
|
|
"""
|
|
Classify what type of failure occurred.
|
|
|
|
Args:
|
|
error: Error message or description
|
|
chunk_id: ID of the chunk that failed
|
|
|
|
Returns:
|
|
FailureType enum value
|
|
"""
|
|
error_lower = error.lower()
|
|
|
|
# Check for broken build indicators
|
|
build_errors = [
|
|
"syntax error", "compilation error", "module not found",
|
|
"import error", "cannot find module", "unexpected token",
|
|
"indentation error", "parse error"
|
|
]
|
|
if any(be in error_lower for be in build_errors):
|
|
return FailureType.BROKEN_BUILD
|
|
|
|
# Check for verification failures
|
|
verification_errors = [
|
|
"verification failed", "expected", "assertion",
|
|
"test failed", "status code"
|
|
]
|
|
if any(ve in error_lower for ve in verification_errors):
|
|
return FailureType.VERIFICATION_FAILED
|
|
|
|
# Check for context exhaustion
|
|
context_errors = [
|
|
"context", "token limit", "maximum length"
|
|
]
|
|
if any(ce in error_lower for ce in context_errors):
|
|
return FailureType.CONTEXT_EXHAUSTED
|
|
|
|
# Check for circular fixes (will be determined by attempt history)
|
|
if self.is_circular_fix(chunk_id, error):
|
|
return FailureType.CIRCULAR_FIX
|
|
|
|
return FailureType.UNKNOWN
|
|
|
|
def get_attempt_count(self, chunk_id: str) -> int:
|
|
"""
|
|
Get how many times this chunk has been attempted.
|
|
|
|
Args:
|
|
chunk_id: ID of the chunk
|
|
|
|
Returns:
|
|
Number of attempts
|
|
"""
|
|
history = self._load_attempt_history()
|
|
chunk_data = history["chunks"].get(chunk_id, {})
|
|
return len(chunk_data.get("attempts", []))
|
|
|
|
def record_attempt(
|
|
self,
|
|
chunk_id: str,
|
|
session: int,
|
|
success: bool,
|
|
approach: str,
|
|
error: Optional[str] = None
|
|
) -> None:
|
|
"""
|
|
Record an attempt at a chunk.
|
|
|
|
Args:
|
|
chunk_id: ID of the chunk
|
|
session: Session number
|
|
success: Whether the attempt succeeded
|
|
approach: Description of the approach taken
|
|
error: Error message if failed
|
|
"""
|
|
history = self._load_attempt_history()
|
|
|
|
# Initialize chunk entry if it doesn't exist
|
|
if chunk_id not in history["chunks"]:
|
|
history["chunks"][chunk_id] = {
|
|
"attempts": [],
|
|
"status": "pending"
|
|
}
|
|
|
|
# Add the attempt
|
|
attempt = {
|
|
"session": session,
|
|
"timestamp": datetime.now().isoformat(),
|
|
"approach": approach,
|
|
"success": success,
|
|
"error": error
|
|
}
|
|
history["chunks"][chunk_id]["attempts"].append(attempt)
|
|
|
|
# Update status
|
|
if success:
|
|
history["chunks"][chunk_id]["status"] = "completed"
|
|
else:
|
|
history["chunks"][chunk_id]["status"] = "failed"
|
|
|
|
self._save_attempt_history(history)
|
|
|
|
def is_circular_fix(self, chunk_id: str, current_approach: str) -> bool:
|
|
"""
|
|
Detect if we're trying the same approach repeatedly.
|
|
|
|
Args:
|
|
chunk_id: ID of the chunk
|
|
current_approach: Description of current approach
|
|
|
|
Returns:
|
|
True if this appears to be a circular fix attempt
|
|
"""
|
|
history = self._load_attempt_history()
|
|
chunk_data = history["chunks"].get(chunk_id, {})
|
|
attempts = chunk_data.get("attempts", [])
|
|
|
|
if len(attempts) < 2:
|
|
return False
|
|
|
|
# Check if last 3 attempts used similar approaches
|
|
# Simple similarity check: look for repeated keywords
|
|
recent_attempts = attempts[-3:] if len(attempts) >= 3 else attempts
|
|
|
|
# Extract key terms from current approach (ignore common words)
|
|
stop_words = {'with', 'using', 'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'trying'}
|
|
current_keywords = set(
|
|
word for word in current_approach.lower().split()
|
|
if word not in stop_words
|
|
)
|
|
|
|
similar_count = 0
|
|
for attempt in recent_attempts:
|
|
attempt_keywords = set(
|
|
word for word in attempt["approach"].lower().split()
|
|
if word not in stop_words
|
|
)
|
|
|
|
# Calculate Jaccard similarity (intersection over union)
|
|
overlap = len(current_keywords & attempt_keywords)
|
|
total = len(current_keywords | attempt_keywords)
|
|
|
|
if total > 0:
|
|
similarity = overlap / total
|
|
# If >30% of meaningful words overlap, consider it similar
|
|
# This catches key technical terms appearing repeatedly
|
|
# (e.g., "async await" across multiple attempts)
|
|
if similarity > 0.3:
|
|
similar_count += 1
|
|
|
|
# If 2+ recent attempts were similar to current approach, it's circular
|
|
return similar_count >= 2
|
|
|
|
def determine_recovery_action(
|
|
self,
|
|
failure_type: FailureType,
|
|
chunk_id: str
|
|
) -> RecoveryAction:
|
|
"""
|
|
Decide what to do based on failure type and history.
|
|
|
|
Args:
|
|
failure_type: Type of failure that occurred
|
|
chunk_id: ID of the chunk that failed
|
|
|
|
Returns:
|
|
RecoveryAction describing what to do
|
|
"""
|
|
attempt_count = self.get_attempt_count(chunk_id)
|
|
|
|
if failure_type == FailureType.BROKEN_BUILD:
|
|
# Broken build: rollback to last good state
|
|
last_good = self.get_last_good_commit()
|
|
if last_good:
|
|
return RecoveryAction(
|
|
action="rollback",
|
|
target=last_good,
|
|
reason=f"Build broken in chunk {chunk_id}, rolling back to working state"
|
|
)
|
|
else:
|
|
return RecoveryAction(
|
|
action="escalate",
|
|
target=chunk_id,
|
|
reason="Build broken and no good commit found to rollback to"
|
|
)
|
|
|
|
elif failure_type == FailureType.VERIFICATION_FAILED:
|
|
# Verification failed: retry with different approach if < 3 attempts
|
|
if attempt_count < 3:
|
|
return RecoveryAction(
|
|
action="retry",
|
|
target=chunk_id,
|
|
reason=f"Verification failed, retry with different approach (attempt {attempt_count + 1}/3)"
|
|
)
|
|
else:
|
|
return RecoveryAction(
|
|
action="skip",
|
|
target=chunk_id,
|
|
reason=f"Verification failed after {attempt_count} attempts, marking as stuck"
|
|
)
|
|
|
|
elif failure_type == FailureType.CIRCULAR_FIX:
|
|
# Circular fix detected: skip and escalate
|
|
return RecoveryAction(
|
|
action="skip",
|
|
target=chunk_id,
|
|
reason="Circular fix detected - same approach tried multiple times"
|
|
)
|
|
|
|
elif failure_type == FailureType.CONTEXT_EXHAUSTED:
|
|
# Context exhausted: commit current progress and continue
|
|
return RecoveryAction(
|
|
action="continue",
|
|
target=chunk_id,
|
|
reason="Context exhausted, will commit progress and continue in next session"
|
|
)
|
|
|
|
else: # UNKNOWN
|
|
# Unknown error: retry once, then escalate
|
|
if attempt_count < 2:
|
|
return RecoveryAction(
|
|
action="retry",
|
|
target=chunk_id,
|
|
reason=f"Unknown error, retrying (attempt {attempt_count + 1}/2)"
|
|
)
|
|
else:
|
|
return RecoveryAction(
|
|
action="escalate",
|
|
target=chunk_id,
|
|
reason=f"Unknown error persists after {attempt_count} attempts"
|
|
)
|
|
|
|
def get_last_good_commit(self) -> Optional[str]:
|
|
"""
|
|
Find the most recent commit where build was working.
|
|
|
|
Returns:
|
|
Commit hash or None
|
|
"""
|
|
commits = self._load_build_commits()
|
|
return commits.get("last_good_commit")
|
|
|
|
def record_good_commit(self, commit_hash: str, chunk_id: str) -> None:
|
|
"""
|
|
Record a commit where the build was working.
|
|
|
|
Args:
|
|
commit_hash: Git commit hash
|
|
chunk_id: Chunk that was successfully completed
|
|
"""
|
|
commits = self._load_build_commits()
|
|
|
|
commit_record = {
|
|
"hash": commit_hash,
|
|
"chunk_id": chunk_id,
|
|
"timestamp": datetime.now().isoformat()
|
|
}
|
|
|
|
commits["commits"].append(commit_record)
|
|
commits["last_good_commit"] = commit_hash
|
|
|
|
self._save_build_commits(commits)
|
|
|
|
def rollback_to_commit(self, commit_hash: str) -> bool:
|
|
"""
|
|
Rollback to a specific commit.
|
|
|
|
Args:
|
|
commit_hash: Git commit hash to rollback to
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
try:
|
|
# Use git reset --hard to rollback
|
|
result = subprocess.run(
|
|
["git", "reset", "--hard", commit_hash],
|
|
cwd=self.project_dir,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True
|
|
)
|
|
return True
|
|
except subprocess.CalledProcessError as e:
|
|
print(f"Error rolling back to {commit_hash}: {e.stderr}")
|
|
return False
|
|
|
|
def mark_chunk_stuck(self, chunk_id: str, reason: str) -> None:
|
|
"""
|
|
Mark a chunk as needing human intervention.
|
|
|
|
Args:
|
|
chunk_id: ID of the chunk
|
|
reason: Why it's stuck
|
|
"""
|
|
history = self._load_attempt_history()
|
|
|
|
stuck_entry = {
|
|
"chunk_id": chunk_id,
|
|
"reason": reason,
|
|
"escalated_at": datetime.now().isoformat(),
|
|
"attempt_count": self.get_attempt_count(chunk_id)
|
|
}
|
|
|
|
# Check if already in stuck list
|
|
existing = [s for s in history["stuck_chunks"] if s["chunk_id"] == chunk_id]
|
|
if not existing:
|
|
history["stuck_chunks"].append(stuck_entry)
|
|
|
|
# Update chunk status
|
|
if chunk_id in history["chunks"]:
|
|
history["chunks"][chunk_id]["status"] = "stuck"
|
|
|
|
self._save_attempt_history(history)
|
|
|
|
def get_stuck_chunks(self) -> list[dict]:
|
|
"""
|
|
Get all chunks marked as stuck.
|
|
|
|
Returns:
|
|
List of stuck chunk entries
|
|
"""
|
|
history = self._load_attempt_history()
|
|
return history.get("stuck_chunks", [])
|
|
|
|
def get_chunk_history(self, chunk_id: str) -> dict:
|
|
"""
|
|
Get the attempt history for a specific chunk.
|
|
|
|
Args:
|
|
chunk_id: ID of the chunk
|
|
|
|
Returns:
|
|
Chunk history dict with attempts
|
|
"""
|
|
history = self._load_attempt_history()
|
|
return history["chunks"].get(chunk_id, {"attempts": [], "status": "pending"})
|
|
|
|
def get_recovery_hints(self, chunk_id: str) -> list[str]:
|
|
"""
|
|
Get hints for recovery based on previous attempts.
|
|
|
|
Args:
|
|
chunk_id: ID of the chunk
|
|
|
|
Returns:
|
|
List of hint strings
|
|
"""
|
|
chunk_history = self.get_chunk_history(chunk_id)
|
|
attempts = chunk_history.get("attempts", [])
|
|
|
|
if not attempts:
|
|
return ["This is the first attempt at this chunk"]
|
|
|
|
hints = [f"Previous attempts: {len(attempts)}"]
|
|
|
|
# Add info about what was tried
|
|
for i, attempt in enumerate(attempts[-3:], 1):
|
|
hints.append(
|
|
f"Attempt {i}: {attempt['approach']} - "
|
|
f"{'SUCCESS' if attempt['success'] else 'FAILED'}"
|
|
)
|
|
if attempt.get("error"):
|
|
hints.append(f" Error: {attempt['error'][:100]}")
|
|
|
|
# Add guidance
|
|
if len(attempts) >= 2:
|
|
hints.append("\n⚠️ IMPORTANT: Try a DIFFERENT approach than previous attempts")
|
|
hints.append("Consider: different library, different pattern, or simpler implementation")
|
|
|
|
return hints
|
|
|
|
def clear_stuck_chunks(self) -> None:
|
|
"""Clear all stuck chunks (for manual resolution)."""
|
|
history = self._load_attempt_history()
|
|
history["stuck_chunks"] = []
|
|
self._save_attempt_history(history)
|
|
|
|
def reset_chunk(self, chunk_id: str) -> None:
|
|
"""
|
|
Reset a chunk's attempt history.
|
|
|
|
Args:
|
|
chunk_id: ID of the chunk to reset
|
|
"""
|
|
history = self._load_attempt_history()
|
|
|
|
# Clear attempt history
|
|
if chunk_id in history["chunks"]:
|
|
history["chunks"][chunk_id] = {
|
|
"attempts": [],
|
|
"status": "pending"
|
|
}
|
|
|
|
# Remove from stuck chunks
|
|
history["stuck_chunks"] = [
|
|
s for s in history["stuck_chunks"]
|
|
if s["chunk_id"] != chunk_id
|
|
]
|
|
|
|
self._save_attempt_history(history)
|
|
|
|
|
|
# Utility functions for integration with agent.py
|
|
|
|
def check_and_recover(
|
|
spec_dir: Path,
|
|
project_dir: Path,
|
|
chunk_id: str,
|
|
error: Optional[str] = None
|
|
) -> Optional[RecoveryAction]:
|
|
"""
|
|
Check if recovery is needed and return appropriate action.
|
|
|
|
Args:
|
|
spec_dir: Spec directory
|
|
project_dir: Project directory
|
|
chunk_id: Current chunk ID
|
|
error: Error message if any
|
|
|
|
Returns:
|
|
RecoveryAction if recovery needed, None otherwise
|
|
"""
|
|
if not error:
|
|
return None
|
|
|
|
manager = RecoveryManager(spec_dir, project_dir)
|
|
failure_type = manager.classify_failure(error, chunk_id)
|
|
|
|
return manager.determine_recovery_action(failure_type, chunk_id)
|
|
|
|
|
|
def get_recovery_context(spec_dir: Path, project_dir: Path, chunk_id: str) -> dict:
|
|
"""
|
|
Get recovery context for a chunk (for prompt generation).
|
|
|
|
Args:
|
|
spec_dir: Spec directory
|
|
project_dir: Project directory
|
|
chunk_id: Chunk ID
|
|
|
|
Returns:
|
|
Dict with recovery hints and history
|
|
"""
|
|
manager = RecoveryManager(spec_dir, project_dir)
|
|
|
|
return {
|
|
"attempt_count": manager.get_attempt_count(chunk_id),
|
|
"hints": manager.get_recovery_hints(chunk_id),
|
|
"chunk_history": manager.get_chunk_history(chunk_id),
|
|
"stuck_chunks": manager.get_stuck_chunks()
|
|
}
|