Files
Aperant/apps/backend/agents/session.py
T
AndyMik90 4a75ea9f99 fix: handle unknown SDK message types (rate_limit_event) to prevent session crashes
The Claude CLI emits message types like rate_limit_event that the SDK's
message_parser doesn't recognize, causing MessageParseError to kill the
entire agent session stream. This adds two layers of defense:

1. Monkey-patch SDK's parse_message to convert unknown types into safe
   SystemMessage objects instead of raising
2. safe_receive_messages() wrapper around receive_response() that filters
   patched messages and catches stream-level errors gracefully

Applied to all agent consumers: session, qa_reviewer, qa_fixer, and
agent_runner. Also bumps minimum SDK to >=0.1.39.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-20 11:16:59 +01:00

728 lines
29 KiB
Python

"""
Agent Session Management
========================
Handles running agent sessions and post-session processing including
memory updates, recovery tracking, and Linear integration.
"""
import logging
from pathlib import Path
from claude_agent_sdk import ClaudeSDKClient
from core.error_utils import (
is_authentication_error,
is_rate_limit_error,
is_tool_concurrency_error,
safe_receive_messages,
)
from core.file_utils import write_json_atomic
from debug import debug, debug_detailed, debug_error, debug_section, debug_success
from insight_extractor import extract_session_insights
from linear_updater import (
linear_subtask_completed,
linear_subtask_failed,
)
from progress import (
count_subtasks_detailed,
is_build_complete,
)
from recovery import RecoveryManager, check_and_recover, reset_subtask
from security.tool_input_validator import get_safe_tool_input
from task_logger import (
LogEntryType,
LogPhase,
get_task_logger,
)
from ui import (
StatusManager,
muted,
print_key_value,
print_status,
)
from .base import sanitize_error_message
from .memory_manager import save_session_memory
from .utils import (
find_subtask_in_plan,
get_commit_count,
get_latest_commit,
load_implementation_plan,
sync_spec_to_source,
)
logger = logging.getLogger(__name__)
def _execute_recovery_action(
recovery_action,
recovery_manager: RecoveryManager,
spec_dir: Path,
project_dir: Path,
subtask_id: str,
) -> None:
"""Execute a recovery action (rollback/retry/skip/escalate)."""
if not recovery_action:
return
print_status(f"Recovery action: {recovery_action.action}", "info")
print_status(f"Reason: {recovery_action.reason}", "info")
if recovery_action.action == "rollback":
print_status(f"Rolling back to {recovery_action.target[:8]}", "warning")
if recovery_manager.rollback_to_commit(recovery_action.target):
print_status("Rollback successful", "success")
else:
print_status("Rollback failed", "error")
elif recovery_action.action == "retry":
print_status(f"Resetting subtask {subtask_id} for retry", "info")
reset_subtask(spec_dir, project_dir, subtask_id)
print_status("Subtask reset - will retry with different approach", "success")
elif recovery_action.action in ("skip", "escalate"):
print_status(f"Marking subtask {subtask_id} as stuck", "warning")
recovery_manager.mark_subtask_stuck(subtask_id, recovery_action.reason)
print_status("Subtask marked for human intervention", "warning")
async def post_session_processing(
spec_dir: Path,
project_dir: Path,
subtask_id: str,
session_num: int,
commit_before: str | None,
commit_count_before: int,
recovery_manager: RecoveryManager,
linear_enabled: bool = False,
status_manager: StatusManager | None = None,
source_spec_dir: Path | None = None,
error_info: dict | None = None,
) -> bool:
"""
Process session results and update memory automatically.
This runs in Python (100% reliable) instead of relying on agent compliance.
Args:
spec_dir: Spec directory containing memory/
project_dir: Project root for git operations
subtask_id: The subtask that was being worked on
session_num: Current session number
commit_before: Git commit hash before session
commit_count_before: Number of commits before session
recovery_manager: Recovery manager instance
linear_enabled: Whether Linear integration is enabled
status_manager: Optional status manager for ccstatusline
source_spec_dir: Original spec directory (for syncing back from worktree)
error_info: Error information from run_agent_session (for rate limit detection)
Returns:
True if subtask was completed successfully
"""
print()
print(muted("--- Post-Session Processing ---"))
# Sync implementation plan back to source (for worktree mode)
if sync_spec_to_source(spec_dir, source_spec_dir):
print_status("Implementation plan synced to main project", "success")
# Check if implementation plan was updated
plan = load_implementation_plan(spec_dir)
if not plan:
print(" Warning: Could not load implementation plan")
return False
subtask = find_subtask_in_plan(plan, subtask_id)
if not subtask:
print(f" Warning: Subtask {subtask_id} not found in plan")
return False
subtask_status = subtask.get("status", "pending")
# Check for new commits
commit_after = get_latest_commit(project_dir)
commit_count_after = get_commit_count(project_dir)
new_commits = commit_count_after - commit_count_before
print_key_value("Subtask status", subtask_status)
print_key_value("New commits", str(new_commits))
if subtask_status == "completed":
# Success! Record the attempt and good commit
print_status(f"Subtask {subtask_id} completed successfully", "success")
# Update status file
if status_manager:
subtasks = count_subtasks_detailed(spec_dir)
status_manager.update_subtasks(
completed=subtasks["completed"],
total=subtasks["total"],
in_progress=0,
)
# Record successful attempt
recovery_manager.record_attempt(
subtask_id=subtask_id,
session=session_num,
success=True,
approach=f"Implemented: {subtask.get('description', 'subtask')[:100]}",
)
# Record good commit for rollback safety
if commit_after and commit_after != commit_before:
recovery_manager.record_good_commit(commit_after, subtask_id)
print_status(f"Recorded good commit: {commit_after[:8]}", "success")
# Record Linear session result (if enabled)
if linear_enabled:
# Get progress counts for the comment
subtasks_detail = count_subtasks_detailed(spec_dir)
await linear_subtask_completed(
spec_dir=spec_dir,
subtask_id=subtask_id,
completed_count=subtasks_detail["completed"],
total_count=subtasks_detail["total"],
)
print_status("Linear progress recorded", "success")
# Extract rich insights from session (LLM-powered analysis)
try:
extracted_insights = await extract_session_insights(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
session_num=session_num,
commit_before=commit_before,
commit_after=commit_after,
success=True,
recovery_manager=recovery_manager,
)
insight_count = len(extracted_insights.get("file_insights", []))
pattern_count = len(extracted_insights.get("patterns_discovered", []))
if insight_count > 0 or pattern_count > 0:
print_status(
f"Extracted {insight_count} file insights, {pattern_count} patterns",
"success",
)
except Exception as e:
logger.warning(f"Insight extraction failed: {e}")
extracted_insights = None
# Save session memory (Graphiti=primary, file-based=fallback)
try:
save_success, storage_type = await save_session_memory(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
session_num=session_num,
success=True,
subtasks_completed=[subtask_id],
discoveries=extracted_insights,
)
if save_success:
if storage_type == "graphiti":
print_status("Session saved to Graphiti memory", "success")
else:
print_status(
"Session saved to file-based memory (fallback)", "info"
)
else:
print_status("Failed to save session memory", "warning")
except Exception as e:
logger.warning(f"Error saving session memory: {e}")
print_status("Memory save failed", "warning")
return True
elif subtask_status == "in_progress":
# Session ended without completion
print_status(f"Subtask {subtask_id} still in progress", "warning")
recovery_manager.record_attempt(
subtask_id=subtask_id,
session=session_num,
success=False,
approach="Session ended with subtask in_progress",
error="Subtask not marked as completed",
)
# Check if this was a concurrency error - if so, reset subtask to pending for retry
is_concurrency_error = (
error_info and error_info.get("type") == "tool_concurrency"
)
if is_concurrency_error:
print_status(
f"Rate limit detected - resetting subtask {subtask_id} to pending for retry",
"info",
)
# Use recovery system's reset_subtask for consistency
reset_subtask(spec_dir, project_dir, subtask_id)
# Also reset in implementation plan
plan = load_implementation_plan(spec_dir)
if plan:
# Find and reset the subtask
subtask_found = False
for phase in plan.get("phases", []):
for subtask in phase.get("subtasks", []):
if subtask.get("id") == subtask_id:
# Reset subtask to pending state
subtask["status"] = "pending"
subtask["started_at"] = None
subtask["completed_at"] = None
subtask_found = True
break
if subtask_found:
break
if subtask_found:
# Save plan atomically to prevent corruption
try:
plan_path = spec_dir / "implementation_plan.json"
write_json_atomic(plan_path, plan, indent=2)
print_status(
f"Subtask {subtask_id} reset to pending status", "success"
)
except Exception as e:
logger.error(
f"Failed to save implementation plan after reset: {e}"
)
print_status("Failed to save plan after reset", "error")
else:
print_status(
f"Warning: Could not find subtask {subtask_id} in plan",
"warning",
)
else:
print_status(
"Warning: Could not load implementation plan for reset", "warning"
)
else:
# Non-rate-limit error - use automatic recovery flow
error_message = (
error_info.get("message", "Subtask not marked as completed")
if error_info
else "Subtask not marked as completed"
)
recovery_action = check_and_recover(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
error=error_message,
)
_execute_recovery_action(
recovery_action, recovery_manager, spec_dir, project_dir, subtask_id
)
# Still record commit if one was made (partial progress)
if commit_after and commit_after != commit_before:
recovery_manager.record_good_commit(commit_after, subtask_id)
print_status(
f"Recorded partial progress commit: {commit_after[:8]}", "info"
)
# Record Linear session result (if enabled)
if linear_enabled:
attempt_count = recovery_manager.get_attempt_count(subtask_id)
await linear_subtask_failed(
spec_dir=spec_dir,
subtask_id=subtask_id,
attempt=attempt_count,
error_summary="Session ended without completion",
)
# Extract insights even from failed sessions (valuable for future attempts)
try:
extracted_insights = await extract_session_insights(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
session_num=session_num,
commit_before=commit_before,
commit_after=commit_after,
success=False,
recovery_manager=recovery_manager,
)
except Exception as e:
logger.debug(f"Insight extraction failed for incomplete session: {e}")
extracted_insights = None
# Save failed session memory (to track what didn't work)
try:
await save_session_memory(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
session_num=session_num,
success=False,
subtasks_completed=[],
discoveries=extracted_insights,
)
except Exception as e:
logger.debug(f"Failed to save incomplete session memory: {e}")
return False
else:
# Subtask still pending or failed
print_status(
f"Subtask {subtask_id} not completed (status: {subtask_status})", "error"
)
recovery_manager.record_attempt(
subtask_id=subtask_id,
session=session_num,
success=False,
approach="Session ended without progress",
error=f"Subtask status is {subtask_status}",
)
# Automatic recovery flow - determine and execute recovery action
error_message = f"Subtask status is {subtask_status}"
if error_info:
error_message = error_info.get("message", error_message)
recovery_action = check_and_recover(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
error=error_message,
)
_execute_recovery_action(
recovery_action, recovery_manager, spec_dir, project_dir, subtask_id
)
# Record Linear session result (if enabled)
if linear_enabled:
attempt_count = recovery_manager.get_attempt_count(subtask_id)
await linear_subtask_failed(
spec_dir=spec_dir,
subtask_id=subtask_id,
attempt=attempt_count,
error_summary=f"Subtask status: {subtask_status}",
)
# Extract insights even from completely failed sessions
try:
extracted_insights = await extract_session_insights(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
session_num=session_num,
commit_before=commit_before,
commit_after=commit_after,
success=False,
recovery_manager=recovery_manager,
)
except Exception as e:
logger.debug(f"Insight extraction failed for failed session: {e}")
extracted_insights = None
# Save failed session memory (to track what didn't work)
try:
await save_session_memory(
spec_dir=spec_dir,
project_dir=project_dir,
subtask_id=subtask_id,
session_num=session_num,
success=False,
subtasks_completed=[],
discoveries=extracted_insights,
)
except Exception as e:
logger.debug(f"Failed to save failed session memory: {e}")
return False
async def run_agent_session(
client: ClaudeSDKClient,
message: str,
spec_dir: Path,
verbose: bool = False,
phase: LogPhase = LogPhase.CODING,
) -> tuple[str, str, dict]:
"""
Run a single agent session using Claude Agent SDK.
Args:
client: Claude SDK client
message: The prompt to send
spec_dir: Spec directory path
verbose: Whether to show detailed output
phase: Current execution phase for logging
Returns:
(status, response_text, error_info) where:
- status: "continue", "complete", or "error"
- response_text: Agent's response text
- error_info: Dict with error details (empty if no error):
- "type": "tool_concurrency" or "other"
- "message": Error message string
- "exception_type": Exception class name string
"""
debug_section("session", f"Agent Session - {phase.value}")
debug(
"session",
"Starting agent session",
spec_dir=str(spec_dir),
phase=phase.value,
prompt_length=len(message),
prompt_preview=message[:200] + "..." if len(message) > 200 else message,
)
print("Sending prompt to Claude Agent SDK...\n")
# Get task logger for this spec
task_logger = get_task_logger(spec_dir)
current_tool = None
message_count = 0
tool_count = 0
try:
# Send the query
debug("session", "Sending query to Claude SDK...")
await client.query(message)
debug_success("session", "Query sent successfully")
# Collect response text and show tool use
response_text = ""
debug("session", "Starting to receive response stream...")
async for msg in safe_receive_messages(client, caller="session"):
msg_type = type(msg).__name__
message_count += 1
debug_detailed(
"session",
f"Received message #{message_count}",
msg_type=msg_type,
)
# Handle AssistantMessage (text and tool use)
if msg_type == "AssistantMessage" and hasattr(msg, "content"):
for block in msg.content:
block_type = type(block).__name__
if block_type == "TextBlock" and hasattr(block, "text"):
response_text += block.text
print(block.text, end="", flush=True)
# Log text to task logger (persist without double-printing)
if task_logger and block.text.strip():
task_logger.log(
block.text,
LogEntryType.TEXT,
phase,
print_to_console=False,
)
elif block_type == "ToolUseBlock" and hasattr(block, "name"):
tool_name = block.name
tool_input_display = None
tool_count += 1
# Safely extract tool input (handles None, non-dict, etc.)
inp = get_safe_tool_input(block)
# Extract meaningful tool input for display
if inp:
if "pattern" in inp:
tool_input_display = f"pattern: {inp['pattern']}"
elif "file_path" in inp:
fp = inp["file_path"]
if len(fp) > 50:
fp = "..." + fp[-47:]
tool_input_display = fp
elif "command" in inp:
cmd = inp["command"]
if len(cmd) > 50:
cmd = cmd[:47] + "..."
tool_input_display = cmd
elif "path" in inp:
tool_input_display = inp["path"]
debug(
"session",
f"Tool call #{tool_count}: {tool_name}",
tool_input=tool_input_display,
full_input=str(inp)[:500] if inp else None,
)
# Log tool start (handles printing too)
if task_logger:
task_logger.tool_start(
tool_name,
tool_input_display,
phase,
print_to_console=True,
)
else:
print(f"\n[Tool: {tool_name}]", flush=True)
if verbose and hasattr(block, "input"):
input_str = str(block.input)
if len(input_str) > 300:
print(f" Input: {input_str[:300]}...", flush=True)
else:
print(f" Input: {input_str}", flush=True)
current_tool = tool_name
# Handle UserMessage (tool results)
elif msg_type == "UserMessage" and hasattr(msg, "content"):
for block in msg.content:
block_type = type(block).__name__
if block_type == "ToolResultBlock":
result_content = getattr(block, "content", "")
is_error = getattr(block, "is_error", False)
# Check if this is an error (not just content containing "blocked")
if is_error and "blocked" in str(result_content).lower():
# Actual blocked command by security hook
debug_error(
"session",
f"Tool BLOCKED: {current_tool}",
result=str(result_content)[:300],
)
print(f" [BLOCKED] {result_content}", flush=True)
if task_logger and current_tool:
task_logger.tool_end(
current_tool,
success=False,
result="BLOCKED",
detail=str(result_content),
phase=phase,
)
elif is_error:
# Show errors (truncated)
error_str = str(result_content)[:500]
debug_error(
"session",
f"Tool error: {current_tool}",
error=error_str[:200],
)
print(f" [Error] {error_str}", flush=True)
if task_logger and current_tool:
# Store full error in detail for expandable view
task_logger.tool_end(
current_tool,
success=False,
result=error_str[:100],
detail=str(result_content),
phase=phase,
)
else:
# Tool succeeded
debug_detailed(
"session",
f"Tool success: {current_tool}",
result_length=len(str(result_content)),
)
if verbose:
result_str = str(result_content)[:200]
print(f" [Done] {result_str}", flush=True)
else:
print(" [Done]", flush=True)
if task_logger and current_tool:
# Store full result in detail for expandable view (only for certain tools)
# Skip storing for very large outputs like Glob results
detail_content = None
if current_tool in (
"Read",
"Grep",
"Bash",
"Edit",
"Write",
):
result_str = str(result_content)
# Only store if not too large (detail truncation happens in logger)
if (
len(result_str) < 50000
): # 50KB max before truncation
detail_content = result_str
task_logger.tool_end(
current_tool,
success=True,
detail=detail_content,
phase=phase,
)
current_tool = None
print("\n" + "-" * 70 + "\n")
# Check if build is complete
if is_build_complete(spec_dir):
debug_success(
"session",
"Session completed - build is complete",
message_count=message_count,
tool_count=tool_count,
response_length=len(response_text),
)
return "complete", response_text, {}
debug_success(
"session",
"Session completed - continuing",
message_count=message_count,
tool_count=tool_count,
response_length=len(response_text),
)
return "continue", response_text, {}
except Exception as e:
# Detect specific error types for better retry handling
is_concurrency = is_tool_concurrency_error(e)
is_rate_limit = is_rate_limit_error(e)
is_auth = is_authentication_error(e)
# Classify error type for appropriate handling
if is_concurrency:
error_type = "tool_concurrency"
elif is_rate_limit:
error_type = "rate_limit"
elif is_auth:
error_type = "authentication"
else:
error_type = "other"
debug_error(
"session",
f"Session error: {e}",
exception_type=type(e).__name__,
error_category=error_type,
message_count=message_count,
tool_count=tool_count,
)
# Sanitize error message to remove potentially sensitive data
# Must happen BEFORE printing to stdout, since stdout is captured by the frontend
sanitized_error = sanitize_error_message(str(e))
# Log errors prominently based on type
if is_concurrency:
print("\n⚠️ Tool concurrency limit reached (400 error)")
print(" Claude API limits concurrent tool use in a single request")
print(f" Error: {sanitized_error[:200]}\n")
elif is_rate_limit:
print("\n⚠️ Rate limit reached")
print(" API usage quota exceeded - waiting for reset")
print(f" Error: {sanitized_error[:200]}\n")
elif is_auth:
print("\n⚠️ Authentication error")
print(" OAuth token may be invalid or expired")
print(f" Error: {sanitized_error[:200]}\n")
else:
print(f"Error during agent session: {sanitized_error}")
if task_logger:
task_logger.log_error(f"Session error: {sanitized_error}", phase)
error_info = {
"type": error_type,
"message": sanitized_error,
"exception_type": type(e).__name__,
}
return "error", sanitized_error, error_info