diff --git a/CLAUDE.md b/CLAUDE.md index ba831a27..e16d6cb4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,17 +60,20 @@ python auto-claude/run.py --spec 001 --qa-status ### Testing ```bash -# Run all tests -pytest tests/ -v +# Install test dependencies (required first time) +cd auto-claude && uv pip install -r ../tests/requirements-test.txt + +# Run all tests (use virtual environment pytest) +auto-claude/.venv/bin/pytest tests/ -v # Run single test file -pytest tests/test_security.py -v +auto-claude/.venv/bin/pytest tests/test_security.py -v # Run specific test -pytest tests/test_security.py::test_bash_command_validation -v +auto-claude/.venv/bin/pytest tests/test_security.py::test_bash_command_validation -v # Skip slow tests -pytest tests/ -m "not slow" +auto-claude/.venv/bin/pytest tests/ -m "not slow" ``` ### Spec Validation diff --git a/auto-claude-ui/src/renderer/components/GitHubIssues.tsx b/auto-claude-ui/src/renderer/components/GitHubIssues.tsx index ea3bd466..1a5193fc 100644 --- a/auto-claude-ui/src/renderer/components/GitHubIssues.tsx +++ b/auto-claude-ui/src/renderer/components/GitHubIssues.tsx @@ -75,7 +75,7 @@ export function GitHubIssues({ onOpenSettings }: GitHubIssuesProps) {
{/* Header */} setSettings({ ...settings, ...updates })} infrastructureStatus={infrastructureStatus} isCheckingInfrastructure={isCheckingInfrastructure} isStartingFalkorDB={isStartingFalkorDB} diff --git a/auto-claude-ui/src/renderer/hooks/index.ts b/auto-claude-ui/src/renderer/hooks/index.ts index 0182ee69..b2ffad81 100644 --- a/auto-claude-ui/src/renderer/hooks/index.ts +++ b/auto-claude-ui/src/renderer/hooks/index.ts @@ -5,5 +5,5 @@ export { useClaudeAuth } from './useClaudeAuth'; export { useLinearConnection } from './useLinearConnection'; export { useGitHubConnection } from './useGitHubConnection'; export { useInfrastructureStatus } from './useInfrastructureStatus'; -export { useIpc } from './useIpc'; +export { useIpcListeners } from './useIpc'; export { useVirtualizedTree } from './useVirtualizedTree'; diff --git a/auto-claude-ui/src/renderer/hooks/useClaudeAuth.ts b/auto-claude-ui/src/renderer/hooks/useClaudeAuth.ts index 05f93551..3ab94031 100644 --- a/auto-claude-ui/src/renderer/hooks/useClaudeAuth.ts +++ b/auto-claude-ui/src/renderer/hooks/useClaudeAuth.ts @@ -1,5 +1,5 @@ import { useState, useEffect } from 'react'; -import type { ProjectEnvConfig } from '../../../shared/types'; +import type { ProjectEnvConfig } from '../../shared/types'; type AuthStatus = 'checking' | 'authenticated' | 'not_authenticated' | 'error'; diff --git a/auto-claude-ui/src/renderer/hooks/useEnvironmentConfig.ts b/auto-claude-ui/src/renderer/hooks/useEnvironmentConfig.ts index 6526c810..a25cccc4 100644 --- a/auto-claude-ui/src/renderer/hooks/useEnvironmentConfig.ts +++ b/auto-claude-ui/src/renderer/hooks/useEnvironmentConfig.ts @@ -1,5 +1,5 @@ import { useState, useEffect } from 'react'; -import type { ProjectEnvConfig } from '../../../shared/types'; +import type { ProjectEnvConfig } from '../../shared/types'; export function useEnvironmentConfig(projectId: string, autoBuildPath: string | null, open: boolean) { const [envConfig, setEnvConfig] = useState(null); diff --git a/auto-claude-ui/src/renderer/hooks/useGitHubConnection.ts b/auto-claude-ui/src/renderer/hooks/useGitHubConnection.ts index 9a872814..d7d5e791 100644 --- a/auto-claude-ui/src/renderer/hooks/useGitHubConnection.ts +++ b/auto-claude-ui/src/renderer/hooks/useGitHubConnection.ts @@ -1,5 +1,5 @@ import { useState, useEffect } from 'react'; -import type { GitHubSyncStatus } from '../../../shared/types'; +import type { GitHubSyncStatus } from '../../shared/types'; export function useGitHubConnection( projectId: string, diff --git a/auto-claude-ui/src/renderer/hooks/useInfrastructureStatus.ts b/auto-claude-ui/src/renderer/hooks/useInfrastructureStatus.ts index 7b4c5bd9..be724229 100644 --- a/auto-claude-ui/src/renderer/hooks/useInfrastructureStatus.ts +++ b/auto-claude-ui/src/renderer/hooks/useInfrastructureStatus.ts @@ -1,5 +1,5 @@ import { useState, useEffect } from 'react'; -import type { InfrastructureStatus } from '../../../shared/types'; +import type { InfrastructureStatus } from '../../shared/types'; export function useInfrastructureStatus( graphitiEnabled: boolean | undefined, diff --git a/auto-claude-ui/src/renderer/hooks/useLinearConnection.ts b/auto-claude-ui/src/renderer/hooks/useLinearConnection.ts index b89ccdcf..84579e49 100644 --- a/auto-claude-ui/src/renderer/hooks/useLinearConnection.ts +++ b/auto-claude-ui/src/renderer/hooks/useLinearConnection.ts @@ -1,5 +1,5 @@ import { useState, useEffect } from 'react'; -import type { LinearSyncStatus } from '../../../shared/types'; +import type { LinearSyncStatus } from '../../shared/types'; export function useLinearConnection( projectId: string, diff --git a/auto-claude-ui/src/renderer/hooks/useProjectSettings.ts b/auto-claude-ui/src/renderer/hooks/useProjectSettings.ts index 053cbbcb..7447bb2c 100644 --- a/auto-claude-ui/src/renderer/hooks/useProjectSettings.ts +++ b/auto-claude-ui/src/renderer/hooks/useProjectSettings.ts @@ -1,5 +1,5 @@ import { useState, useEffect } from 'react'; -import type { Project, ProjectSettings, AutoBuildVersionInfo, ProjectEnvConfig } from '../../../shared/types'; +import type { Project, ProjectSettings, AutoBuildVersionInfo, ProjectEnvConfig } from '../../shared/types'; import { checkProjectVersion } from '../stores/project-store'; export function useProjectSettings(project: Project, open: boolean) { diff --git a/auto-claude-ui/src/renderer/lib/mocks/claude-profile-mock.ts b/auto-claude-ui/src/renderer/lib/mocks/claude-profile-mock.ts index 72650329..2ae8432f 100644 --- a/auto-claude-ui/src/renderer/lib/mocks/claude-profile-mock.ts +++ b/auto-claude-ui/src/renderer/lib/mocks/claude-profile-mock.ts @@ -11,9 +11,16 @@ export const claudeProfileMock = { } }), - saveClaudeProfile: async (profile: unknown) => ({ + saveClaudeProfile: async (profile: { id: string; name: string; oauthToken?: string; email?: string; isDefault?: boolean; createdAt?: Date }) => ({ success: true, - data: profile + data: { + id: profile.id, + name: profile.name, + oauthToken: profile.oauthToken, + email: profile.email, + isDefault: profile.isDefault ?? false, + createdAt: profile.createdAt ?? new Date(), + } }), deleteClaudeProfile: async () => ({ success: true }), diff --git a/auto-claude/__init__.py b/auto-claude/__init__.py new file mode 100644 index 00000000..3a292c89 --- /dev/null +++ b/auto-claude/__init__.py @@ -0,0 +1,9 @@ +""" +Auto Claude - Autonomous Coding Framework +========================================== + +Multi-agent autonomous coding framework that builds software through +coordinated AI agent sessions. +""" + +__version__ = "0.1.0" diff --git a/auto-claude/agent.py b/auto-claude/agent.py index 8b2cc8d5..cb1ad480 100644 --- a/auto-claude/agent.py +++ b/auto-claude/agent.py @@ -1,63 +1,2 @@ -""" -Agent Session Logic -=================== - -Core agent interaction functions for running autonomous coding sessions. -Uses subtask-based implementation plans with minimal, focused prompts. - -Architecture: -- Orchestrator (Python) handles all bookkeeping: memory, commits, progress -- Agent focuses ONLY on implementing code -- Post-session processing updates memory automatically (100% reliable) - -Enhanced with status file updates for ccstatusline integration. -Enhanced with Graphiti memory for cross-session context retrieval. - -NOTE: This module is now a facade that imports from agents/ submodules. -All logic has been refactored into focused modules for better maintainability. -""" - -# Re-export everything from the agents module to maintain backwards compatibility -from agents import ( - # Constants - AUTO_CONTINUE_DELAY_SECONDS, - HUMAN_INTERVENTION_FILE, - # Memory functions - debug_memory_system_status, - find_phase_for_subtask, - find_subtask_in_plan, - get_commit_count, - get_graphiti_context, - # Utility functions - get_latest_commit, - load_implementation_plan, - post_session_processing, - # Session management - run_agent_session, - # Main API - run_autonomous_agent, - run_followup_planner, - save_session_memory, - save_session_to_graphiti, - sync_plan_to_source, -) - -# Ensure all exports are available at module level -__all__ = [ - "run_autonomous_agent", - "run_followup_planner", - "debug_memory_system_status", - "get_graphiti_context", - "save_session_memory", - "save_session_to_graphiti", - "run_agent_session", - "post_session_processing", - "get_latest_commit", - "get_commit_count", - "load_implementation_plan", - "find_subtask_in_plan", - "find_phase_for_subtask", - "sync_plan_to_source", - "AUTO_CONTINUE_DELAY_SECONDS", - "HUMAN_INTERVENTION_FILE", -] +"""Backward compatibility shim - import from core.agent instead.""" +from core.agent import * diff --git a/auto-claude/agents/__init__.py b/auto-claude/agents/__init__.py index 0c288347..977fcb13 100644 --- a/auto-claude/agents/__init__.py +++ b/auto-claude/agents/__init__.py @@ -21,7 +21,7 @@ from .base import ( from .coder import run_autonomous_agent # Memory functions -from .memory import ( +from .memory_manager import ( debug_memory_system_status, get_graphiti_context, save_session_memory, diff --git a/auto-claude/agents/auto_claude_tools.py b/auto-claude/agents/auto_claude_tools.py new file mode 100644 index 00000000..ad6dc0a5 --- /dev/null +++ b/auto-claude/agents/auto_claude_tools.py @@ -0,0 +1,62 @@ +""" +Custom MCP Tools for Auto-Claude Agents +======================================== + +DEPRECATED: This module is now a compatibility shim. +Please import from the tools_pkg package instead: + + from agents.tools_pkg import create_auto_claude_mcp_server, get_allowed_tools + +This file remains for backward compatibility with existing imports. +All functionality has been moved to the tools_pkg package for better +organization and maintainability. +""" + +# Import everything from the package to maintain backward compatibility +# Use try/except to handle both relative and absolute imports +try: + from .tools_pkg import ( + ELECTRON_TOOLS, + TOOL_GET_BUILD_PROGRESS, + TOOL_GET_SESSION_CONTEXT, + TOOL_RECORD_DISCOVERY, + TOOL_RECORD_GOTCHA, + TOOL_UPDATE_QA_STATUS, + TOOL_UPDATE_SUBTASK_STATUS, + create_auto_claude_mcp_server, + get_allowed_tools, + is_electron_mcp_enabled, + is_tools_available, + ) +except ImportError: + # Fallback for direct execution - import from tools_pkg directly + from tools_pkg import ( + ELECTRON_TOOLS, + TOOL_GET_BUILD_PROGRESS, + TOOL_GET_SESSION_CONTEXT, + TOOL_RECORD_DISCOVERY, + TOOL_RECORD_GOTCHA, + TOOL_UPDATE_QA_STATUS, + TOOL_UPDATE_SUBTASK_STATUS, + create_auto_claude_mcp_server, + get_allowed_tools, + is_electron_mcp_enabled, + is_tools_available, + ) + +__all__ = [ + # Main API + "create_auto_claude_mcp_server", + "get_allowed_tools", + "is_tools_available", + # Tool name constants + "TOOL_UPDATE_SUBTASK_STATUS", + "TOOL_GET_BUILD_PROGRESS", + "TOOL_RECORD_DISCOVERY", + "TOOL_RECORD_GOTCHA", + "TOOL_GET_SESSION_CONTEXT", + "TOOL_UPDATE_QA_STATUS", + # Electron MCP + "ELECTRON_TOOLS", + "is_electron_mcp_enabled", +] diff --git a/auto-claude/agents/coder.py b/auto-claude/agents/coder.py index 182ea818..2af5233f 100644 --- a/auto-claude/agents/coder.py +++ b/auto-claude/agents/coder.py @@ -53,7 +53,7 @@ from ui import ( ) from .base import AUTO_CONTINUE_DELAY_SECONDS, HUMAN_INTERVENTION_FILE -from .memory import debug_memory_system_status, get_graphiti_context +from .memory_manager import debug_memory_system_status, get_graphiti_context from .session import post_session_processing, run_agent_session from .utils import ( find_phase_for_subtask, diff --git a/auto-claude/agents/memory.py b/auto-claude/agents/memory_manager.py similarity index 99% rename from auto-claude/agents/memory.py rename to auto-claude/agents/memory_manager.py index ff82462a..4fd2f502 100644 --- a/auto-claude/agents/memory.py +++ b/auto-claude/agents/memory_manager.py @@ -8,6 +8,7 @@ Handles session memory storage using dual-layer approach: """ import logging +import sys from pathlib import Path from debug import ( @@ -20,6 +21,9 @@ from debug import ( is_debug_enabled, ) from graphiti_config import get_graphiti_status, is_graphiti_enabled + +# Import from parent memory package +# Now safe since this module is named memory_manager (not memory) from memory import save_session_insights as save_file_based_memory logger = logging.getLogger(__name__) diff --git a/auto-claude/agents/session.py b/auto-claude/agents/session.py index 1b9a9d4b..84bdfd08 100644 --- a/auto-claude/agents/session.py +++ b/auto-claude/agents/session.py @@ -32,7 +32,7 @@ from ui import ( print_status, ) -from .memory import save_session_memory +from .memory_manager import save_session_memory from .utils import ( find_subtask_in_plan, get_commit_count, diff --git a/auto-claude/auto_claude_tools/__init__.py b/auto-claude/agents/tools_pkg/__init__.py similarity index 100% rename from auto-claude/auto_claude_tools/__init__.py rename to auto-claude/agents/tools_pkg/__init__.py diff --git a/auto-claude/auto_claude_tools/models.py b/auto-claude/agents/tools_pkg/models.py similarity index 100% rename from auto-claude/auto_claude_tools/models.py rename to auto-claude/agents/tools_pkg/models.py diff --git a/auto-claude/auto_claude_tools/permissions.py b/auto-claude/agents/tools_pkg/permissions.py similarity index 100% rename from auto-claude/auto_claude_tools/permissions.py rename to auto-claude/agents/tools_pkg/permissions.py diff --git a/auto-claude/auto_claude_tools/registry.py b/auto-claude/agents/tools_pkg/registry.py similarity index 100% rename from auto-claude/auto_claude_tools/registry.py rename to auto-claude/agents/tools_pkg/registry.py diff --git a/auto-claude/auto_claude_tools/tools/__init__.py b/auto-claude/agents/tools_pkg/tools/__init__.py similarity index 100% rename from auto-claude/auto_claude_tools/tools/__init__.py rename to auto-claude/agents/tools_pkg/tools/__init__.py diff --git a/auto-claude/auto_claude_tools/tools/memory.py b/auto-claude/agents/tools_pkg/tools/memory.py similarity index 100% rename from auto-claude/auto_claude_tools/tools/memory.py rename to auto-claude/agents/tools_pkg/tools/memory.py diff --git a/auto-claude/auto_claude_tools/tools/progress.py b/auto-claude/agents/tools_pkg/tools/progress.py similarity index 100% rename from auto-claude/auto_claude_tools/tools/progress.py rename to auto-claude/agents/tools_pkg/tools/progress.py diff --git a/auto-claude/auto_claude_tools/tools/qa.py b/auto-claude/agents/tools_pkg/tools/qa.py similarity index 100% rename from auto-claude/auto_claude_tools/tools/qa.py rename to auto-claude/agents/tools_pkg/tools/qa.py diff --git a/auto-claude/auto_claude_tools/tools/subtask.py b/auto-claude/agents/tools_pkg/tools/subtask.py similarity index 100% rename from auto-claude/auto_claude_tools/tools/subtask.py rename to auto-claude/agents/tools_pkg/tools/subtask.py diff --git a/auto-claude/analysis/__init__.py b/auto-claude/analysis/__init__.py new file mode 100644 index 00000000..9f67ff3e --- /dev/null +++ b/auto-claude/analysis/__init__.py @@ -0,0 +1,36 @@ +""" +Analysis Module +=============== + +Code analysis and project scanning tools. +""" + +# Import from analyzers subpackage (these are the modular analyzers) +from .analyzers import ( + ProjectAnalyzer as ModularProjectAnalyzer, + ServiceAnalyzer, + analyze_project, + analyze_service, +) + +# Import from analysis module root (these are other analysis tools) +from .project_analyzer import ProjectAnalyzer +from .risk_classifier import RiskClassifier +from .security_scanner import SecurityScanner +from .ci_discovery import CIDiscovery +from .test_discovery import TestDiscovery + +# insight_extractor is a module with functions, not a class, so don't import it here +# Import it directly when needed: from analysis import insight_extractor + +__all__ = [ + "ProjectAnalyzer", + "ModularProjectAnalyzer", + "ServiceAnalyzer", + "analyze_project", + "analyze_service", + "RiskClassifier", + "SecurityScanner", + "CIDiscovery", + "TestDiscovery", +] diff --git a/auto-claude/analysis/analyzer.py b/auto-claude/analysis/analyzer.py new file mode 100644 index 00000000..46f07a23 --- /dev/null +++ b/auto-claude/analysis/analyzer.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +""" +Codebase Analyzer +================= + +Automatically detects project structure, frameworks, and services. +Supports monorepos with multiple services. + +Usage: + # Index entire project (creates project_index.json) + python auto-claude/analyzer.py --index + + # Analyze specific service + python auto-claude/analyzer.py --service backend + + # Output to specific file + python auto-claude/analyzer.py --index --output path/to/output.json + +The analyzer will: +1. Detect if this is a monorepo or single project +2. Find all services/packages and analyze each separately +3. Map interdependencies between services +4. Identify infrastructure (Docker, CI/CD) +5. Document conventions (linting, testing) + +This module now serves as a facade to the modular analyzer system in the analyzers/ package. +All actual implementation is in focused submodules for better maintainability. +""" + +import json +from pathlib import Path + +# Import from the new modular structure +from .analyzers import ( + ProjectAnalyzer, + ServiceAnalyzer, + analyze_project, + analyze_service, +) + +# Re-export for backward compatibility +__all__ = [ + "ServiceAnalyzer", + "ProjectAnalyzer", + "analyze_project", + "analyze_service", +] + + +def main(): + """CLI entry point.""" + import argparse + + parser = argparse.ArgumentParser( + description="Analyze project structure, frameworks, and services" + ) + parser.add_argument( + "--project-dir", + type=Path, + default=Path.cwd(), + help="Project directory to analyze (default: current directory)", + ) + parser.add_argument( + "--index", + action="store_true", + help="Create full project index (default behavior)", + ) + parser.add_argument( + "--service", + type=str, + default=None, + help="Analyze a specific service only", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Output file for JSON results", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="Only output JSON, no status messages", + ) + + args = parser.parse_args() + + # Determine what to analyze + if args.service: + results = analyze_service(args.project_dir, args.service, args.output) + else: + results = analyze_project(args.project_dir, args.output) + + # Print results + if not args.quiet or not args.output: + print(json.dumps(results, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/auto-claude/analysis/analyzers/__init__.py b/auto-claude/analysis/analyzers/__init__.py new file mode 100644 index 00000000..b425b119 --- /dev/null +++ b/auto-claude/analysis/analyzers/__init__.py @@ -0,0 +1,92 @@ +""" +Analyzers Package +================= + +Modular analyzer system for detecting project structure, frameworks, and services. + +Main exports: +- ServiceAnalyzer: Analyzes a single service/package +- ProjectAnalyzer: Analyzes entire projects (single or monorepo) +- analyze_project: Convenience function for project analysis +- analyze_service: Convenience function for service analysis +""" + +from pathlib import Path +from typing import Any + +from .project_analyzer_module import ProjectAnalyzer +from .service_analyzer import ServiceAnalyzer + +# Re-export main classes +__all__ = [ + "ServiceAnalyzer", + "ProjectAnalyzer", + "analyze_project", + "analyze_service", +] + + +def analyze_project(project_dir: Path, output_file: Path | None = None) -> dict: + """ + Analyze a project and optionally save results. + + Args: + project_dir: Path to the project root + output_file: Optional path to save JSON output + + Returns: + Project index as a dictionary + """ + import json + + analyzer = ProjectAnalyzer(project_dir) + results = analyzer.analyze() + + if output_file: + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(results, f, indent=2) + print(f"Project index saved to: {output_file}") + + return results + + +def analyze_service( + project_dir: Path, service_name: str, output_file: Path | None = None +) -> dict: + """ + Analyze a specific service within a project. + + Args: + project_dir: Path to the project root + service_name: Name of the service to analyze + output_file: Optional path to save JSON output + + Returns: + Service analysis as a dictionary + """ + import json + + # Find the service + service_path = project_dir / service_name + if not service_path.exists(): + # Check common locations + for parent in ["packages", "apps", "services"]: + candidate = project_dir / parent / service_name + if candidate.exists(): + service_path = candidate + break + + if not service_path.exists(): + raise ValueError(f"Service '{service_name}' not found in {project_dir}") + + analyzer = ServiceAnalyzer(service_path, service_name) + results = analyzer.analyze() + + if output_file: + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w") as f: + json.dump(results, f, indent=2) + print(f"Service analysis saved to: {output_file}") + + return results diff --git a/auto-claude/analyzers/base.py b/auto-claude/analysis/analyzers/base.py similarity index 100% rename from auto-claude/analyzers/base.py rename to auto-claude/analysis/analyzers/base.py diff --git a/auto-claude/analyzers/context/__init__.py b/auto-claude/analysis/analyzers/context/__init__.py similarity index 100% rename from auto-claude/analyzers/context/__init__.py rename to auto-claude/analysis/analyzers/context/__init__.py diff --git a/auto-claude/analyzers/context/api_docs_detector.py b/auto-claude/analysis/analyzers/context/api_docs_detector.py similarity index 100% rename from auto-claude/analyzers/context/api_docs_detector.py rename to auto-claude/analysis/analyzers/context/api_docs_detector.py diff --git a/auto-claude/analyzers/context/auth_detector.py b/auto-claude/analysis/analyzers/context/auth_detector.py similarity index 100% rename from auto-claude/analyzers/context/auth_detector.py rename to auto-claude/analysis/analyzers/context/auth_detector.py diff --git a/auto-claude/analyzers/context/env_detector.py b/auto-claude/analysis/analyzers/context/env_detector.py similarity index 100% rename from auto-claude/analyzers/context/env_detector.py rename to auto-claude/analysis/analyzers/context/env_detector.py diff --git a/auto-claude/analyzers/context/jobs_detector.py b/auto-claude/analysis/analyzers/context/jobs_detector.py similarity index 100% rename from auto-claude/analyzers/context/jobs_detector.py rename to auto-claude/analysis/analyzers/context/jobs_detector.py diff --git a/auto-claude/analyzers/context/migrations_detector.py b/auto-claude/analysis/analyzers/context/migrations_detector.py similarity index 100% rename from auto-claude/analyzers/context/migrations_detector.py rename to auto-claude/analysis/analyzers/context/migrations_detector.py diff --git a/auto-claude/analyzers/context/monitoring_detector.py b/auto-claude/analysis/analyzers/context/monitoring_detector.py similarity index 100% rename from auto-claude/analyzers/context/monitoring_detector.py rename to auto-claude/analysis/analyzers/context/monitoring_detector.py diff --git a/auto-claude/analyzers/context/services_detector.py b/auto-claude/analysis/analyzers/context/services_detector.py similarity index 100% rename from auto-claude/analyzers/context/services_detector.py rename to auto-claude/analysis/analyzers/context/services_detector.py diff --git a/auto-claude/analyzers/context_analyzer.py b/auto-claude/analysis/analyzers/context_analyzer.py similarity index 100% rename from auto-claude/analyzers/context_analyzer.py rename to auto-claude/analysis/analyzers/context_analyzer.py diff --git a/auto-claude/analyzers/database_detector.py b/auto-claude/analysis/analyzers/database_detector.py similarity index 100% rename from auto-claude/analyzers/database_detector.py rename to auto-claude/analysis/analyzers/database_detector.py diff --git a/auto-claude/analyzers/framework_analyzer.py b/auto-claude/analysis/analyzers/framework_analyzer.py similarity index 100% rename from auto-claude/analyzers/framework_analyzer.py rename to auto-claude/analysis/analyzers/framework_analyzer.py diff --git a/auto-claude/analyzers/port_detector.py b/auto-claude/analysis/analyzers/port_detector.py similarity index 100% rename from auto-claude/analyzers/port_detector.py rename to auto-claude/analysis/analyzers/port_detector.py diff --git a/auto-claude/analyzers/project_analyzer_module.py b/auto-claude/analysis/analyzers/project_analyzer_module.py similarity index 100% rename from auto-claude/analyzers/project_analyzer_module.py rename to auto-claude/analysis/analyzers/project_analyzer_module.py diff --git a/auto-claude/analyzers/route_detector.py b/auto-claude/analysis/analyzers/route_detector.py similarity index 100% rename from auto-claude/analyzers/route_detector.py rename to auto-claude/analysis/analyzers/route_detector.py diff --git a/auto-claude/analyzers/service_analyzer.py b/auto-claude/analysis/analyzers/service_analyzer.py similarity index 100% rename from auto-claude/analyzers/service_analyzer.py rename to auto-claude/analysis/analyzers/service_analyzer.py diff --git a/auto-claude/analysis/ci_discovery.py b/auto-claude/analysis/ci_discovery.py new file mode 100644 index 00000000..347546c4 --- /dev/null +++ b/auto-claude/analysis/ci_discovery.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +""" +CI Discovery Module +=================== + +Parses CI/CD configuration files to extract test commands and workflows. +Supports GitHub Actions, GitLab CI, CircleCI, and Jenkins. + +The CI discovery results are used by: +- QA Agent: To understand existing CI test patterns +- Validation Strategy: To match CI commands +- Planner: To align verification with CI + +Usage: + from ci_discovery import CIDiscovery + + discovery = CIDiscovery() + result = discovery.discover(project_dir) + + if result: + print(f"CI System: {result.ci_system}") + print(f"Test Commands: {result.test_commands}") +""" + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# Try to import yaml, fall back gracefully +try: + import yaml + + HAS_YAML = True +except ImportError: + HAS_YAML = False + + +# ============================================================================= +# DATA CLASSES +# ============================================================================= + + +@dataclass +class CIWorkflow: + """ + Represents a CI workflow or job. + + Attributes: + name: Name of the workflow/job + trigger: What triggers this workflow (push, pull_request, etc.) + steps: List of step names or commands + test_related: Whether this appears to be test-related + """ + + name: str + trigger: list[str] = field(default_factory=list) + steps: list[str] = field(default_factory=list) + test_related: bool = False + + +@dataclass +class CIConfig: + """ + Result of CI configuration discovery. + + Attributes: + ci_system: Name of CI system (github_actions, gitlab, circleci, jenkins) + config_files: List of CI config files found + test_commands: Extracted test commands by type + coverage_command: Coverage command if found + workflows: List of discovered workflows + environment_variables: Environment variables used + """ + + ci_system: str + config_files: list[str] = field(default_factory=list) + test_commands: dict[str, str] = field(default_factory=dict) + coverage_command: str | None = None + workflows: list[CIWorkflow] = field(default_factory=list) + environment_variables: list[str] = field(default_factory=list) + + +# ============================================================================= +# CI PARSERS +# ============================================================================= + + +class CIDiscovery: + """ + Discovers CI/CD configurations in a project. + + Analyzes: + - GitHub Actions (.github/workflows/*.yml) + - GitLab CI (.gitlab-ci.yml) + - CircleCI (.circleci/config.yml) + - Jenkins (Jenkinsfile) + """ + + def __init__(self) -> None: + """Initialize CI discovery.""" + self._cache: dict[str, CIConfig | None] = {} + + def discover(self, project_dir: Path) -> CIConfig | None: + """ + Discover CI configuration in the project. + + Args: + project_dir: Path to the project root + + Returns: + CIConfig if CI found, None otherwise + """ + project_dir = Path(project_dir) + cache_key = str(project_dir.resolve()) + + if cache_key in self._cache: + return self._cache[cache_key] + + # Try each CI system + result = None + + # GitHub Actions + github_workflows = project_dir / ".github" / "workflows" + if github_workflows.exists(): + result = self._parse_github_actions(github_workflows) + + # GitLab CI + if not result: + gitlab_ci = project_dir / ".gitlab-ci.yml" + if gitlab_ci.exists(): + result = self._parse_gitlab_ci(gitlab_ci) + + # CircleCI + if not result: + circleci = project_dir / ".circleci" / "config.yml" + if circleci.exists(): + result = self._parse_circleci(circleci) + + # Jenkins + if not result: + jenkinsfile = project_dir / "Jenkinsfile" + if jenkinsfile.exists(): + result = self._parse_jenkinsfile(jenkinsfile) + + self._cache[cache_key] = result + return result + + def _parse_github_actions(self, workflows_dir: Path) -> CIConfig: + """Parse GitHub Actions workflow files.""" + result = CIConfig(ci_system="github_actions") + + workflow_files = list(workflows_dir.glob("*.yml")) + list( + workflows_dir.glob("*.yaml") + ) + + for wf_file in workflow_files: + result.config_files.append( + str(wf_file.relative_to(workflows_dir.parent.parent)) + ) + + try: + content = wf_file.read_text() + workflow_data = self._parse_yaml(content) + + if not workflow_data: + continue + + # Get workflow name + wf_name = workflow_data.get("name", wf_file.stem) + + # Get triggers + triggers = [] + on_trigger = workflow_data.get("on", {}) + if isinstance(on_trigger, str): + triggers = [on_trigger] + elif isinstance(on_trigger, list): + triggers = on_trigger + elif isinstance(on_trigger, dict): + triggers = list(on_trigger.keys()) + + # Parse jobs + jobs = workflow_data.get("jobs", {}) + for job_name, job_config in jobs.items(): + if not isinstance(job_config, dict): + continue + + steps = job_config.get("steps", []) + step_commands = [] + test_related = False + + for step in steps: + if not isinstance(step, dict): + continue + + # Get step name or command + step_name = step.get("name", "") + run_cmd = step.get("run", "") + uses = step.get("uses", "") + + if step_name: + step_commands.append(step_name) + if run_cmd: + step_commands.append(run_cmd) + # Extract test commands + self._extract_test_commands(run_cmd, result) + if uses: + step_commands.append(f"uses: {uses}") + + # Check if test-related + test_keywords = ["test", "pytest", "jest", "vitest", "coverage"] + if any(kw in str(step).lower() for kw in test_keywords): + test_related = True + + result.workflows.append( + CIWorkflow( + name=f"{wf_name}/{job_name}", + trigger=triggers, + steps=step_commands, + test_related=test_related, + ) + ) + + # Extract environment variables + env = workflow_data.get("env", {}) + if isinstance(env, dict): + result.environment_variables.extend(env.keys()) + + except Exception: + continue + + return result + + def _parse_gitlab_ci(self, config_file: Path) -> CIConfig: + """Parse GitLab CI configuration.""" + result = CIConfig( + ci_system="gitlab", + config_files=[".gitlab-ci.yml"], + ) + + try: + content = config_file.read_text() + data = self._parse_yaml(content) + + if not data: + return result + + # Parse jobs (top-level keys that aren't special keywords) + special_keys = { + "stages", + "variables", + "image", + "services", + "before_script", + "after_script", + "cache", + "include", + "default", + "workflow", + } + + for key, value in data.items(): + if key.startswith(".") or key in special_keys: + continue + + if not isinstance(value, dict): + continue + + job_config = value + script = job_config.get("script", []) + if isinstance(script, str): + script = [script] + + test_related = any( + kw in str(script).lower() + for kw in ["test", "pytest", "jest", "vitest", "coverage"] + ) + + result.workflows.append( + CIWorkflow( + name=key, + trigger=job_config.get("only", []) + or job_config.get("rules", []), + steps=script, + test_related=test_related, + ) + ) + + # Extract test commands + for cmd in script: + if isinstance(cmd, str): + self._extract_test_commands(cmd, result) + + # Extract variables + variables = data.get("variables", {}) + if isinstance(variables, dict): + result.environment_variables.extend(variables.keys()) + + except Exception: + pass + + return result + + def _parse_circleci(self, config_file: Path) -> CIConfig: + """Parse CircleCI configuration.""" + result = CIConfig( + ci_system="circleci", + config_files=[".circleci/config.yml"], + ) + + try: + content = config_file.read_text() + data = self._parse_yaml(content) + + if not data: + return result + + # Parse jobs + jobs = data.get("jobs", {}) + for job_name, job_config in jobs.items(): + if not isinstance(job_config, dict): + continue + + steps = job_config.get("steps", []) + step_commands = [] + test_related = False + + for step in steps: + if isinstance(step, str): + step_commands.append(step) + elif isinstance(step, dict): + if "run" in step: + run = step["run"] + if isinstance(run, str): + step_commands.append(run) + self._extract_test_commands(run, result) + elif isinstance(run, dict): + cmd = run.get("command", "") + step_commands.append(cmd) + self._extract_test_commands(cmd, result) + + if any( + kw in str(step).lower() + for kw in ["test", "pytest", "jest", "coverage"] + ): + test_related = True + + result.workflows.append( + CIWorkflow( + name=job_name, + trigger=[], + steps=step_commands, + test_related=test_related, + ) + ) + + except Exception: + pass + + return result + + def _parse_jenkinsfile(self, jenkinsfile: Path) -> CIConfig: + """Parse Jenkinsfile (basic extraction).""" + result = CIConfig( + ci_system="jenkins", + config_files=["Jenkinsfile"], + ) + + try: + content = jenkinsfile.read_text() + + # Extract sh commands using regex + sh_pattern = re.compile(r'sh\s+[\'"]([^\'"]+)[\'"]') + matches = sh_pattern.findall(content) + + steps = [] + test_related = False + + for cmd in matches: + steps.append(cmd) + self._extract_test_commands(cmd, result) + + if any( + kw in cmd.lower() for kw in ["test", "pytest", "jest", "coverage"] + ): + test_related = True + + # Extract stage names + stage_pattern = re.compile(r'stage\s*\([\'"]([^\'"]+)[\'"]\)') + stages = stage_pattern.findall(content) + + for stage in stages: + result.workflows.append( + CIWorkflow( + name=stage, + trigger=[], + steps=steps if "test" in stage.lower() else [], + test_related="test" in stage.lower(), + ) + ) + + except Exception: + pass + + return result + + def _parse_yaml(self, content: str) -> dict | None: + """Parse YAML content, with fallback to basic parsing if yaml not available.""" + if HAS_YAML: + try: + return yaml.safe_load(content) + except Exception: + return None + + # Basic fallback for simple YAML (very limited) + # This won't work for complex structures + return None + + def _extract_test_commands(self, cmd: str, result: CIConfig) -> None: + """Extract test commands from a command string.""" + cmd_lower = cmd.lower() + + # Python pytest + if "pytest" in cmd_lower: + if "pytest" not in result.test_commands: + result.test_commands["unit"] = cmd.strip() + if "--cov" in cmd_lower: + result.coverage_command = cmd.strip() + + # Node.js test commands + if ( + "npm test" in cmd_lower + or "yarn test" in cmd_lower + or "pnpm test" in cmd_lower + ): + if "unit" not in result.test_commands: + result.test_commands["unit"] = cmd.strip() + + # Jest/Vitest + if "jest" in cmd_lower or "vitest" in cmd_lower: + if "unit" not in result.test_commands: + result.test_commands["unit"] = cmd.strip() + if "--coverage" in cmd_lower: + result.coverage_command = cmd.strip() + + # E2E testing + if "playwright" in cmd_lower: + result.test_commands["e2e"] = cmd.strip() + if "cypress" in cmd_lower: + result.test_commands["e2e"] = cmd.strip() + + # Integration tests + if "integration" in cmd_lower: + result.test_commands["integration"] = cmd.strip() + + # Go tests + if "go test" in cmd_lower: + if "unit" not in result.test_commands: + result.test_commands["unit"] = cmd.strip() + + # Rust tests + if "cargo test" in cmd_lower: + if "unit" not in result.test_commands: + result.test_commands["unit"] = cmd.strip() + + def to_dict(self, result: CIConfig) -> dict[str, Any]: + """Convert result to dictionary for JSON serialization.""" + return { + "ci_system": result.ci_system, + "config_files": result.config_files, + "test_commands": result.test_commands, + "coverage_command": result.coverage_command, + "workflows": [ + { + "name": w.name, + "trigger": w.trigger, + "steps": w.steps, + "test_related": w.test_related, + } + for w in result.workflows + ], + "environment_variables": result.environment_variables, + } + + def clear_cache(self) -> None: + """Clear the internal cache.""" + self._cache.clear() + + +# ============================================================================= +# CONVENIENCE FUNCTIONS +# ============================================================================= + + +def discover_ci(project_dir: Path) -> CIConfig | None: + """ + Convenience function to discover CI configuration. + + Args: + project_dir: Path to project root + + Returns: + CIConfig if found, None otherwise + """ + discovery = CIDiscovery() + return discovery.discover(project_dir) + + +def get_ci_test_commands(project_dir: Path) -> dict[str, str]: + """ + Get test commands from CI configuration. + + Args: + project_dir: Path to project root + + Returns: + Dictionary of test type to command + """ + discovery = CIDiscovery() + result = discovery.discover(project_dir) + if result: + return result.test_commands + return {} + + +def get_ci_system(project_dir: Path) -> str | None: + """ + Get the CI system name if configured. + + Args: + project_dir: Path to project root + + Returns: + CI system name or None + """ + discovery = CIDiscovery() + result = discovery.discover(project_dir) + if result: + return result.ci_system + return None + + +# ============================================================================= +# CLI +# ============================================================================= + + +def main() -> None: + """CLI entry point for testing.""" + import argparse + + parser = argparse.ArgumentParser(description="Discover CI configuration") + parser.add_argument("project_dir", type=Path, help="Path to project root") + parser.add_argument("--json", action="store_true", help="Output as JSON") + + args = parser.parse_args() + + discovery = CIDiscovery() + result = discovery.discover(args.project_dir) + + if not result: + print("No CI configuration found") + return + + if args.json: + print(json.dumps(discovery.to_dict(result), indent=2)) + else: + print(f"CI System: {result.ci_system}") + print(f"Config Files: {', '.join(result.config_files)}") + print("\nTest Commands:") + for test_type, cmd in result.test_commands.items(): + print(f" {test_type}: {cmd}") + if result.coverage_command: + print(f"\nCoverage Command: {result.coverage_command}") + print(f"\nWorkflows ({len(result.workflows)}):") + for w in result.workflows: + marker = "[TEST]" if w.test_related else "" + print(f" - {w.name} {marker}") + if w.trigger: + print(f" Triggers: {', '.join(str(t) for t in w.trigger)}") + if result.environment_variables: + print(f"\nEnvironment Variables: {', '.join(result.environment_variables)}") + + +if __name__ == "__main__": + main() diff --git a/auto-claude/analysis/insight_extractor.py b/auto-claude/analysis/insight_extractor.py new file mode 100644 index 00000000..281fdb87 --- /dev/null +++ b/auto-claude/analysis/insight_extractor.py @@ -0,0 +1,584 @@ +""" +Insight Extractor +================= + +Automatically extracts structured insights from completed coding sessions. +Runs after each session to capture rich, actionable knowledge for Graphiti memory. + +Uses the Claude Agent SDK (same as the rest of the system) for extraction. +Falls back to generic insights if extraction fails (never blocks the build). +""" + +import json +import logging +import os +import subprocess +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +# Check for Claude SDK availability +try: + from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient + SDK_AVAILABLE = True +except ImportError: + SDK_AVAILABLE = False + ClaudeAgentOptions = None + ClaudeSDKClient = None + +# Default model for insight extraction (fast and cheap) +DEFAULT_EXTRACTION_MODEL = "claude-3-5-haiku-latest" + +# Maximum diff size to send to the LLM (avoid context limits) +MAX_DIFF_CHARS = 15000 + +# Maximum attempt history entries to include +MAX_ATTEMPTS_TO_INCLUDE = 3 + + +def is_extraction_enabled() -> bool: + """Check if insight extraction is enabled.""" + # Extraction requires Claude SDK and OAuth token + if not SDK_AVAILABLE: + return False + if not os.environ.get("CLAUDE_CODE_OAUTH_TOKEN"): + return False + enabled_str = os.environ.get("INSIGHT_EXTRACTION_ENABLED", "true").lower() + return enabled_str in ("true", "1", "yes") + + +def get_extraction_model() -> str: + """Get the model to use for insight extraction.""" + return os.environ.get("INSIGHT_EXTRACTOR_MODEL", DEFAULT_EXTRACTION_MODEL) + + +# ============================================================================= +# Git Helpers +# ============================================================================= + + +def get_session_diff( + project_dir: Path, + commit_before: str | None, + commit_after: str | None, +) -> str: + """ + Get the git diff between two commits. + + Args: + project_dir: Project root directory + commit_before: Commit hash before session (or None) + commit_after: Commit hash after session (or None) + + Returns: + Diff text (truncated if too large) + """ + if not commit_before or not commit_after: + return "(No commits to diff)" + + if commit_before == commit_after: + return "(No changes - same commit)" + + try: + result = subprocess.run( + ["git", "diff", commit_before, commit_after], + cwd=project_dir, + capture_output=True, + text=True, + timeout=30, + ) + diff = result.stdout + + if len(diff) > MAX_DIFF_CHARS: + # Truncate and add note + diff = ( + diff[:MAX_DIFF_CHARS] + f"\n\n... (truncated, {len(diff)} chars total)" + ) + + return diff if diff else "(Empty diff)" + + except subprocess.TimeoutExpired: + logger.warning("Git diff timed out") + return "(Git diff timed out)" + except Exception as e: + logger.warning(f"Failed to get git diff: {e}") + return f"(Failed to get diff: {e})" + + +def get_changed_files( + project_dir: Path, + commit_before: str | None, + commit_after: str | None, +) -> list[str]: + """ + Get list of files changed between two commits. + + Args: + project_dir: Project root directory + commit_before: Commit hash before session + commit_after: Commit hash after session + + Returns: + List of changed file paths + """ + if not commit_before or not commit_after or commit_before == commit_after: + return [] + + try: + result = subprocess.run( + ["git", "diff", "--name-only", commit_before, commit_after], + cwd=project_dir, + capture_output=True, + text=True, + timeout=10, + ) + files = [f.strip() for f in result.stdout.strip().split("\n") if f.strip()] + return files + + except Exception as e: + logger.warning(f"Failed to get changed files: {e}") + return [] + + +def get_commit_messages( + project_dir: Path, + commit_before: str | None, + commit_after: str | None, +) -> str: + """Get commit messages between two commits.""" + if not commit_before or not commit_after or commit_before == commit_after: + return "(No commits)" + + try: + result = subprocess.run( + ["git", "log", "--oneline", f"{commit_before}..{commit_after}"], + cwd=project_dir, + capture_output=True, + text=True, + timeout=10, + ) + return result.stdout.strip() if result.stdout.strip() else "(No commits)" + + except Exception as e: + logger.warning(f"Failed to get commit messages: {e}") + return f"(Failed: {e})" + + +# ============================================================================= +# Input Gathering +# ============================================================================= + + +def gather_extraction_inputs( + spec_dir: Path, + project_dir: Path, + subtask_id: str, + session_num: int, + commit_before: str | None, + commit_after: str | None, + success: bool, + recovery_manager: Any, +) -> dict: + """ + Gather all inputs needed for insight extraction. + + Args: + spec_dir: Spec directory + project_dir: Project root + subtask_id: The subtask that was worked on + session_num: Session number + commit_before: Commit before session + commit_after: Commit after session + success: Whether session succeeded + recovery_manager: Recovery manager with attempt history + + Returns: + Dict with all inputs for the extractor + """ + # Get subtask description from implementation plan + subtask_description = _get_subtask_description(spec_dir, subtask_id) + + # Get git diff + diff = get_session_diff(project_dir, commit_before, commit_after) + + # Get changed files + changed_files = get_changed_files(project_dir, commit_before, commit_after) + + # Get commit messages + commit_messages = get_commit_messages(project_dir, commit_before, commit_after) + + # Get attempt history + attempt_history = _get_attempt_history(recovery_manager, subtask_id) + + return { + "subtask_id": subtask_id, + "subtask_description": subtask_description, + "session_num": session_num, + "success": success, + "diff": diff, + "changed_files": changed_files, + "commit_messages": commit_messages, + "attempt_history": attempt_history, + } + + +def _get_subtask_description(spec_dir: Path, subtask_id: str) -> str: + """Get subtask description from implementation plan.""" + plan_file = spec_dir / "implementation_plan.json" + if not plan_file.exists(): + return f"Subtask: {subtask_id}" + + try: + with open(plan_file) as f: + plan = json.load(f) + + # Search through phases for the subtask + for phase in plan.get("phases", []): + for subtask in phase.get("subtasks", []): + if subtask.get("id") == subtask_id: + return subtask.get("description", f"Subtask: {subtask_id}") + + return f"Subtask: {subtask_id}" + + except Exception as e: + logger.warning(f"Failed to load subtask description: {e}") + return f"Subtask: {subtask_id}" + + +def _get_attempt_history(recovery_manager: Any, subtask_id: str) -> list[dict]: + """Get previous attempt history for this subtask.""" + if not recovery_manager: + return [] + + try: + history = recovery_manager.get_subtask_history(subtask_id) + attempts = history.get("attempts", []) + + # Limit to recent attempts + return attempts[-MAX_ATTEMPTS_TO_INCLUDE:] + + except Exception as e: + logger.warning(f"Failed to get attempt history: {e}") + return [] + + +# ============================================================================= +# LLM Extraction +# ============================================================================= + + +def _build_extraction_prompt(inputs: dict) -> str: + """Build the prompt for insight extraction.""" + prompt_file = Path(__file__).parent / "prompts" / "insight_extractor.md" + + if prompt_file.exists(): + base_prompt = prompt_file.read_text() + else: + # Fallback if prompt file missing + base_prompt = """Extract structured insights from this coding session. +Output ONLY valid JSON with: file_insights, patterns_discovered, gotchas_discovered, approach_outcome, recommendations""" + + # Build session context + session_context = f""" +--- + +## SESSION DATA + +### Subtask +- **ID**: {inputs["subtask_id"]} +- **Description**: {inputs["subtask_description"]} +- **Session Number**: {inputs["session_num"]} +- **Outcome**: {"SUCCESS" if inputs["success"] else "FAILED"} + +### Files Changed +{chr(10).join(f"- {f}" for f in inputs["changed_files"]) if inputs["changed_files"] else "(No files changed)"} + +### Commit Messages +{inputs["commit_messages"]} + +### Git Diff +```diff +{inputs["diff"]} +``` + +### Previous Attempts +{_format_attempt_history(inputs["attempt_history"])} + +--- + +Now analyze this session and output ONLY the JSON object. +""" + + return base_prompt + session_context + + +def _format_attempt_history(attempts: list[dict]) -> str: + """Format attempt history for the prompt.""" + if not attempts: + return "(First attempt - no previous history)" + + lines = [] + for i, attempt in enumerate(attempts, 1): + success = "SUCCESS" if attempt.get("success") else "FAILED" + approach = attempt.get("approach", "Unknown approach") + error = attempt.get("error", "") + lines.append(f"**Attempt {i}** ({success}): {approach}") + if error: + lines.append(f" Error: {error}") + + return "\n".join(lines) + + +async def run_insight_extraction(inputs: dict, project_dir: Path | None = None) -> dict | None: + """ + Run the insight extraction using Claude Agent SDK. + + Args: + inputs: Gathered session inputs + project_dir: Project directory for SDK context (optional) + + Returns: + Extracted insights dict or None if failed + """ + if not SDK_AVAILABLE: + logger.warning("Claude SDK not available, skipping insight extraction") + return None + + oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") + if not oauth_token: + logger.warning("CLAUDE_CODE_OAUTH_TOKEN not set, skipping insight extraction") + return None + + model = get_extraction_model() + prompt = _build_extraction_prompt(inputs) + + # Use current directory if project_dir not specified + cwd = str(project_dir.resolve()) if project_dir else os.getcwd() + + try: + # Create a minimal SDK client for insight extraction + # No tools needed - just text generation + client = ClaudeSDKClient( + options=ClaudeAgentOptions( + model=model, + system_prompt=( + "You are an expert code analyst. You extract structured insights from coding sessions. " + "Always respond with valid JSON only, no markdown formatting or explanations." + ), + allowed_tools=[], # No tools needed for extraction + max_turns=1, # Single turn extraction + cwd=cwd, + ) + ) + + # Use async context manager + async with client: + await client.query(prompt) + + # Collect the response + response_text = "" + async for msg in client.receive_response(): + msg_type = type(msg).__name__ + if msg_type == "AssistantMessage" and hasattr(msg, "content"): + for block in msg.content: + if hasattr(block, "text"): + response_text += block.text + + # Parse JSON from response + return parse_insights(response_text) + + except Exception as e: + logger.warning(f"Insight extraction failed: {e}") + return None + + +def parse_insights(response_text: str) -> dict | None: + """ + Parse the LLM response into structured insights. + + Args: + response_text: Raw LLM response + + Returns: + Parsed insights dict or None if parsing failed + """ + # Try to extract JSON from the response + text = response_text.strip() + + # Handle markdown code blocks + if text.startswith("```"): + # Remove code block markers + lines = text.split("\n") + # Remove first line (```json or ```) + if lines[0].startswith("```"): + lines = lines[1:] + # Remove last line if it's `` + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + text = "\n".join(lines) + + try: + insights = json.loads(text) + + # Validate structure + if not isinstance(insights, dict): + logger.warning("Insights is not a dict") + return None + + # Ensure required keys exist with defaults + insights.setdefault("file_insights", []) + insights.setdefault("patterns_discovered", []) + insights.setdefault("gotchas_discovered", []) + insights.setdefault("approach_outcome", {}) + insights.setdefault("recommendations", []) + + return insights + + except json.JSONDecodeError as e: + logger.warning(f"Failed to parse insights JSON: {e}") + logger.debug(f"Response text was: {text[:500]}") + return None + + +# ============================================================================= +# Main Entry Point +# ============================================================================= + + +async def extract_session_insights( + spec_dir: Path, + project_dir: Path, + subtask_id: str, + session_num: int, + commit_before: str | None, + commit_after: str | None, + success: bool, + recovery_manager: Any, +) -> dict: + """ + Extract insights from a completed coding session. + + This is the main entry point called from post_session_processing(). + Falls back to generic insights if extraction fails. + + Args: + spec_dir: Spec directory + project_dir: Project root + subtask_id: Subtask that was worked on + session_num: Session number + commit_before: Commit before session + commit_after: Commit after session + success: Whether session succeeded + recovery_manager: Recovery manager with attempt history + + Returns: + Insights dict (rich if extraction succeeded, generic if failed) + """ + # Check if extraction is enabled + if not is_extraction_enabled(): + logger.info("Insight extraction disabled") + return _get_generic_insights(subtask_id, success) + + # Check for no changes + if commit_before == commit_after: + logger.info("No changes to extract insights from") + return _get_generic_insights(subtask_id, success) + + try: + # Gather inputs + inputs = gather_extraction_inputs( + spec_dir=spec_dir, + project_dir=project_dir, + subtask_id=subtask_id, + session_num=session_num, + commit_before=commit_before, + commit_after=commit_after, + success=success, + recovery_manager=recovery_manager, + ) + + # Run extraction + extracted = await run_insight_extraction(inputs, project_dir=project_dir) + + if extracted: + # Add metadata + extracted["subtask_id"] = subtask_id + extracted["session_num"] = session_num + extracted["success"] = success + extracted["changed_files"] = inputs["changed_files"] + + logger.info( + f"Extracted insights: {len(extracted.get('file_insights', []))} file insights, " + f"{len(extracted.get('patterns_discovered', []))} patterns, " + f"{len(extracted.get('gotchas_discovered', []))} gotchas" + ) + return extracted + else: + logger.warning("Extraction returned no results, using generic insights") + return _get_generic_insights(subtask_id, success) + + except Exception as e: + logger.warning(f"Insight extraction failed: {e}, using generic insights") + return _get_generic_insights(subtask_id, success) + + +def _get_generic_insights(subtask_id: str, success: bool) -> dict: + """Return generic insights when extraction fails or is disabled.""" + return { + "file_insights": [], + "patterns_discovered": [], + "gotchas_discovered": [], + "approach_outcome": { + "success": success, + "approach_used": f"Implemented subtask: {subtask_id}", + "why_it_worked": None, + "why_it_failed": None, + "alternatives_tried": [], + }, + "recommendations": [], + "subtask_id": subtask_id, + "success": success, + "changed_files": [], + } + + +# ============================================================================= +# CLI for Testing +# ============================================================================= + +if __name__ == "__main__": + import argparse + import asyncio + + parser = argparse.ArgumentParser(description="Test insight extraction") + parser.add_argument("--spec-dir", type=Path, required=True, help="Spec directory") + parser.add_argument( + "--project-dir", type=Path, required=True, help="Project directory" + ) + parser.add_argument( + "--commit-before", type=str, required=True, help="Commit before session" + ) + parser.add_argument( + "--commit-after", type=str, required=True, help="Commit after session" + ) + parser.add_argument( + "--subtask-id", type=str, default="test-subtask", help="Subtask ID" + ) + + args = parser.parse_args() + + async def main(): + insights = await extract_session_insights( + spec_dir=args.spec_dir, + project_dir=args.project_dir, + subtask_id=args.subtask_id, + session_num=1, + commit_before=args.commit_before, + commit_after=args.commit_after, + success=True, + recovery_manager=None, + ) + print(json.dumps(insights, indent=2)) + + asyncio.run(main()) diff --git a/auto-claude/analysis/project_analyzer.py b/auto-claude/analysis/project_analyzer.py new file mode 100644 index 00000000..74484684 --- /dev/null +++ b/auto-claude/analysis/project_analyzer.py @@ -0,0 +1,106 @@ +""" +Smart Project Analyzer for Dynamic Security Profiles +===================================================== + +FACADE MODULE: This module re-exports all functionality from the +auto-claude/project/ package for backward compatibility. + +The implementation has been refactored into focused modules: +- project/command_registry.py - Command registries +- project/models.py - Data structures +- project/config_parser.py - Config file parsing +- project/stack_detector.py - Stack detection +- project/framework_detector.py - Framework detection +- project/structure_analyzer.py - Project structure analysis +- project/analyzer.py - Main orchestration + +This file maintains the original API so existing imports continue to work. + +This system: +1. Detects languages, frameworks, databases, and infrastructure +2. Parses package.json scripts, Makefile targets, pyproject.toml scripts +3. Builds a tailored security profile for the specific project +4. Caches the profile for subsequent runs +5. Can re-analyze when project structure changes + +The goal: Allow an AI developer to run any command that's legitimately +needed for the detected tech stack, while blocking dangerous operations. +""" + +# Re-export all public API from the project module +from project import ( + # Command registries + BASE_COMMANDS, + VALIDATED_COMMANDS, + CustomScripts, + # Main classes + ProjectAnalyzer, + SecurityProfile, + TechnologyStack, + # Utility functions + get_or_create_profile, + is_command_allowed, + needs_validation, +) + +# Also re-export command registries for backward compatibility +from project.command_registry import ( + CLOUD_COMMANDS, + CODE_QUALITY_COMMANDS, + DATABASE_COMMANDS, + FRAMEWORK_COMMANDS, + INFRASTRUCTURE_COMMANDS, + LANGUAGE_COMMANDS, + PACKAGE_MANAGER_COMMANDS, + VERSION_MANAGER_COMMANDS, +) + +__all__ = [ + # Main classes + "ProjectAnalyzer", + "SecurityProfile", + "TechnologyStack", + "CustomScripts", + # Utility functions + "get_or_create_profile", + "is_command_allowed", + "needs_validation", + # Base command sets + "BASE_COMMANDS", + "VALIDATED_COMMANDS", + # Technology-specific command sets + "LANGUAGE_COMMANDS", + "PACKAGE_MANAGER_COMMANDS", + "FRAMEWORK_COMMANDS", + "DATABASE_COMMANDS", + "INFRASTRUCTURE_COMMANDS", + "CLOUD_COMMANDS", + "CODE_QUALITY_COMMANDS", + "VERSION_MANAGER_COMMANDS", +] + + +# ============================================================================= +# CLI for testing +# ============================================================================= + +if __name__ == "__main__": + import sys + from pathlib import Path + + if len(sys.argv) < 2: + print("Usage: python project_analyzer.py [--force]") + sys.exit(1) + + project_dir = Path(sys.argv[1]) + force = "--force" in sys.argv + + if not project_dir.exists(): + print(f"Error: {project_dir} does not exist") + sys.exit(1) + + profile = get_or_create_profile(project_dir, force_reanalyze=force) + + print("\nAllowed commands:") + for cmd in sorted(profile.get_all_allowed_commands()): + print(f" {cmd}") diff --git a/auto-claude/analysis/risk_classifier.py b/auto-claude/analysis/risk_classifier.py new file mode 100644 index 00000000..37488c08 --- /dev/null +++ b/auto-claude/analysis/risk_classifier.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python3 +""" +Risk Classifier Module +====================== + +Reads the AI-generated complexity_assessment.json and provides programmatic +access to risk classification and validation recommendations. + +This module serves as the bridge between the AI complexity assessor prompt +and the rest of the validation system. + +Usage: + from risk_classifier import RiskClassifier + + classifier = RiskClassifier() + assessment = classifier.load_assessment(spec_dir) + + if classifier.should_skip_validation(spec_dir): + print("Validation can be skipped for this task") + + test_types = classifier.get_required_test_types(spec_dir) +""" + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# ============================================================================= +# DATA CLASSES +# ============================================================================= + + +@dataclass +class ScopeAnalysis: + """Analysis of task scope.""" + + estimated_files: int = 0 + estimated_services: int = 0 + is_cross_cutting: bool = False + notes: str = "" + + +@dataclass +class IntegrationAnalysis: + """Analysis of external integrations.""" + + external_services: list[str] = field(default_factory=list) + new_dependencies: list[str] = field(default_factory=list) + research_needed: bool = False + notes: str = "" + + +@dataclass +class InfrastructureAnalysis: + """Analysis of infrastructure requirements.""" + + docker_changes: bool = False + database_changes: bool = False + config_changes: bool = False + notes: str = "" + + +@dataclass +class KnowledgeAnalysis: + """Analysis of knowledge requirements.""" + + patterns_exist: bool = True + research_required: bool = False + unfamiliar_tech: list[str] = field(default_factory=list) + notes: str = "" + + +@dataclass +class RiskAnalysis: + """Analysis of task risk.""" + + level: str = "low" # low, medium, high + concerns: list[str] = field(default_factory=list) + notes: str = "" + + +@dataclass +class ComplexityAnalysis: + """Full complexity analysis from the AI assessor.""" + + scope: ScopeAnalysis = field(default_factory=ScopeAnalysis) + integrations: IntegrationAnalysis = field(default_factory=IntegrationAnalysis) + infrastructure: InfrastructureAnalysis = field( + default_factory=InfrastructureAnalysis + ) + knowledge: KnowledgeAnalysis = field(default_factory=KnowledgeAnalysis) + risk: RiskAnalysis = field(default_factory=RiskAnalysis) + + +@dataclass +class ValidationRecommendations: + """Validation recommendations from the AI assessor.""" + + risk_level: str = "medium" # trivial, low, medium, high, critical + skip_validation: bool = False + minimal_mode: bool = False + test_types_required: list[str] = field(default_factory=lambda: ["unit"]) + security_scan_required: bool = False + staging_deployment_required: bool = False + reasoning: str = "" + + +@dataclass +class AssessmentFlags: + """Flags indicating special requirements.""" + + needs_research: bool = False + needs_self_critique: bool = False + needs_infrastructure_setup: bool = False + + +@dataclass +class RiskAssessment: + """Complete risk assessment from complexity_assessment.json.""" + + complexity: str # simple, standard, complex + workflow_type: str # feature, refactor, investigation, migration, simple + confidence: float + reasoning: str + analysis: ComplexityAnalysis + recommended_phases: list[str] + flags: AssessmentFlags + validation: ValidationRecommendations + created_at: str | None = None + + @property + def risk_level(self) -> str: + """Get the risk level from validation recommendations.""" + return self.validation.risk_level + + +# ============================================================================= +# RISK CLASSIFIER +# ============================================================================= + + +class RiskClassifier: + """ + Reads AI-generated complexity_assessment.json and provides risk classification. + + The complexity_assessment.json is generated by the AI complexity assessor + agent using the complexity_assessor.md prompt. This module parses that output + and provides programmatic access to the risk classification. + """ + + def __init__(self) -> None: + """Initialize the risk classifier.""" + self._cache: dict[str, RiskAssessment] = {} + + def load_assessment(self, spec_dir: Path) -> RiskAssessment | None: + """ + Load complexity_assessment.json from spec directory. + + Args: + spec_dir: Path to the spec directory containing complexity_assessment.json + + Returns: + RiskAssessment object if file exists and is valid, None otherwise + """ + spec_dir = Path(spec_dir) + cache_key = str(spec_dir.resolve()) + + # Return cached result if available + if cache_key in self._cache: + return self._cache[cache_key] + + assessment_file = spec_dir / "complexity_assessment.json" + if not assessment_file.exists(): + return None + + try: + with open(assessment_file, encoding="utf-8") as f: + data = json.load(f) + + assessment = self._parse_assessment(data) + self._cache[cache_key] = assessment + return assessment + + except (json.JSONDecodeError, KeyError, TypeError) as e: + # Log error but don't crash - return None to allow fallback behavior + print(f"Warning: Failed to parse complexity_assessment.json: {e}") + return None + + def _parse_assessment(self, data: dict[str, Any]) -> RiskAssessment: + """Parse raw JSON data into a RiskAssessment object.""" + # Parse analysis sections + analysis_data = data.get("analysis", {}) + analysis = ComplexityAnalysis( + scope=self._parse_scope(analysis_data.get("scope", {})), + integrations=self._parse_integrations( + analysis_data.get("integrations", {}) + ), + infrastructure=self._parse_infrastructure( + analysis_data.get("infrastructure", {}) + ), + knowledge=self._parse_knowledge(analysis_data.get("knowledge", {})), + risk=self._parse_risk(analysis_data.get("risk", {})), + ) + + # Parse flags + flags_data = data.get("flags", {}) + flags = AssessmentFlags( + needs_research=flags_data.get("needs_research", False), + needs_self_critique=flags_data.get("needs_self_critique", False), + needs_infrastructure_setup=flags_data.get( + "needs_infrastructure_setup", False + ), + ) + + # Parse validation recommendations + validation_data = data.get("validation_recommendations", {}) + validation = self._parse_validation_recommendations(validation_data, analysis) + + return RiskAssessment( + complexity=data.get("complexity", "standard"), + workflow_type=data.get("workflow_type", "feature"), + confidence=float(data.get("confidence", 0.5)), + reasoning=data.get("reasoning", ""), + analysis=analysis, + recommended_phases=data.get("recommended_phases", []), + flags=flags, + validation=validation, + created_at=data.get("created_at"), + ) + + def _parse_scope(self, data: dict[str, Any]) -> ScopeAnalysis: + """Parse scope analysis section.""" + return ScopeAnalysis( + estimated_files=int(data.get("estimated_files", 0)), + estimated_services=int(data.get("estimated_services", 0)), + is_cross_cutting=bool(data.get("is_cross_cutting", False)), + notes=str(data.get("notes", "")), + ) + + def _parse_integrations(self, data: dict[str, Any]) -> IntegrationAnalysis: + """Parse integrations analysis section.""" + return IntegrationAnalysis( + external_services=list(data.get("external_services", [])), + new_dependencies=list(data.get("new_dependencies", [])), + research_needed=bool(data.get("research_needed", False)), + notes=str(data.get("notes", "")), + ) + + def _parse_infrastructure(self, data: dict[str, Any]) -> InfrastructureAnalysis: + """Parse infrastructure analysis section.""" + return InfrastructureAnalysis( + docker_changes=bool(data.get("docker_changes", False)), + database_changes=bool(data.get("database_changes", False)), + config_changes=bool(data.get("config_changes", False)), + notes=str(data.get("notes", "")), + ) + + def _parse_knowledge(self, data: dict[str, Any]) -> KnowledgeAnalysis: + """Parse knowledge analysis section.""" + return KnowledgeAnalysis( + patterns_exist=bool(data.get("patterns_exist", True)), + research_required=bool(data.get("research_required", False)), + unfamiliar_tech=list(data.get("unfamiliar_tech", [])), + notes=str(data.get("notes", "")), + ) + + def _parse_risk(self, data: dict[str, Any]) -> RiskAnalysis: + """Parse risk analysis section.""" + return RiskAnalysis( + level=str(data.get("level", "low")), + concerns=list(data.get("concerns", [])), + notes=str(data.get("notes", "")), + ) + + def _parse_validation_recommendations( + self, data: dict[str, Any], analysis: ComplexityAnalysis + ) -> ValidationRecommendations: + """ + Parse validation recommendations section. + + If validation_recommendations is not present in the JSON (older assessments), + infer appropriate values from the analysis. + """ + if data: + # New format with explicit validation recommendations + return ValidationRecommendations( + risk_level=str(data.get("risk_level", "medium")), + skip_validation=bool(data.get("skip_validation", False)), + minimal_mode=bool(data.get("minimal_mode", False)), + test_types_required=list(data.get("test_types_required", ["unit"])), + security_scan_required=bool(data.get("security_scan_required", False)), + staging_deployment_required=bool( + data.get("staging_deployment_required", False) + ), + reasoning=str(data.get("reasoning", "")), + ) + else: + # Infer from analysis (backward compatibility) + return self._infer_validation_recommendations(analysis) + + def _infer_validation_recommendations( + self, analysis: ComplexityAnalysis + ) -> ValidationRecommendations: + """ + Infer validation recommendations from analysis when not explicitly provided. + + This provides backward compatibility with older complexity assessments + that don't have the validation_recommendations section. + """ + risk_level = analysis.risk.level + + # Map old risk levels to new ones + risk_mapping = { + "low": "low", + "medium": "medium", + "high": "high", + } + normalized_risk = risk_mapping.get(risk_level, "medium") + + # Infer test types based on risk + test_types_map = { + "low": ["unit"], + "medium": ["unit", "integration"], + "high": ["unit", "integration", "e2e"], + } + test_types = test_types_map.get(normalized_risk, ["unit", "integration"]) + + # Security scan for high risk or security-related concerns + security_keywords = [ + "security", + "auth", + "password", + "credential", + "token", + "api key", + ] + has_security_concerns = any( + kw in str(analysis.risk.concerns).lower() for kw in security_keywords + ) + security_scan_required = normalized_risk == "high" or has_security_concerns + + # Staging for database or infrastructure changes + staging_required = ( + analysis.infrastructure.database_changes + and normalized_risk in ["medium", "high"] + ) + + # Minimal mode for simple changes + minimal_mode = ( + analysis.scope.estimated_files <= 2 + and analysis.scope.estimated_services <= 1 + and not analysis.integrations.external_services + ) + + return ValidationRecommendations( + risk_level=normalized_risk, + skip_validation=False, # Never skip by inference + minimal_mode=minimal_mode, + test_types_required=test_types, + security_scan_required=security_scan_required, + staging_deployment_required=staging_required, + reasoning="Inferred from complexity analysis (no explicit recommendations found)", + ) + + def should_skip_validation(self, spec_dir: Path) -> bool: + """ + Quick check if validation can be skipped entirely. + + Args: + spec_dir: Path to the spec directory + + Returns: + True if validation can be skipped (trivial changes), False otherwise + """ + assessment = self.load_assessment(spec_dir) + if not assessment: + return False # When in doubt, don't skip + + return assessment.validation.skip_validation + + def should_use_minimal_mode(self, spec_dir: Path) -> bool: + """ + Check if minimal validation mode should be used. + + Args: + spec_dir: Path to the spec directory + + Returns: + True if minimal mode is recommended, False otherwise + """ + assessment = self.load_assessment(spec_dir) + if not assessment: + return False + + return assessment.validation.minimal_mode + + def get_required_test_types(self, spec_dir: Path) -> list[str]: + """ + Get list of required test types based on risk. + + Args: + spec_dir: Path to the spec directory + + Returns: + List of test types (e.g., ["unit", "integration", "e2e"]) + """ + assessment = self.load_assessment(spec_dir) + if not assessment: + return ["unit"] # Default to unit tests + + return assessment.validation.test_types_required + + def requires_security_scan(self, spec_dir: Path) -> bool: + """ + Check if security scanning is required. + + Args: + spec_dir: Path to the spec directory + + Returns: + True if security scan is required, False otherwise + """ + assessment = self.load_assessment(spec_dir) + if not assessment: + return False + + return assessment.validation.security_scan_required + + def requires_staging_deployment(self, spec_dir: Path) -> bool: + """ + Check if staging deployment is required. + + Args: + spec_dir: Path to the spec directory + + Returns: + True if staging deployment is required, False otherwise + """ + assessment = self.load_assessment(spec_dir) + if not assessment: + return False + + return assessment.validation.staging_deployment_required + + def get_risk_level(self, spec_dir: Path) -> str: + """ + Get the risk level for the task. + + Args: + spec_dir: Path to the spec directory + + Returns: + Risk level string (trivial, low, medium, high, critical) + """ + assessment = self.load_assessment(spec_dir) + if not assessment: + return "medium" # Default to medium when unknown + + return assessment.validation.risk_level + + def get_complexity(self, spec_dir: Path) -> str: + """ + Get the complexity level for the task. + + Args: + spec_dir: Path to the spec directory + + Returns: + Complexity level string (simple, standard, complex) + """ + assessment = self.load_assessment(spec_dir) + if not assessment: + return "standard" # Default to standard when unknown + + return assessment.complexity + + def get_validation_summary(self, spec_dir: Path) -> dict[str, Any]: + """ + Get a summary of validation requirements. + + Args: + spec_dir: Path to the spec directory + + Returns: + Dictionary with validation summary + """ + assessment = self.load_assessment(spec_dir) + if not assessment: + return { + "risk_level": "unknown", + "complexity": "unknown", + "skip_validation": False, + "minimal_mode": False, + "test_types": ["unit"], + "security_scan": False, + "staging_deployment": False, + "confidence": 0.0, + } + + return { + "risk_level": assessment.validation.risk_level, + "complexity": assessment.complexity, + "skip_validation": assessment.validation.skip_validation, + "minimal_mode": assessment.validation.minimal_mode, + "test_types": assessment.validation.test_types_required, + "security_scan": assessment.validation.security_scan_required, + "staging_deployment": assessment.validation.staging_deployment_required, + "confidence": assessment.confidence, + "reasoning": assessment.validation.reasoning, + } + + def clear_cache(self) -> None: + """Clear the internal cache of loaded assessments.""" + self._cache.clear() + + +# ============================================================================= +# CONVENIENCE FUNCTIONS +# ============================================================================= + + +def load_risk_assessment(spec_dir: Path) -> RiskAssessment | None: + """ + Convenience function to load a risk assessment. + + Args: + spec_dir: Path to the spec directory + + Returns: + RiskAssessment object or None + """ + classifier = RiskClassifier() + return classifier.load_assessment(spec_dir) + + +def get_validation_requirements(spec_dir: Path) -> dict[str, Any]: + """ + Convenience function to get validation requirements. + + Args: + spec_dir: Path to the spec directory + + Returns: + Dictionary with validation requirements + """ + classifier = RiskClassifier() + return classifier.get_validation_summary(spec_dir) + + +# ============================================================================= +# CLI +# ============================================================================= + + +def main() -> None: + """CLI entry point for testing.""" + import argparse + + parser = argparse.ArgumentParser(description="Load and display risk assessment") + parser.add_argument( + "spec_dir", + type=Path, + help="Path to spec directory with complexity_assessment.json", + ) + parser.add_argument("--json", action="store_true", help="Output as JSON") + + args = parser.parse_args() + + classifier = RiskClassifier() + summary = classifier.get_validation_summary(args.spec_dir) + + if args.json: + print(json.dumps(summary, indent=2)) + else: + print(f"Risk Level: {summary['risk_level']}") + print(f"Complexity: {summary['complexity']}") + print(f"Skip Validation: {summary['skip_validation']}") + print(f"Minimal Mode: {summary['minimal_mode']}") + print(f"Test Types: {', '.join(summary['test_types'])}") + print(f"Security Scan: {summary['security_scan']}") + print(f"Staging Deployment: {summary['staging_deployment']}") + print(f"Confidence: {summary['confidence']:.2f}") + if summary.get("reasoning"): + print(f"Reasoning: {summary['reasoning']}") + + +if __name__ == "__main__": + main() diff --git a/auto-claude/analysis/security_scanner.py b/auto-claude/analysis/security_scanner.py new file mode 100644 index 00000000..3b1b8b42 --- /dev/null +++ b/auto-claude/analysis/security_scanner.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python3 +""" +Security Scanner Module +======================= + +Consolidates security scanning including secrets detection and SAST tools. +This module integrates the existing scan_secrets.py and provides a unified +interface for all security scanning. + +The security scanner is used by: +- QA Agent: To verify no secrets are committed +- Validation Strategy: To run security scans for high-risk changes + +Usage: + from analysis.security_scanner import SecurityScanner + + scanner = SecurityScanner() + results = scanner.scan(project_dir, spec_dir) + + if results.has_critical_issues: + print("Security issues found - blocking QA approval") +""" + +import json +import subprocess +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# Import the existing secrets scanner +try: + from security.scan_secrets import SecretMatch, get_all_tracked_files, scan_files + + HAS_SECRETS_SCANNER = True +except ImportError: + HAS_SECRETS_SCANNER = False + SecretMatch = None + + +# ============================================================================= +# DATA CLASSES +# ============================================================================= + + +@dataclass +class SecurityVulnerability: + """ + Represents a security vulnerability found during scanning. + + Attributes: + severity: Severity level (critical, high, medium, low, info) + source: Which scanner found this (secrets, bandit, npm_audit, etc.) + title: Short title of the vulnerability + description: Detailed description + file: File where vulnerability was found (if applicable) + line: Line number (if applicable) + cwe: CWE identifier if available + """ + + severity: str # critical, high, medium, low, info + source: str # secrets, bandit, npm_audit, semgrep, etc. + title: str + description: str + file: str | None = None + line: int | None = None + cwe: str | None = None + + +@dataclass +class SecurityScanResult: + """ + Result of a security scan. + + Attributes: + secrets: List of detected secrets + vulnerabilities: List of security vulnerabilities + scan_errors: List of errors during scanning + has_critical_issues: Whether any critical issues were found + should_block_qa: Whether these results should block QA approval + """ + + secrets: list[dict[str, Any]] = field(default_factory=list) + vulnerabilities: list[SecurityVulnerability] = field(default_factory=list) + scan_errors: list[str] = field(default_factory=list) + has_critical_issues: bool = False + should_block_qa: bool = False + + +# ============================================================================= +# SECURITY SCANNER +# ============================================================================= + + +class SecurityScanner: + """ + Consolidates all security scanning operations. + + Integrates: + - scan_secrets.py for secrets detection + - Bandit for Python SAST (if available) + - npm audit for JavaScript vulnerabilities (if applicable) + """ + + def __init__(self) -> None: + """Initialize the security scanner.""" + self._bandit_available: bool | None = None + self._npm_available: bool | None = None + + def scan( + self, + project_dir: Path, + spec_dir: Path | None = None, + changed_files: list[str] | None = None, + run_secrets: bool = True, + run_sast: bool = True, + run_dependency_audit: bool = True, + ) -> SecurityScanResult: + """ + Run all applicable security scans. + + Args: + project_dir: Path to the project root + spec_dir: Path to the spec directory (for storing results) + changed_files: Optional list of files to scan (if None, scans all) + run_secrets: Whether to run secrets scanning + run_sast: Whether to run SAST tools + run_dependency_audit: Whether to run dependency audits + + Returns: + SecurityScanResult with all findings + """ + project_dir = Path(project_dir) + result = SecurityScanResult() + + # Run secrets scan + if run_secrets: + self._run_secrets_scan(project_dir, changed_files, result) + + # Run SAST based on project type + if run_sast: + self._run_sast_scans(project_dir, result) + + # Run dependency audits + if run_dependency_audit: + self._run_dependency_audits(project_dir, result) + + # Determine if should block QA + result.has_critical_issues = ( + any(v.severity in ["critical", "high"] for v in result.vulnerabilities) + or len(result.secrets) > 0 + ) + + # Any secrets always block, critical vulnerabilities block + result.should_block_qa = len(result.secrets) > 0 or any( + v.severity == "critical" for v in result.vulnerabilities + ) + + # Save results if spec_dir provided + if spec_dir: + self._save_results(spec_dir, result) + + return result + + def _run_secrets_scan( + self, + project_dir: Path, + changed_files: list[str] | None, + result: SecurityScanResult, + ) -> None: + """Run secrets scanning using scan_secrets.py.""" + if not HAS_SECRETS_SCANNER: + result.scan_errors.append("scan_secrets module not available") + return + + try: + # Get files to scan + if changed_files: + files_to_scan = changed_files + else: + files_to_scan = get_all_tracked_files() + + # Run scan + matches = scan_files(files_to_scan, project_dir) + + # Convert matches to result format + for match in matches: + result.secrets.append( + { + "file": match.file_path, + "line": match.line_number, + "pattern": match.pattern_name, + "matched_text": self._redact_secret(match.matched_text), + } + ) + + # Also add as vulnerability + result.vulnerabilities.append( + SecurityVulnerability( + severity="critical", + source="secrets", + title=f"Potential secret: {match.pattern_name}", + description=f"Found potential {match.pattern_name} in file", + file=match.file_path, + line=match.line_number, + ) + ) + + except Exception as e: + result.scan_errors.append(f"Secrets scan error: {str(e)}") + + def _run_sast_scans(self, project_dir: Path, result: SecurityScanResult) -> None: + """Run SAST tools based on project type.""" + # Python SAST with Bandit + if self._is_python_project(project_dir): + self._run_bandit(project_dir, result) + + # JavaScript/Node.js - npm audit + # (handled in dependency audits for Node projects) + + def _run_bandit(self, project_dir: Path, result: SecurityScanResult) -> None: + """Run Bandit security scanner for Python projects.""" + if not self._check_bandit_available(): + return + + try: + # Find Python source directories + src_dirs = [] + for candidate in ["src", "app", project_dir.name, "."]: + candidate_path = project_dir / candidate + if ( + candidate_path.exists() + and (candidate_path / "__init__.py").exists() + ): + src_dirs.append(str(candidate_path)) + + if not src_dirs: + # Try to find any Python files + py_files = list(project_dir.glob("**/*.py")) + if not py_files: + return + src_dirs = ["."] + + # Run bandit + cmd = [ + "bandit", + "-r", + *src_dirs, + "-f", + "json", + "--exit-zero", # Don't fail on findings + ] + + proc = subprocess.run( + cmd, + cwd=project_dir, + capture_output=True, + text=True, + timeout=120, + ) + + if proc.stdout: + try: + bandit_output = json.loads(proc.stdout) + for finding in bandit_output.get("results", []): + severity = finding.get("issue_severity", "MEDIUM").lower() + if severity == "high": + severity = "high" + elif severity == "medium": + severity = "medium" + else: + severity = "low" + + result.vulnerabilities.append( + SecurityVulnerability( + severity=severity, + source="bandit", + title=finding.get("issue_text", "Unknown issue"), + description=finding.get("issue_text", ""), + file=finding.get("filename"), + line=finding.get("line_number"), + cwe=finding.get("issue_cwe", {}).get("id"), + ) + ) + except json.JSONDecodeError: + result.scan_errors.append("Failed to parse Bandit output") + + except subprocess.TimeoutExpired: + result.scan_errors.append("Bandit scan timed out") + except FileNotFoundError: + result.scan_errors.append("Bandit not found") + except Exception as e: + result.scan_errors.append(f"Bandit error: {str(e)}") + + def _run_dependency_audits( + self, project_dir: Path, result: SecurityScanResult + ) -> None: + """Run dependency vulnerability audits.""" + # npm audit for JavaScript projects + if (project_dir / "package.json").exists(): + self._run_npm_audit(project_dir, result) + + # pip-audit for Python projects (if available) + if self._is_python_project(project_dir): + self._run_pip_audit(project_dir, result) + + def _run_npm_audit(self, project_dir: Path, result: SecurityScanResult) -> None: + """Run npm audit for JavaScript projects.""" + try: + cmd = ["npm", "audit", "--json"] + + proc = subprocess.run( + cmd, + cwd=project_dir, + capture_output=True, + text=True, + timeout=120, + ) + + if proc.stdout: + try: + audit_output = json.loads(proc.stdout) + + # npm audit v2+ format + vulnerabilities = audit_output.get("vulnerabilities", {}) + for pkg_name, vuln_info in vulnerabilities.items(): + severity = vuln_info.get("severity", "moderate") + if severity == "critical": + severity = "critical" + elif severity == "high": + severity = "high" + elif severity == "moderate": + severity = "medium" + else: + severity = "low" + + result.vulnerabilities.append( + SecurityVulnerability( + severity=severity, + source="npm_audit", + title=f"Vulnerable dependency: {pkg_name}", + description=vuln_info.get("via", [{}])[0].get( + "title", "" + ) + if isinstance(vuln_info.get("via"), list) + and vuln_info.get("via") + else str(vuln_info.get("via", "")), + file="package.json", + ) + ) + except json.JSONDecodeError: + pass # npm audit may return invalid JSON on no findings + + except subprocess.TimeoutExpired: + result.scan_errors.append("npm audit timed out") + except FileNotFoundError: + pass # npm not available + except Exception as e: + result.scan_errors.append(f"npm audit error: {str(e)}") + + def _run_pip_audit(self, project_dir: Path, result: SecurityScanResult) -> None: + """Run pip-audit for Python projects (if available).""" + try: + cmd = ["pip-audit", "--format", "json"] + + proc = subprocess.run( + cmd, + cwd=project_dir, + capture_output=True, + text=True, + timeout=120, + ) + + if proc.stdout: + try: + audit_output = json.loads(proc.stdout) + for vuln in audit_output: + severity = "high" if vuln.get("fix_versions") else "medium" + + result.vulnerabilities.append( + SecurityVulnerability( + severity=severity, + source="pip_audit", + title=f"Vulnerable package: {vuln.get('name')}", + description=vuln.get("description", ""), + cwe=vuln.get("aliases", [""])[0] + if vuln.get("aliases") + else None, + ) + ) + except json.JSONDecodeError: + pass + + except FileNotFoundError: + pass # pip-audit not available + except subprocess.TimeoutExpired: + pass + except Exception: + pass + + def _is_python_project(self, project_dir: Path) -> bool: + """Check if this is a Python project.""" + indicators = [ + project_dir / "pyproject.toml", + project_dir / "requirements.txt", + project_dir / "setup.py", + project_dir / "setup.cfg", + ] + return any(p.exists() for p in indicators) + + def _check_bandit_available(self) -> bool: + """Check if Bandit is available.""" + if self._bandit_available is None: + try: + subprocess.run( + ["bandit", "--version"], + capture_output=True, + timeout=5, + ) + self._bandit_available = True + except (FileNotFoundError, subprocess.TimeoutExpired): + self._bandit_available = False + return self._bandit_available + + def _redact_secret(self, text: str) -> str: + """Redact a secret for safe logging.""" + if len(text) <= 8: + return "*" * len(text) + return text[:4] + "*" * (len(text) - 8) + text[-4:] + + def _save_results(self, spec_dir: Path, result: SecurityScanResult) -> None: + """Save scan results to spec directory.""" + spec_dir = Path(spec_dir) + spec_dir.mkdir(parents=True, exist_ok=True) + + output_file = spec_dir / "security_scan_results.json" + output_data = self.to_dict(result) + + with open(output_file, "w", encoding="utf-8") as f: + json.dump(output_data, f, indent=2) + + def to_dict(self, result: SecurityScanResult) -> dict[str, Any]: + """Convert result to dictionary for JSON serialization.""" + return { + "secrets": result.secrets, + "vulnerabilities": [ + { + "severity": v.severity, + "source": v.source, + "title": v.title, + "description": v.description, + "file": v.file, + "line": v.line, + "cwe": v.cwe, + } + for v in result.vulnerabilities + ], + "scan_errors": result.scan_errors, + "has_critical_issues": result.has_critical_issues, + "should_block_qa": result.should_block_qa, + "summary": { + "total_secrets": len(result.secrets), + "total_vulnerabilities": len(result.vulnerabilities), + "critical_count": sum( + 1 for v in result.vulnerabilities if v.severity == "critical" + ), + "high_count": sum( + 1 for v in result.vulnerabilities if v.severity == "high" + ), + "medium_count": sum( + 1 for v in result.vulnerabilities if v.severity == "medium" + ), + "low_count": sum( + 1 for v in result.vulnerabilities if v.severity == "low" + ), + }, + } + + +# ============================================================================= +# CONVENIENCE FUNCTIONS +# ============================================================================= + + +def scan_for_security_issues( + project_dir: Path, + spec_dir: Path | None = None, + changed_files: list[str] | None = None, +) -> SecurityScanResult: + """ + Convenience function to run security scan. + + Args: + project_dir: Path to project root + spec_dir: Optional spec directory to save results + changed_files: Optional list of files to scan + + Returns: + SecurityScanResult with all findings + """ + scanner = SecurityScanner() + return scanner.scan(project_dir, spec_dir, changed_files) + + +def has_security_issues(project_dir: Path) -> bool: + """ + Quick check if project has security issues. + + Args: + project_dir: Path to project root + + Returns: + True if any critical/high issues found + """ + scanner = SecurityScanner() + result = scanner.scan(project_dir, run_sast=False, run_dependency_audit=False) + return result.has_critical_issues + + +def scan_secrets_only( + project_dir: Path, + changed_files: list[str] | None = None, +) -> list[dict[str, Any]]: + """ + Scan only for secrets (quick scan). + + Args: + project_dir: Path to project root + changed_files: Optional list of files to scan + + Returns: + List of detected secrets + """ + scanner = SecurityScanner() + result = scanner.scan( + project_dir, + changed_files=changed_files, + run_sast=False, + run_dependency_audit=False, + ) + return result.secrets + + +# ============================================================================= +# CLI +# ============================================================================= + + +def main() -> None: + """CLI entry point for testing.""" + import argparse + + parser = argparse.ArgumentParser(description="Run security scans") + parser.add_argument("project_dir", type=Path, help="Path to project root") + parser.add_argument("--spec-dir", type=Path, help="Path to spec directory") + parser.add_argument( + "--secrets-only", action="store_true", help="Only scan for secrets" + ) + parser.add_argument("--json", action="store_true", help="Output as JSON") + + args = parser.parse_args() + + scanner = SecurityScanner() + result = scanner.scan( + args.project_dir, + spec_dir=args.spec_dir, + run_sast=not args.secrets_only, + run_dependency_audit=not args.secrets_only, + ) + + if args.json: + print(json.dumps(scanner.to_dict(result), indent=2)) + else: + print(f"Secrets Found: {len(result.secrets)}") + print(f"Vulnerabilities: {len(result.vulnerabilities)}") + print(f"Has Critical Issues: {result.has_critical_issues}") + print(f"Should Block QA: {result.should_block_qa}") + + if result.secrets: + print("\nSecrets Detected:") + for secret in result.secrets: + print(f" - {secret['pattern']} in {secret['file']}:{secret['line']}") + + if result.vulnerabilities: + print(f"\nVulnerabilities ({len(result.vulnerabilities)}):") + for v in result.vulnerabilities: + print(f" [{v.severity.upper()}] {v.title}") + if v.file: + print(f" File: {v.file}:{v.line or ''}") + + if result.scan_errors: + print(f"\nScan Errors ({len(result.scan_errors)}):") + for error in result.scan_errors: + print(f" - {error}") + + +if __name__ == "__main__": + main() diff --git a/auto-claude/analysis/test_discovery.py b/auto-claude/analysis/test_discovery.py new file mode 100644 index 00000000..9cd1c689 --- /dev/null +++ b/auto-claude/analysis/test_discovery.py @@ -0,0 +1,688 @@ +#!/usr/bin/env python3 +""" +Test Discovery Module +===================== + +Detects test frameworks, test commands, and test directories in a project. +This module analyzes project configuration files to discover how tests +should be run. + +The test discovery results are used by: +- QA Agent: To determine what test commands to run +- Test Creator: To know what framework to use when creating tests +- Planner: To include correct test commands in verification strategy + +Usage: + from test_discovery import TestDiscovery + + discovery = TestDiscovery() + result = discovery.discover(project_dir) + + print(f"Test frameworks: {result['frameworks']}") + print(f"Test command: {result['test_command']}") +""" + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# ============================================================================= +# DATA CLASSES +# ============================================================================= + + +@dataclass +class TestFramework: + """ + Represents a detected test framework. + + Attributes: + name: Name of the framework (e.g., "pytest", "jest", "vitest") + type: Type of testing (unit, integration, e2e, all) + command: Command to run tests + config_file: Configuration file if found + version: Version if detected + coverage_command: Command for coverage if available + """ + + __test__ = False # Prevent pytest from collecting this as a test class + + name: str + type: str # unit, integration, e2e, all + command: str + config_file: str | None = None + version: str | None = None + coverage_command: str | None = None + + +@dataclass +class TestDiscoveryResult: + """ + Result of test framework discovery. + + Attributes: + frameworks: List of detected test frameworks + test_command: Primary test command to run + test_directories: Discovered test directories + package_manager: Detected package manager + has_tests: Whether any test files were found + coverage_command: Command for coverage if available + """ + + __test__ = False # Prevent pytest from collecting this as a test class + + frameworks: list[TestFramework] = field(default_factory=list) + test_command: str = "" + test_directories: list[str] = field(default_factory=list) + package_manager: str = "" + has_tests: bool = False + coverage_command: str | None = None + + +# ============================================================================= +# FRAMEWORK DETECTORS +# ============================================================================= + + +# Pattern-based framework detection +FRAMEWORK_PATTERNS = { + # JavaScript/TypeScript + "jest": { + "config_files": [ + "jest.config.js", + "jest.config.ts", + "jest.config.mjs", + "jest.config.cjs", + ], + "package_key": "jest", + "type": "unit", + "command": "npx jest", + "coverage_command": "npx jest --coverage", + }, + "vitest": { + "config_files": ["vitest.config.js", "vitest.config.ts", "vitest.config.mjs"], + "package_key": "vitest", + "type": "unit", + "command": "npx vitest run", + "coverage_command": "npx vitest run --coverage", + }, + "mocha": { + "config_files": [ + ".mocharc.js", + ".mocharc.json", + ".mocharc.yaml", + ".mocharc.yml", + ], + "package_key": "mocha", + "type": "unit", + "command": "npx mocha", + "coverage_command": "npx nyc mocha", + }, + "playwright": { + "config_files": ["playwright.config.js", "playwright.config.ts"], + "package_key": "@playwright/test", + "type": "e2e", + "command": "npx playwright test", + "coverage_command": None, + }, + "cypress": { + "config_files": ["cypress.config.js", "cypress.config.ts", "cypress.json"], + "package_key": "cypress", + "type": "e2e", + "command": "npx cypress run", + "coverage_command": None, + }, + # Python + "pytest": { + "config_files": ["pytest.ini", "pyproject.toml", "setup.cfg", "conftest.py"], + "pyproject_key": "pytest", + "requirements_key": "pytest", + "type": "all", + "command": "pytest", + "coverage_command": "pytest --cov", + }, + "unittest": { + "config_files": [], + "type": "unit", + "command": "python -m unittest discover", + "coverage_command": "coverage run -m unittest discover", + }, + # Rust + "cargo_test": { + "config_files": ["Cargo.toml"], + "type": "all", + "command": "cargo test", + "coverage_command": "cargo tarpaulin", + }, + # Go + "go_test": { + "config_files": ["go.mod"], + "type": "all", + "command": "go test ./...", + "coverage_command": "go test -cover ./...", + }, + # Ruby + "rspec": { + "config_files": [".rspec", "spec/spec_helper.rb"], + "gemfile_key": "rspec", + "type": "all", + "command": "bundle exec rspec", + "coverage_command": "bundle exec rspec --format documentation", + }, + "minitest": { + "config_files": [], + "gemfile_key": "minitest", + "type": "unit", + "command": "bundle exec rake test", + "coverage_command": None, + }, +} + + +# ============================================================================= +# TEST DISCOVERY +# ============================================================================= + + +class TestDiscovery: + """ + Discovers test frameworks and configurations in a project. + + Analyzes: + - Package files (package.json, pyproject.toml, Cargo.toml, etc.) + - Configuration files (jest.config.js, pytest.ini, etc.) + - Directory structure (tests/, spec/, __tests__/) + """ + + __test__ = False # Prevent pytest from collecting this as a test class + + def __init__(self) -> None: + """Initialize the test discovery.""" + self._cache: dict[str, TestDiscoveryResult] = {} + + def discover(self, project_dir: Path) -> TestDiscoveryResult: + """ + Discover test frameworks and configuration in the project. + + Args: + project_dir: Path to the project root + + Returns: + TestDiscoveryResult with detected frameworks and commands + """ + project_dir = Path(project_dir) + cache_key = str(project_dir.resolve()) + + if cache_key in self._cache: + return self._cache[cache_key] + + result = TestDiscoveryResult() + + # Detect package manager + result.package_manager = self._detect_package_manager(project_dir) + + # Discover frameworks based on project type + if (project_dir / "package.json").exists(): + self._discover_js_frameworks(project_dir, result) + + # Check for Python project indicators + python_indicators = [ + project_dir / "pyproject.toml", + project_dir / "requirements.txt", + project_dir / "setup.py", + project_dir / "pytest.ini", + project_dir / "conftest.py", + project_dir / "tests" / "conftest.py", + ] + if any(p.exists() for p in python_indicators): + self._discover_python_frameworks(project_dir, result) + + if (project_dir / "Cargo.toml").exists(): + self._discover_rust_frameworks(project_dir, result) + if (project_dir / "go.mod").exists(): + self._discover_go_frameworks(project_dir, result) + if (project_dir / "Gemfile").exists(): + self._discover_ruby_frameworks(project_dir, result) + + # Find test directories + result.test_directories = self._find_test_directories(project_dir) + + # Check if tests exist + result.has_tests = self._has_test_files(project_dir, result.test_directories) + + # Set primary test command + if result.frameworks: + result.test_command = result.frameworks[0].command + + # Set coverage command from first framework that has one + if not result.coverage_command: + for framework in result.frameworks: + if framework.coverage_command: + result.coverage_command = framework.coverage_command + break + + self._cache[cache_key] = result + return result + + def _detect_package_manager(self, project_dir: Path) -> str: + """Detect the package manager used by the project.""" + if (project_dir / "pnpm-lock.yaml").exists(): + return "pnpm" + if (project_dir / "yarn.lock").exists(): + return "yarn" + if (project_dir / "package-lock.json").exists(): + return "npm" + if (project_dir / "bun.lockb").exists(): + return "bun" + if (project_dir / "uv.lock").exists(): + return "uv" + if (project_dir / "poetry.lock").exists(): + return "poetry" + if (project_dir / "Pipfile.lock").exists(): + return "pipenv" + if (project_dir / "Cargo.lock").exists(): + return "cargo" + if (project_dir / "go.sum").exists(): + return "go" + if (project_dir / "Gemfile.lock").exists(): + return "bundler" + return "" + + def _discover_js_frameworks( + self, project_dir: Path, result: TestDiscoveryResult + ) -> None: + """Discover JavaScript/TypeScript test frameworks.""" + package_json = project_dir / "package.json" + if not package_json.exists(): + return + + try: + with open(package_json, encoding="utf-8") as f: + pkg = json.load(f) + except (OSError, json.JSONDecodeError): + return + + deps = pkg.get("dependencies", {}) + dev_deps = pkg.get("devDependencies", {}) + all_deps = {**deps, **dev_deps} + scripts = pkg.get("scripts", {}) + + # Check for test frameworks in dependencies + for name, pattern in FRAMEWORK_PATTERNS.items(): + if "package_key" not in pattern: + continue + + if pattern["package_key"] in all_deps: + # Check for config file + config_file = None + for cf in pattern.get("config_files", []): + if (project_dir / cf).exists(): + config_file = cf + break + + # Get version + version = all_deps.get(pattern["package_key"], "") + if version.startswith("^") or version.startswith("~"): + version = version[1:] + + # Determine command - prefer npm scripts if available + command = pattern["command"] + if "test" in scripts and pattern["package_key"] in scripts.get( + "test", "" + ): + command = f"{result.package_manager or 'npm'} test" + + result.frameworks.append( + TestFramework( + name=name, + type=pattern["type"], + command=command, + config_file=config_file, + version=version, + coverage_command=pattern.get("coverage_command"), + ) + ) + + # Check npm scripts for test commands + if not result.frameworks and "test" in scripts: + test_script = scripts["test"] + if ( + test_script + and test_script != 'echo "Error: no test specified" && exit 1' + ): + # Try to infer framework from script + framework_name = "npm_test" + framework_type = "unit" + + if "jest" in test_script: + framework_name = "jest" + elif "vitest" in test_script: + framework_name = "vitest" + elif "mocha" in test_script: + framework_name = "mocha" + elif "playwright" in test_script: + framework_name = "playwright" + framework_type = "e2e" + elif "cypress" in test_script: + framework_name = "cypress" + framework_type = "e2e" + + result.frameworks.append( + TestFramework( + name=framework_name, + type=framework_type, + command=f"{result.package_manager or 'npm'} test", + config_file=None, + ) + ) + + def _discover_python_frameworks( + self, project_dir: Path, result: TestDiscoveryResult + ) -> None: + """Discover Python test frameworks.""" + # Check for pytest.ini first (explicit pytest config) + if (project_dir / "pytest.ini").exists(): + if not any(f.name == "pytest" for f in result.frameworks): + result.frameworks.append( + TestFramework( + name="pytest", + type="all", + command="pytest", + config_file="pytest.ini", + ) + ) + + # Check pyproject.toml + pyproject = project_dir / "pyproject.toml" + if pyproject.exists(): + content = pyproject.read_text() + + # Check for pytest + if "pytest" in content: + if not any(f.name == "pytest" for f in result.frameworks): + config_file = ( + "pyproject.toml" if "[tool.pytest" in content else None + ) + result.frameworks.append( + TestFramework( + name="pytest", + type="all", + command="pytest", + config_file=config_file, + ) + ) + + # Check requirements.txt + requirements = project_dir / "requirements.txt" + if requirements.exists(): + content = requirements.read_text().lower() + if "pytest" in content and not any( + f.name == "pytest" for f in result.frameworks + ): + result.frameworks.append( + TestFramework( + name="pytest", + type="all", + command="pytest", + config_file=None, + ) + ) + + # Check for conftest.py (pytest marker) + conftest_root = project_dir / "conftest.py" + conftest_tests = project_dir / "tests" / "conftest.py" + if conftest_root.exists() or conftest_tests.exists(): + if not any(f.name == "pytest" for f in result.frameworks): + result.frameworks.append( + TestFramework( + name="pytest", + type="all", + command="pytest", + config_file="conftest.py", + ) + ) + + # Fall back to unittest if test files exist but no framework detected + if not result.frameworks: + test_dirs = self._find_test_directories(project_dir) + if test_dirs: + result.frameworks.append( + TestFramework( + name="unittest", + type="unit", + command="python -m unittest discover", + config_file=None, + ) + ) + + def _discover_rust_frameworks( + self, project_dir: Path, result: TestDiscoveryResult + ) -> None: + """Discover Rust test frameworks.""" + cargo_toml = project_dir / "Cargo.toml" + if cargo_toml.exists(): + result.frameworks.append( + TestFramework( + name="cargo_test", + type="all", + command="cargo test", + config_file="Cargo.toml", + ) + ) + + def _discover_go_frameworks( + self, project_dir: Path, result: TestDiscoveryResult + ) -> None: + """Discover Go test frameworks.""" + go_mod = project_dir / "go.mod" + if go_mod.exists(): + result.frameworks.append( + TestFramework( + name="go_test", + type="all", + command="go test ./...", + config_file="go.mod", + ) + ) + + def _discover_ruby_frameworks( + self, project_dir: Path, result: TestDiscoveryResult + ) -> None: + """Discover Ruby test frameworks.""" + gemfile = project_dir / "Gemfile" + if not gemfile.exists(): + return + + content = gemfile.read_text().lower() + + if "rspec" in content or (project_dir / ".rspec").exists(): + result.frameworks.append( + TestFramework( + name="rspec", + type="all", + command="bundle exec rspec", + config_file=".rspec" if (project_dir / ".rspec").exists() else None, + ) + ) + elif "minitest" in content: + result.frameworks.append( + TestFramework( + name="minitest", + type="unit", + command="bundle exec rake test", + config_file=None, + ) + ) + + def _find_test_directories(self, project_dir: Path) -> list[str]: + """Find test directories in the project.""" + test_dir_patterns = [ + "tests", + "test", + "spec", + "__tests__", + "specs", + "test_*", + ] + + found_dirs = [] + for pattern in test_dir_patterns: + if pattern.endswith("*"): + # Glob pattern + for d in project_dir.glob(pattern): + if d.is_dir(): + found_dirs.append(str(d.relative_to(project_dir))) + else: + # Exact name + test_dir = project_dir / pattern + if test_dir.is_dir(): + found_dirs.append(pattern) + + return found_dirs + + def _has_test_files(self, project_dir: Path, test_directories: list[str]) -> bool: + """Check if any test files exist.""" + test_file_patterns = [ + "**/test_*.py", + "**/*_test.py", + "**/*.test.js", + "**/*.test.ts", + "**/*.test.tsx", + "**/*.spec.js", + "**/*.spec.ts", + "**/*.spec.tsx", + "**/test_*.go", + "**/*_test.go", + "**/*_test.rs", + "**/spec/**/*_spec.rb", + ] + + # Check in test directories + for test_dir in test_directories: + test_path = project_dir / test_dir + if test_path.exists(): + for pattern in test_file_patterns: + if list(test_path.glob(pattern.replace("**/", ""))): + return True + + # Check project-wide + for pattern in test_file_patterns: + if list(project_dir.glob(pattern)): + return True + + return False + + def to_dict(self, result: TestDiscoveryResult) -> dict[str, Any]: + """Convert result to dictionary for JSON serialization.""" + return { + "frameworks": [ + { + "name": f.name, + "type": f.type, + "command": f.command, + "config_file": f.config_file, + "version": f.version, + "coverage_command": f.coverage_command, + } + for f in result.frameworks + ], + "test_command": result.test_command, + "test_directories": result.test_directories, + "package_manager": result.package_manager, + "has_tests": result.has_tests, + "coverage_command": result.coverage_command, + } + + def clear_cache(self) -> None: + """Clear the internal cache.""" + self._cache.clear() + + +# ============================================================================= +# CONVENIENCE FUNCTIONS +# ============================================================================= + + +def discover_tests(project_dir: Path) -> TestDiscoveryResult: + """ + Convenience function to discover tests in a project. + + Args: + project_dir: Path to project root + + Returns: + TestDiscoveryResult with detected frameworks + """ + discovery = TestDiscovery() + return discovery.discover(project_dir) + + +def get_test_command(project_dir: Path) -> str: + """ + Get the primary test command for a project. + + Args: + project_dir: Path to project root + + Returns: + Test command string, or empty string if not found + """ + discovery = TestDiscovery() + result = discovery.discover(project_dir) + return result.test_command + + +def get_test_frameworks(project_dir: Path) -> list[str]: + """ + Get list of test framework names in a project. + + Args: + project_dir: Path to project root + + Returns: + List of framework names + """ + discovery = TestDiscovery() + result = discovery.discover(project_dir) + return [f.name for f in result.frameworks] + + +# ============================================================================= +# CLI +# ============================================================================= + + +def main() -> None: + """CLI entry point for testing.""" + import argparse + + parser = argparse.ArgumentParser(description="Discover test frameworks") + parser.add_argument("project_dir", type=Path, help="Path to project root") + parser.add_argument("--json", action="store_true", help="Output as JSON") + + args = parser.parse_args() + + discovery = TestDiscovery() + result = discovery.discover(args.project_dir) + + if args.json: + print(json.dumps(discovery.to_dict(result), indent=2)) + else: + print(f"Package Manager: {result.package_manager or 'unknown'}") + print(f"Has Tests: {result.has_tests}") + print(f"Test Command: {result.test_command or 'none'}") + print(f"Test Directories: {', '.join(result.test_directories) or 'none'}") + print(f"Coverage Command: {result.coverage_command or 'none'}") + print(f"\nFrameworks ({len(result.frameworks)}):") + for f in result.frameworks: + print(f" - {f.name} ({f.type})") + print(f" Command: {f.command}") + if f.config_file: + print(f" Config: {f.config_file}") + if f.version: + print(f" Version: {f.version}") + + +if __name__ == "__main__": + main() diff --git a/auto-claude/analyzer.py b/auto-claude/analyzer.py index 3513fa14..99b7ddb8 100644 --- a/auto-claude/analyzer.py +++ b/auto-claude/analyzer.py @@ -1,100 +1,2 @@ -#!/usr/bin/env python3 -""" -Codebase Analyzer -================= - -Automatically detects project structure, frameworks, and services. -Supports monorepos with multiple services. - -Usage: - # Index entire project (creates project_index.json) - python auto-claude/analyzer.py --index - - # Analyze specific service - python auto-claude/analyzer.py --service backend - - # Output to specific file - python auto-claude/analyzer.py --index --output path/to/output.json - -The analyzer will: -1. Detect if this is a monorepo or single project -2. Find all services/packages and analyze each separately -3. Map interdependencies between services -4. Identify infrastructure (Docker, CI/CD) -5. Document conventions (linting, testing) - -This module now serves as a facade to the modular analyzer system in the analyzers/ package. -All actual implementation is in focused submodules for better maintainability. -""" - -import json -from pathlib import Path - -# Import from the new modular structure -from analyzers import ( - ProjectAnalyzer, - ServiceAnalyzer, - analyze_project, - analyze_service, -) - -# Re-export for backward compatibility -__all__ = [ - "ServiceAnalyzer", - "ProjectAnalyzer", - "analyze_project", - "analyze_service", -] - - -def main(): - """CLI entry point.""" - import argparse - - parser = argparse.ArgumentParser( - description="Analyze project structure, frameworks, and services" - ) - parser.add_argument( - "--project-dir", - type=Path, - default=Path.cwd(), - help="Project directory to analyze (default: current directory)", - ) - parser.add_argument( - "--index", - action="store_true", - help="Create full project index (default behavior)", - ) - parser.add_argument( - "--service", - type=str, - default=None, - help="Analyze a specific service only", - ) - parser.add_argument( - "--output", - type=Path, - default=None, - help="Output file for JSON results", - ) - parser.add_argument( - "--quiet", - action="store_true", - help="Only output JSON, no status messages", - ) - - args = parser.parse_args() - - # Determine what to analyze - if args.service: - results = analyze_service(args.project_dir, args.service, args.output) - else: - results = analyze_project(args.project_dir, args.output) - - # Print results - if not args.quiet or not args.output: - print(json.dumps(results, indent=2)) - - -if __name__ == "__main__": - main() +"""Backward compatibility shim - import from analysis.analyzer instead.""" +from analysis.analyzer import * diff --git a/auto-claude/analyzers/__init__.py b/auto-claude/analyzers/__init__.py index b425b119..72fc2eff 100644 --- a/auto-claude/analyzers/__init__.py +++ b/auto-claude/analyzers/__init__.py @@ -1,92 +1,2 @@ -""" -Analyzers Package -================= - -Modular analyzer system for detecting project structure, frameworks, and services. - -Main exports: -- ServiceAnalyzer: Analyzes a single service/package -- ProjectAnalyzer: Analyzes entire projects (single or monorepo) -- analyze_project: Convenience function for project analysis -- analyze_service: Convenience function for service analysis -""" - -from pathlib import Path -from typing import Any - -from .project_analyzer_module import ProjectAnalyzer -from .service_analyzer import ServiceAnalyzer - -# Re-export main classes -__all__ = [ - "ServiceAnalyzer", - "ProjectAnalyzer", - "analyze_project", - "analyze_service", -] - - -def analyze_project(project_dir: Path, output_file: Path | None = None) -> dict: - """ - Analyze a project and optionally save results. - - Args: - project_dir: Path to the project root - output_file: Optional path to save JSON output - - Returns: - Project index as a dictionary - """ - import json - - analyzer = ProjectAnalyzer(project_dir) - results = analyzer.analyze() - - if output_file: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(results, f, indent=2) - print(f"Project index saved to: {output_file}") - - return results - - -def analyze_service( - project_dir: Path, service_name: str, output_file: Path | None = None -) -> dict: - """ - Analyze a specific service within a project. - - Args: - project_dir: Path to the project root - service_name: Name of the service to analyze - output_file: Optional path to save JSON output - - Returns: - Service analysis as a dictionary - """ - import json - - # Find the service - service_path = project_dir / service_name - if not service_path.exists(): - # Check common locations - for parent in ["packages", "apps", "services"]: - candidate = project_dir / parent / service_name - if candidate.exists(): - service_path = candidate - break - - if not service_path.exists(): - raise ValueError(f"Service '{service_name}' not found in {project_dir}") - - analyzer = ServiceAnalyzer(service_path, service_name) - results = analyzer.analyze() - - if output_file: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as f: - json.dump(results, f, indent=2) - print(f"Service analysis saved to: {output_file}") - - return results +"""Backward compatibility shim - import from analysis.analyzers instead.""" +from analysis.analyzers import * diff --git a/auto-claude/auto_claude_tools.py b/auto-claude/auto_claude_tools.py index cd4e18bb..5c7cee4e 100644 --- a/auto-claude/auto_claude_tools.py +++ b/auto-claude/auto_claude_tools.py @@ -1,45 +1,13 @@ -""" -Custom MCP Tools for Auto-Claude Agents -======================================== +"""Backward compatibility shim - import from agents.tools_pkg instead.""" -DEPRECATED: This module is now a compatibility shim. -Please import from the auto_claude_tools package instead: +# Direct import to avoid triggering agents.__init__ circular dependencies +import sys +from pathlib import Path - from auto_claude_tools import create_auto_claude_mcp_server, get_allowed_tools +# Add agents directory to path if needed +agents_dir = Path(__file__).parent / "agents" +if str(agents_dir) not in sys.path: + sys.path.insert(0, str(agents_dir)) -This file remains for backward compatibility with existing imports. -All functionality has been moved to the auto_claude_tools package for better -organization and maintainability. -""" - -# Import everything from the package to maintain backward compatibility -from auto_claude_tools import ( - ELECTRON_TOOLS, - TOOL_GET_BUILD_PROGRESS, - TOOL_GET_SESSION_CONTEXT, - TOOL_RECORD_DISCOVERY, - TOOL_RECORD_GOTCHA, - TOOL_UPDATE_QA_STATUS, - TOOL_UPDATE_SUBTASK_STATUS, - create_auto_claude_mcp_server, - get_allowed_tools, - is_electron_mcp_enabled, - is_tools_available, -) - -__all__ = [ - # Main API - "create_auto_claude_mcp_server", - "get_allowed_tools", - "is_tools_available", - # Tool name constants - "TOOL_UPDATE_SUBTASK_STATUS", - "TOOL_GET_BUILD_PROGRESS", - "TOOL_RECORD_DISCOVERY", - "TOOL_RECORD_GOTCHA", - "TOOL_GET_SESSION_CONTEXT", - "TOOL_UPDATE_QA_STATUS", - # Electron MCP - "ELECTRON_TOOLS", - "is_electron_mcp_enabled", -] +# Import directly from tools_pkg to avoid agents.__init__ circular imports +from tools_pkg import * # noqa: F403, E402 diff --git a/auto-claude/ci_discovery.py b/auto-claude/ci_discovery.py index 347546c4..a034aa69 100644 --- a/auto-claude/ci_discovery.py +++ b/auto-claude/ci_discovery.py @@ -1,587 +1,20 @@ -#!/usr/bin/env python3 -""" -CI Discovery Module -=================== - -Parses CI/CD configuration files to extract test commands and workflows. -Supports GitHub Actions, GitLab CI, CircleCI, and Jenkins. - -The CI discovery results are used by: -- QA Agent: To understand existing CI test patterns -- Validation Strategy: To match CI commands -- Planner: To align verification with CI - -Usage: - from ci_discovery import CIDiscovery - - discovery = CIDiscovery() - result = discovery.discover(project_dir) - - if result: - print(f"CI System: {result.ci_system}") - print(f"Test Commands: {result.test_commands}") -""" - -import json -import re -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -# Try to import yaml, fall back gracefully -try: - import yaml - - HAS_YAML = True -except ImportError: - HAS_YAML = False - - -# ============================================================================= -# DATA CLASSES -# ============================================================================= - - -@dataclass -class CIWorkflow: - """ - Represents a CI workflow or job. - - Attributes: - name: Name of the workflow/job - trigger: What triggers this workflow (push, pull_request, etc.) - steps: List of step names or commands - test_related: Whether this appears to be test-related - """ - - name: str - trigger: list[str] = field(default_factory=list) - steps: list[str] = field(default_factory=list) - test_related: bool = False - - -@dataclass -class CIConfig: - """ - Result of CI configuration discovery. - - Attributes: - ci_system: Name of CI system (github_actions, gitlab, circleci, jenkins) - config_files: List of CI config files found - test_commands: Extracted test commands by type - coverage_command: Coverage command if found - workflows: List of discovered workflows - environment_variables: Environment variables used - """ - - ci_system: str - config_files: list[str] = field(default_factory=list) - test_commands: dict[str, str] = field(default_factory=dict) - coverage_command: str | None = None - workflows: list[CIWorkflow] = field(default_factory=list) - environment_variables: list[str] = field(default_factory=list) - - -# ============================================================================= -# CI PARSERS -# ============================================================================= - - -class CIDiscovery: - """ - Discovers CI/CD configurations in a project. - - Analyzes: - - GitHub Actions (.github/workflows/*.yml) - - GitLab CI (.gitlab-ci.yml) - - CircleCI (.circleci/config.yml) - - Jenkins (Jenkinsfile) - """ - - def __init__(self) -> None: - """Initialize CI discovery.""" - self._cache: dict[str, CIConfig | None] = {} - - def discover(self, project_dir: Path) -> CIConfig | None: - """ - Discover CI configuration in the project. - - Args: - project_dir: Path to the project root - - Returns: - CIConfig if CI found, None otherwise - """ - project_dir = Path(project_dir) - cache_key = str(project_dir.resolve()) - - if cache_key in self._cache: - return self._cache[cache_key] - - # Try each CI system - result = None - - # GitHub Actions - github_workflows = project_dir / ".github" / "workflows" - if github_workflows.exists(): - result = self._parse_github_actions(github_workflows) - - # GitLab CI - if not result: - gitlab_ci = project_dir / ".gitlab-ci.yml" - if gitlab_ci.exists(): - result = self._parse_gitlab_ci(gitlab_ci) - - # CircleCI - if not result: - circleci = project_dir / ".circleci" / "config.yml" - if circleci.exists(): - result = self._parse_circleci(circleci) - - # Jenkins - if not result: - jenkinsfile = project_dir / "Jenkinsfile" - if jenkinsfile.exists(): - result = self._parse_jenkinsfile(jenkinsfile) - - self._cache[cache_key] = result - return result - - def _parse_github_actions(self, workflows_dir: Path) -> CIConfig: - """Parse GitHub Actions workflow files.""" - result = CIConfig(ci_system="github_actions") - - workflow_files = list(workflows_dir.glob("*.yml")) + list( - workflows_dir.glob("*.yaml") - ) - - for wf_file in workflow_files: - result.config_files.append( - str(wf_file.relative_to(workflows_dir.parent.parent)) - ) - - try: - content = wf_file.read_text() - workflow_data = self._parse_yaml(content) - - if not workflow_data: - continue - - # Get workflow name - wf_name = workflow_data.get("name", wf_file.stem) - - # Get triggers - triggers = [] - on_trigger = workflow_data.get("on", {}) - if isinstance(on_trigger, str): - triggers = [on_trigger] - elif isinstance(on_trigger, list): - triggers = on_trigger - elif isinstance(on_trigger, dict): - triggers = list(on_trigger.keys()) - - # Parse jobs - jobs = workflow_data.get("jobs", {}) - for job_name, job_config in jobs.items(): - if not isinstance(job_config, dict): - continue - - steps = job_config.get("steps", []) - step_commands = [] - test_related = False - - for step in steps: - if not isinstance(step, dict): - continue - - # Get step name or command - step_name = step.get("name", "") - run_cmd = step.get("run", "") - uses = step.get("uses", "") - - if step_name: - step_commands.append(step_name) - if run_cmd: - step_commands.append(run_cmd) - # Extract test commands - self._extract_test_commands(run_cmd, result) - if uses: - step_commands.append(f"uses: {uses}") - - # Check if test-related - test_keywords = ["test", "pytest", "jest", "vitest", "coverage"] - if any(kw in str(step).lower() for kw in test_keywords): - test_related = True - - result.workflows.append( - CIWorkflow( - name=f"{wf_name}/{job_name}", - trigger=triggers, - steps=step_commands, - test_related=test_related, - ) - ) - - # Extract environment variables - env = workflow_data.get("env", {}) - if isinstance(env, dict): - result.environment_variables.extend(env.keys()) - - except Exception: - continue - - return result - - def _parse_gitlab_ci(self, config_file: Path) -> CIConfig: - """Parse GitLab CI configuration.""" - result = CIConfig( - ci_system="gitlab", - config_files=[".gitlab-ci.yml"], - ) - - try: - content = config_file.read_text() - data = self._parse_yaml(content) - - if not data: - return result - - # Parse jobs (top-level keys that aren't special keywords) - special_keys = { - "stages", - "variables", - "image", - "services", - "before_script", - "after_script", - "cache", - "include", - "default", - "workflow", - } - - for key, value in data.items(): - if key.startswith(".") or key in special_keys: - continue - - if not isinstance(value, dict): - continue - - job_config = value - script = job_config.get("script", []) - if isinstance(script, str): - script = [script] - - test_related = any( - kw in str(script).lower() - for kw in ["test", "pytest", "jest", "vitest", "coverage"] - ) - - result.workflows.append( - CIWorkflow( - name=key, - trigger=job_config.get("only", []) - or job_config.get("rules", []), - steps=script, - test_related=test_related, - ) - ) - - # Extract test commands - for cmd in script: - if isinstance(cmd, str): - self._extract_test_commands(cmd, result) - - # Extract variables - variables = data.get("variables", {}) - if isinstance(variables, dict): - result.environment_variables.extend(variables.keys()) - - except Exception: - pass - - return result - - def _parse_circleci(self, config_file: Path) -> CIConfig: - """Parse CircleCI configuration.""" - result = CIConfig( - ci_system="circleci", - config_files=[".circleci/config.yml"], - ) - - try: - content = config_file.read_text() - data = self._parse_yaml(content) - - if not data: - return result - - # Parse jobs - jobs = data.get("jobs", {}) - for job_name, job_config in jobs.items(): - if not isinstance(job_config, dict): - continue - - steps = job_config.get("steps", []) - step_commands = [] - test_related = False - - for step in steps: - if isinstance(step, str): - step_commands.append(step) - elif isinstance(step, dict): - if "run" in step: - run = step["run"] - if isinstance(run, str): - step_commands.append(run) - self._extract_test_commands(run, result) - elif isinstance(run, dict): - cmd = run.get("command", "") - step_commands.append(cmd) - self._extract_test_commands(cmd, result) - - if any( - kw in str(step).lower() - for kw in ["test", "pytest", "jest", "coverage"] - ): - test_related = True - - result.workflows.append( - CIWorkflow( - name=job_name, - trigger=[], - steps=step_commands, - test_related=test_related, - ) - ) - - except Exception: - pass - - return result - - def _parse_jenkinsfile(self, jenkinsfile: Path) -> CIConfig: - """Parse Jenkinsfile (basic extraction).""" - result = CIConfig( - ci_system="jenkins", - config_files=["Jenkinsfile"], - ) - - try: - content = jenkinsfile.read_text() - - # Extract sh commands using regex - sh_pattern = re.compile(r'sh\s+[\'"]([^\'"]+)[\'"]') - matches = sh_pattern.findall(content) - - steps = [] - test_related = False - - for cmd in matches: - steps.append(cmd) - self._extract_test_commands(cmd, result) - - if any( - kw in cmd.lower() for kw in ["test", "pytest", "jest", "coverage"] - ): - test_related = True - - # Extract stage names - stage_pattern = re.compile(r'stage\s*\([\'"]([^\'"]+)[\'"]\)') - stages = stage_pattern.findall(content) - - for stage in stages: - result.workflows.append( - CIWorkflow( - name=stage, - trigger=[], - steps=steps if "test" in stage.lower() else [], - test_related="test" in stage.lower(), - ) - ) - - except Exception: - pass - - return result - - def _parse_yaml(self, content: str) -> dict | None: - """Parse YAML content, with fallback to basic parsing if yaml not available.""" - if HAS_YAML: - try: - return yaml.safe_load(content) - except Exception: - return None - - # Basic fallback for simple YAML (very limited) - # This won't work for complex structures - return None - - def _extract_test_commands(self, cmd: str, result: CIConfig) -> None: - """Extract test commands from a command string.""" - cmd_lower = cmd.lower() - - # Python pytest - if "pytest" in cmd_lower: - if "pytest" not in result.test_commands: - result.test_commands["unit"] = cmd.strip() - if "--cov" in cmd_lower: - result.coverage_command = cmd.strip() - - # Node.js test commands - if ( - "npm test" in cmd_lower - or "yarn test" in cmd_lower - or "pnpm test" in cmd_lower - ): - if "unit" not in result.test_commands: - result.test_commands["unit"] = cmd.strip() - - # Jest/Vitest - if "jest" in cmd_lower or "vitest" in cmd_lower: - if "unit" not in result.test_commands: - result.test_commands["unit"] = cmd.strip() - if "--coverage" in cmd_lower: - result.coverage_command = cmd.strip() - - # E2E testing - if "playwright" in cmd_lower: - result.test_commands["e2e"] = cmd.strip() - if "cypress" in cmd_lower: - result.test_commands["e2e"] = cmd.strip() - - # Integration tests - if "integration" in cmd_lower: - result.test_commands["integration"] = cmd.strip() - - # Go tests - if "go test" in cmd_lower: - if "unit" not in result.test_commands: - result.test_commands["unit"] = cmd.strip() - - # Rust tests - if "cargo test" in cmd_lower: - if "unit" not in result.test_commands: - result.test_commands["unit"] = cmd.strip() - - def to_dict(self, result: CIConfig) -> dict[str, Any]: - """Convert result to dictionary for JSON serialization.""" - return { - "ci_system": result.ci_system, - "config_files": result.config_files, - "test_commands": result.test_commands, - "coverage_command": result.coverage_command, - "workflows": [ - { - "name": w.name, - "trigger": w.trigger, - "steps": w.steps, - "test_related": w.test_related, - } - for w in result.workflows - ], - "environment_variables": result.environment_variables, - } - - def clear_cache(self) -> None: - """Clear the internal cache.""" - self._cache.clear() - - -# ============================================================================= -# CONVENIENCE FUNCTIONS -# ============================================================================= - - -def discover_ci(project_dir: Path) -> CIConfig | None: - """ - Convenience function to discover CI configuration. - - Args: - project_dir: Path to project root - - Returns: - CIConfig if found, None otherwise - """ - discovery = CIDiscovery() - return discovery.discover(project_dir) - - -def get_ci_test_commands(project_dir: Path) -> dict[str, str]: - """ - Get test commands from CI configuration. - - Args: - project_dir: Path to project root - - Returns: - Dictionary of test type to command - """ - discovery = CIDiscovery() - result = discovery.discover(project_dir) - if result: - return result.test_commands - return {} - - -def get_ci_system(project_dir: Path) -> str | None: - """ - Get the CI system name if configured. - - Args: - project_dir: Path to project root - - Returns: - CI system name or None - """ - discovery = CIDiscovery() - result = discovery.discover(project_dir) - if result: - return result.ci_system - return None - - -# ============================================================================= -# CLI -# ============================================================================= - - -def main() -> None: - """CLI entry point for testing.""" - import argparse - - parser = argparse.ArgumentParser(description="Discover CI configuration") - parser.add_argument("project_dir", type=Path, help="Path to project root") - parser.add_argument("--json", action="store_true", help="Output as JSON") - - args = parser.parse_args() - - discovery = CIDiscovery() - result = discovery.discover(args.project_dir) - - if not result: - print("No CI configuration found") - return - - if args.json: - print(json.dumps(discovery.to_dict(result), indent=2)) - else: - print(f"CI System: {result.ci_system}") - print(f"Config Files: {', '.join(result.config_files)}") - print("\nTest Commands:") - for test_type, cmd in result.test_commands.items(): - print(f" {test_type}: {cmd}") - if result.coverage_command: - print(f"\nCoverage Command: {result.coverage_command}") - print(f"\nWorkflows ({len(result.workflows)}):") - for w in result.workflows: - marker = "[TEST]" if w.test_related else "" - print(f" - {w.name} {marker}") - if w.trigger: - print(f" Triggers: {', '.join(str(t) for t in w.trigger)}") - if result.environment_variables: - print(f"\nEnvironment Variables: {', '.join(result.environment_variables)}") - - -if __name__ == "__main__": - main() +"""Backward compatibility shim - import from analysis.ci_discovery instead.""" +from analysis.ci_discovery import ( + CIConfig, + CIWorkflow, + CIDiscovery, + discover_ci, + get_ci_test_commands, + get_ci_system, + HAS_YAML, +) + +__all__ = [ + "CIConfig", + "CIWorkflow", + "CIDiscovery", + "discover_ci", + "get_ci_test_commands", + "get_ci_system", + "HAS_YAML", +] diff --git a/auto-claude/client.py b/auto-claude/client.py index 8a628f2a..0416d7dd 100644 --- a/auto-claude/client.py +++ b/auto-claude/client.py @@ -1,315 +1,24 @@ -""" -Claude SDK Client Configuration -=============================== - -Functions for creating and configuring the Claude Agent SDK client. -""" - -import json +"""Backward compatibility shim - import from core.client instead.""" +import sys import os -from pathlib import Path -from auto_claude_tools import ( - create_auto_claude_mcp_server, - is_tools_available, -) -from auto_claude_tools import ( - get_allowed_tools as get_agent_allowed_tools, -) -from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient -from claude_agent_sdk.types import HookMatcher -from linear_updater import is_linear_enabled -from security import bash_security_hook +# Add auto-claude to path if not present +_auto_claude_dir = os.path.dirname(os.path.abspath(__file__)) +if _auto_claude_dir not in sys.path: + sys.path.insert(0, _auto_claude_dir) +# Use lazy imports to avoid circular dependency +def __getattr__(name): + """Lazy import to avoid circular imports with auto_claude_tools.""" + from core import client as _client + return getattr(_client, name) -def is_graphiti_mcp_enabled() -> bool: - """ - Check if Graphiti MCP server integration is enabled. - - Requires GRAPHITI_MCP_URL to be set (e.g., http://localhost:8000/mcp/) - This is separate from GRAPHITI_ENABLED which controls the Python library integration. - """ - return bool(os.environ.get("GRAPHITI_MCP_URL")) - - -def get_graphiti_mcp_url() -> str: - """Get the Graphiti MCP server URL.""" - return os.environ.get("GRAPHITI_MCP_URL", "http://localhost:8000/mcp/") - - -def is_electron_mcp_enabled() -> bool: - """ - Check if Electron MCP server integration is enabled. - - Requires ELECTRON_MCP_ENABLED to be set to 'true'. - When enabled, QA agents can use Puppeteer MCP tools to connect to Electron apps - via Chrome DevTools Protocol on the configured debug port. - """ - return os.environ.get("ELECTRON_MCP_ENABLED", "").lower() == "true" - - -def get_electron_debug_port() -> int: - """Get the Electron remote debugging port (default: 9222).""" - return int(os.environ.get("ELECTRON_DEBUG_PORT", "9222")) - - -# Puppeteer MCP tools for browser automation -PUPPETEER_TOOLS = [ - "mcp__puppeteer__puppeteer_connect_active_tab", - "mcp__puppeteer__puppeteer_navigate", - "mcp__puppeteer__puppeteer_screenshot", - "mcp__puppeteer__puppeteer_click", - "mcp__puppeteer__puppeteer_fill", - "mcp__puppeteer__puppeteer_select", - "mcp__puppeteer__puppeteer_hover", - "mcp__puppeteer__puppeteer_evaluate", +# For wildcard imports, define __all__ +__all__ = [ + "create_claude_client", + "ClaudeClient", + "is_graphiti_mcp_enabled", + "get_graphiti_mcp_url", + "is_electron_mcp_enabled", + "get_electron_debug_port", ] - -# Linear MCP tools for project management (when LINEAR_API_KEY is set) -LINEAR_TOOLS = [ - "mcp__linear-server__list_teams", - "mcp__linear-server__get_team", - "mcp__linear-server__list_projects", - "mcp__linear-server__get_project", - "mcp__linear-server__create_project", - "mcp__linear-server__update_project", - "mcp__linear-server__list_issues", - "mcp__linear-server__get_issue", - "mcp__linear-server__create_issue", - "mcp__linear-server__update_issue", - "mcp__linear-server__list_comments", - "mcp__linear-server__create_comment", - "mcp__linear-server__list_issue_statuses", - "mcp__linear-server__list_issue_labels", - "mcp__linear-server__list_users", - "mcp__linear-server__get_user", -] - -# Context7 MCP tools for documentation lookup (always enabled) -CONTEXT7_TOOLS = [ - "mcp__context7__resolve-library-id", - "mcp__context7__get-library-docs", -] - -# Graphiti MCP tools for knowledge graph memory (when GRAPHITI_MCP_ENABLED is set) -# See: https://docs.falkordb.com/agentic-memory/graphiti-mcp-server.html -GRAPHITI_MCP_TOOLS = [ - "mcp__graphiti-memory__search_nodes", # Search entity summaries - "mcp__graphiti-memory__search_facts", # Search relationships between entities - "mcp__graphiti-memory__add_episode", # Add data to knowledge graph - "mcp__graphiti-memory__get_episodes", # Retrieve recent episodes - "mcp__graphiti-memory__get_entity_edge", # Get specific entity/relationship -] - -# Electron MCP tools for desktop app automation (when ELECTRON_MCP_ENABLED is set) -# Uses puppeteer-mcp-server to connect to Electron apps via Chrome DevTools Protocol. -# Electron app must be started with --remote-debugging-port=9222 (or ELECTRON_DEBUG_PORT). -# These tools are only available to QA agents (qa_reviewer, qa_fixer), not Coder/Planner. -ELECTRON_TOOLS = [ - "mcp__electron__electron_connect", # Connect to Electron app via DevTools - "mcp__electron__electron_screenshot", # Take screenshot of Electron window - "mcp__electron__electron_click", # Click element in Electron app - "mcp__electron__electron_fill", # Fill input field in Electron app - "mcp__electron__electron_evaluate", # Execute JS in Electron renderer - "mcp__electron__electron_get_window_info", # Get window state/bounds - "mcp__electron__electron_get_console", # Get console logs from renderer -] - -# Built-in tools -BUILTIN_TOOLS = [ - "Read", - "Write", - "Edit", - "Glob", - "Grep", - "Bash", -] - - -def create_client( - project_dir: Path, - spec_dir: Path, - model: str, - agent_type: str = "coder", -) -> ClaudeSDKClient: - """ - Create a Claude Agent SDK client with multi-layered security. - - Args: - project_dir: Root directory for the project (working directory) - spec_dir: Directory containing the spec (for settings file) - model: Claude model to use - agent_type: Type of agent - 'planner', 'coder', 'qa_reviewer', or 'qa_fixer' - This determines which custom auto-claude tools are available. - - Returns: - Configured ClaudeSDKClient - - Security layers (defense in depth): - 1. Sandbox - OS-level bash command isolation prevents filesystem escape - 2. Permissions - File operations restricted to project_dir only - 3. Security hooks - Bash commands validated against an allowlist - (see security.py for ALLOWED_COMMANDS) - 4. Tool filtering - Each agent type only sees relevant tools (prevents misuse) - """ - oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") - if not oauth_token: - raise ValueError( - "CLAUDE_CODE_OAUTH_TOKEN environment variable not set.\n" - "Get your token by running: claude setup-token" - ) - - # Check if Linear integration is enabled - linear_enabled = is_linear_enabled() - linear_api_key = os.environ.get("LINEAR_API_KEY", "") - - # Check if custom auto-claude tools are available - auto_claude_tools_enabled = is_tools_available() - - # Build the list of allowed tools - # Start with agent-specific tools (includes base tools + auto-claude tools) - if auto_claude_tools_enabled: - allowed_tools_list = get_agent_allowed_tools(agent_type) - else: - allowed_tools_list = [*BUILTIN_TOOLS] - - # Check if Graphiti MCP is enabled - graphiti_mcp_enabled = is_graphiti_mcp_enabled() - - # Check if Electron MCP is enabled (for QA agents testing Electron apps) - electron_mcp_enabled = is_electron_mcp_enabled() - - # Add external MCP tools - allowed_tools_list.extend(PUPPETEER_TOOLS) - allowed_tools_list.extend(CONTEXT7_TOOLS) - if linear_enabled: - allowed_tools_list.extend(LINEAR_TOOLS) - if graphiti_mcp_enabled: - allowed_tools_list.extend(GRAPHITI_MCP_TOOLS) - # Add Electron MCP tools only for QA agents (qa_reviewer, qa_fixer) when enabled - # This prevents context bloat for coder/planner agents who don't need desktop automation - if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer"): - allowed_tools_list.extend(ELECTRON_TOOLS) - - # Create comprehensive security settings - # Note: Using relative paths ("./**") restricts access to project directory - # since cwd is set to project_dir - security_settings = { - "sandbox": {"enabled": True, "autoAllowBashIfSandboxed": True}, - "permissions": { - "defaultMode": "acceptEdits", # Auto-approve edits within allowed directories - "allow": [ - # Allow all file operations within the project directory - "Read(./**)", - "Write(./**)", - "Edit(./**)", - "Glob(./**)", - "Grep(./**)", - # Bash permission granted here, but actual commands are validated - # by the bash_security_hook (see security.py for allowed commands) - "Bash(*)", - # Allow Puppeteer MCP tools for browser automation - *PUPPETEER_TOOLS, - # Allow Context7 MCP tools for documentation lookup - *CONTEXT7_TOOLS, - # Allow Linear MCP tools for project management (if enabled) - *(LINEAR_TOOLS if linear_enabled else []), - # Allow Graphiti MCP tools for knowledge graph memory (if enabled) - *(GRAPHITI_MCP_TOOLS if graphiti_mcp_enabled else []), - # Allow Electron MCP tools for QA agents only (if enabled) - *(ELECTRON_TOOLS if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer") else []), - ], - }, - } - - # Write settings to a file in the project directory - settings_file = project_dir / ".claude_settings.json" - with open(settings_file, "w") as f: - json.dump(security_settings, f, indent=2) - - print(f"Security settings: {settings_file}") - print(" - Sandbox enabled (OS-level bash isolation)") - print(f" - Filesystem restricted to: {project_dir.resolve()}") - print(" - Bash commands restricted to allowlist") - - mcp_servers_list = ["puppeteer (browser automation)", "context7 (documentation)"] - if linear_enabled: - mcp_servers_list.append("linear (project management)") - if graphiti_mcp_enabled: - mcp_servers_list.append("graphiti-memory (knowledge graph)") - if electron_mcp_enabled: - mcp_servers_list.append(f"electron (desktop automation, port {get_electron_debug_port()})") - if auto_claude_tools_enabled: - mcp_servers_list.append(f"auto-claude ({agent_type} tools)") - print(f" - MCP servers: {', '.join(mcp_servers_list)}") - print() - - # Configure MCP servers - mcp_servers = { - "puppeteer": {"command": "npx", "args": ["puppeteer-mcp-server"]}, - "context7": {"command": "npx", "args": ["-y", "@upstash/context7-mcp"]}, - } - - # Add Linear MCP server if enabled - if linear_enabled: - mcp_servers["linear"] = { - "type": "http", - "url": "https://mcp.linear.app/mcp", - "headers": {"Authorization": f"Bearer {linear_api_key}"}, - } - - # Add Graphiti MCP server if enabled - # Requires running: docker run -d -p 8000:8000 falkordb/graphiti-knowledge-graph-mcp - if graphiti_mcp_enabled: - mcp_servers["graphiti-memory"] = { - "type": "http", - "url": get_graphiti_mcp_url(), - } - - # Add Electron MCP server if enabled - # Electron app must be started with --remote-debugging-port= for connection - # Uses puppeteer-mcp-server to connect to Electron via Chrome DevTools Protocol - if electron_mcp_enabled: - electron_port = get_electron_debug_port() - mcp_servers["electron"] = { - "command": "npx", - "args": [ - "-y", - "@anthropic/puppeteer-mcp-server", - "--debug-port", - str(electron_port), - ], - } - - # Add custom auto-claude MCP server if available - auto_claude_mcp_server = None - if auto_claude_tools_enabled: - auto_claude_mcp_server = create_auto_claude_mcp_server(spec_dir, project_dir) - if auto_claude_mcp_server: - mcp_servers["auto-claude"] = auto_claude_mcp_server - - return ClaudeSDKClient( - options=ClaudeAgentOptions( - model=model, - system_prompt=( - f"You are an expert full-stack developer building production-quality software. " - f"Your working directory is: {project_dir.resolve()}\n" - f"Your filesystem access is RESTRICTED to this directory only. " - f"Use relative paths (starting with ./) for all file operations. " - f"Never use absolute paths or try to access files outside your working directory.\n\n" - f"You follow existing code patterns, write clean maintainable code, and verify " - f"your work through thorough testing. You communicate progress through Git commits " - f"and build-progress.txt updates." - ), - allowed_tools=allowed_tools_list, - mcp_servers=mcp_servers, - hooks={ - "PreToolUse": [ - HookMatcher(matcher="Bash", hooks=[bash_security_hook]), - ], - }, - max_turns=1000, - cwd=str(project_dir.resolve()), - settings=str(settings_file.resolve()), - ) - ) diff --git a/auto-claude/context.py b/auto-claude/context/main.py similarity index 100% rename from auto-claude/context.py rename to auto-claude/context/main.py diff --git a/auto-claude/core/__init__.py b/auto-claude/core/__init__.py new file mode 100644 index 00000000..9c6bc9f1 --- /dev/null +++ b/auto-claude/core/__init__.py @@ -0,0 +1,36 @@ +""" +Core Framework Module +===================== + +Core components for the Auto Claude autonomous coding framework. +""" + +# Note: We use lazy imports here because the full agent module has many dependencies +# that may not be needed for basic operations like workspace management. + +__all__ = [ + "run_autonomous_agent", + "run_followup_planner", + "WorkspaceManager", + "WorktreeManager", + "ProgressTracker", +] + +def __getattr__(name): + """Lazy imports to avoid circular dependencies and heavy imports.""" + if name in ("run_autonomous_agent", "run_followup_planner"): + from .agent import run_autonomous_agent, run_followup_planner + return locals()[name] + elif name == "WorkspaceManager": + from .workspace import WorkspaceManager + return WorkspaceManager + elif name == "WorktreeManager": + from .worktree import WorktreeManager + return WorktreeManager + elif name == "ProgressTracker": + from .progress import ProgressTracker + return ProgressTracker + elif name in ("create_claude_client", "ClaudeClient"): + from . import client as _client + return getattr(_client, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/auto-claude/core/agent.py b/auto-claude/core/agent.py new file mode 100644 index 00000000..8b2cc8d5 --- /dev/null +++ b/auto-claude/core/agent.py @@ -0,0 +1,63 @@ +""" +Agent Session Logic +=================== + +Core agent interaction functions for running autonomous coding sessions. +Uses subtask-based implementation plans with minimal, focused prompts. + +Architecture: +- Orchestrator (Python) handles all bookkeeping: memory, commits, progress +- Agent focuses ONLY on implementing code +- Post-session processing updates memory automatically (100% reliable) + +Enhanced with status file updates for ccstatusline integration. +Enhanced with Graphiti memory for cross-session context retrieval. + +NOTE: This module is now a facade that imports from agents/ submodules. +All logic has been refactored into focused modules for better maintainability. +""" + +# Re-export everything from the agents module to maintain backwards compatibility +from agents import ( + # Constants + AUTO_CONTINUE_DELAY_SECONDS, + HUMAN_INTERVENTION_FILE, + # Memory functions + debug_memory_system_status, + find_phase_for_subtask, + find_subtask_in_plan, + get_commit_count, + get_graphiti_context, + # Utility functions + get_latest_commit, + load_implementation_plan, + post_session_processing, + # Session management + run_agent_session, + # Main API + run_autonomous_agent, + run_followup_planner, + save_session_memory, + save_session_to_graphiti, + sync_plan_to_source, +) + +# Ensure all exports are available at module level +__all__ = [ + "run_autonomous_agent", + "run_followup_planner", + "debug_memory_system_status", + "get_graphiti_context", + "save_session_memory", + "save_session_to_graphiti", + "run_agent_session", + "post_session_processing", + "get_latest_commit", + "get_commit_count", + "load_implementation_plan", + "find_subtask_in_plan", + "find_phase_for_subtask", + "sync_plan_to_source", + "AUTO_CONTINUE_DELAY_SECONDS", + "HUMAN_INTERVENTION_FILE", +] diff --git a/auto-claude/core/client.py b/auto-claude/core/client.py new file mode 100644 index 00000000..8a628f2a --- /dev/null +++ b/auto-claude/core/client.py @@ -0,0 +1,315 @@ +""" +Claude SDK Client Configuration +=============================== + +Functions for creating and configuring the Claude Agent SDK client. +""" + +import json +import os +from pathlib import Path + +from auto_claude_tools import ( + create_auto_claude_mcp_server, + is_tools_available, +) +from auto_claude_tools import ( + get_allowed_tools as get_agent_allowed_tools, +) +from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient +from claude_agent_sdk.types import HookMatcher +from linear_updater import is_linear_enabled +from security import bash_security_hook + + +def is_graphiti_mcp_enabled() -> bool: + """ + Check if Graphiti MCP server integration is enabled. + + Requires GRAPHITI_MCP_URL to be set (e.g., http://localhost:8000/mcp/) + This is separate from GRAPHITI_ENABLED which controls the Python library integration. + """ + return bool(os.environ.get("GRAPHITI_MCP_URL")) + + +def get_graphiti_mcp_url() -> str: + """Get the Graphiti MCP server URL.""" + return os.environ.get("GRAPHITI_MCP_URL", "http://localhost:8000/mcp/") + + +def is_electron_mcp_enabled() -> bool: + """ + Check if Electron MCP server integration is enabled. + + Requires ELECTRON_MCP_ENABLED to be set to 'true'. + When enabled, QA agents can use Puppeteer MCP tools to connect to Electron apps + via Chrome DevTools Protocol on the configured debug port. + """ + return os.environ.get("ELECTRON_MCP_ENABLED", "").lower() == "true" + + +def get_electron_debug_port() -> int: + """Get the Electron remote debugging port (default: 9222).""" + return int(os.environ.get("ELECTRON_DEBUG_PORT", "9222")) + + +# Puppeteer MCP tools for browser automation +PUPPETEER_TOOLS = [ + "mcp__puppeteer__puppeteer_connect_active_tab", + "mcp__puppeteer__puppeteer_navigate", + "mcp__puppeteer__puppeteer_screenshot", + "mcp__puppeteer__puppeteer_click", + "mcp__puppeteer__puppeteer_fill", + "mcp__puppeteer__puppeteer_select", + "mcp__puppeteer__puppeteer_hover", + "mcp__puppeteer__puppeteer_evaluate", +] + +# Linear MCP tools for project management (when LINEAR_API_KEY is set) +LINEAR_TOOLS = [ + "mcp__linear-server__list_teams", + "mcp__linear-server__get_team", + "mcp__linear-server__list_projects", + "mcp__linear-server__get_project", + "mcp__linear-server__create_project", + "mcp__linear-server__update_project", + "mcp__linear-server__list_issues", + "mcp__linear-server__get_issue", + "mcp__linear-server__create_issue", + "mcp__linear-server__update_issue", + "mcp__linear-server__list_comments", + "mcp__linear-server__create_comment", + "mcp__linear-server__list_issue_statuses", + "mcp__linear-server__list_issue_labels", + "mcp__linear-server__list_users", + "mcp__linear-server__get_user", +] + +# Context7 MCP tools for documentation lookup (always enabled) +CONTEXT7_TOOLS = [ + "mcp__context7__resolve-library-id", + "mcp__context7__get-library-docs", +] + +# Graphiti MCP tools for knowledge graph memory (when GRAPHITI_MCP_ENABLED is set) +# See: https://docs.falkordb.com/agentic-memory/graphiti-mcp-server.html +GRAPHITI_MCP_TOOLS = [ + "mcp__graphiti-memory__search_nodes", # Search entity summaries + "mcp__graphiti-memory__search_facts", # Search relationships between entities + "mcp__graphiti-memory__add_episode", # Add data to knowledge graph + "mcp__graphiti-memory__get_episodes", # Retrieve recent episodes + "mcp__graphiti-memory__get_entity_edge", # Get specific entity/relationship +] + +# Electron MCP tools for desktop app automation (when ELECTRON_MCP_ENABLED is set) +# Uses puppeteer-mcp-server to connect to Electron apps via Chrome DevTools Protocol. +# Electron app must be started with --remote-debugging-port=9222 (or ELECTRON_DEBUG_PORT). +# These tools are only available to QA agents (qa_reviewer, qa_fixer), not Coder/Planner. +ELECTRON_TOOLS = [ + "mcp__electron__electron_connect", # Connect to Electron app via DevTools + "mcp__electron__electron_screenshot", # Take screenshot of Electron window + "mcp__electron__electron_click", # Click element in Electron app + "mcp__electron__electron_fill", # Fill input field in Electron app + "mcp__electron__electron_evaluate", # Execute JS in Electron renderer + "mcp__electron__electron_get_window_info", # Get window state/bounds + "mcp__electron__electron_get_console", # Get console logs from renderer +] + +# Built-in tools +BUILTIN_TOOLS = [ + "Read", + "Write", + "Edit", + "Glob", + "Grep", + "Bash", +] + + +def create_client( + project_dir: Path, + spec_dir: Path, + model: str, + agent_type: str = "coder", +) -> ClaudeSDKClient: + """ + Create a Claude Agent SDK client with multi-layered security. + + Args: + project_dir: Root directory for the project (working directory) + spec_dir: Directory containing the spec (for settings file) + model: Claude model to use + agent_type: Type of agent - 'planner', 'coder', 'qa_reviewer', or 'qa_fixer' + This determines which custom auto-claude tools are available. + + Returns: + Configured ClaudeSDKClient + + Security layers (defense in depth): + 1. Sandbox - OS-level bash command isolation prevents filesystem escape + 2. Permissions - File operations restricted to project_dir only + 3. Security hooks - Bash commands validated against an allowlist + (see security.py for ALLOWED_COMMANDS) + 4. Tool filtering - Each agent type only sees relevant tools (prevents misuse) + """ + oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") + if not oauth_token: + raise ValueError( + "CLAUDE_CODE_OAUTH_TOKEN environment variable not set.\n" + "Get your token by running: claude setup-token" + ) + + # Check if Linear integration is enabled + linear_enabled = is_linear_enabled() + linear_api_key = os.environ.get("LINEAR_API_KEY", "") + + # Check if custom auto-claude tools are available + auto_claude_tools_enabled = is_tools_available() + + # Build the list of allowed tools + # Start with agent-specific tools (includes base tools + auto-claude tools) + if auto_claude_tools_enabled: + allowed_tools_list = get_agent_allowed_tools(agent_type) + else: + allowed_tools_list = [*BUILTIN_TOOLS] + + # Check if Graphiti MCP is enabled + graphiti_mcp_enabled = is_graphiti_mcp_enabled() + + # Check if Electron MCP is enabled (for QA agents testing Electron apps) + electron_mcp_enabled = is_electron_mcp_enabled() + + # Add external MCP tools + allowed_tools_list.extend(PUPPETEER_TOOLS) + allowed_tools_list.extend(CONTEXT7_TOOLS) + if linear_enabled: + allowed_tools_list.extend(LINEAR_TOOLS) + if graphiti_mcp_enabled: + allowed_tools_list.extend(GRAPHITI_MCP_TOOLS) + # Add Electron MCP tools only for QA agents (qa_reviewer, qa_fixer) when enabled + # This prevents context bloat for coder/planner agents who don't need desktop automation + if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer"): + allowed_tools_list.extend(ELECTRON_TOOLS) + + # Create comprehensive security settings + # Note: Using relative paths ("./**") restricts access to project directory + # since cwd is set to project_dir + security_settings = { + "sandbox": {"enabled": True, "autoAllowBashIfSandboxed": True}, + "permissions": { + "defaultMode": "acceptEdits", # Auto-approve edits within allowed directories + "allow": [ + # Allow all file operations within the project directory + "Read(./**)", + "Write(./**)", + "Edit(./**)", + "Glob(./**)", + "Grep(./**)", + # Bash permission granted here, but actual commands are validated + # by the bash_security_hook (see security.py for allowed commands) + "Bash(*)", + # Allow Puppeteer MCP tools for browser automation + *PUPPETEER_TOOLS, + # Allow Context7 MCP tools for documentation lookup + *CONTEXT7_TOOLS, + # Allow Linear MCP tools for project management (if enabled) + *(LINEAR_TOOLS if linear_enabled else []), + # Allow Graphiti MCP tools for knowledge graph memory (if enabled) + *(GRAPHITI_MCP_TOOLS if graphiti_mcp_enabled else []), + # Allow Electron MCP tools for QA agents only (if enabled) + *(ELECTRON_TOOLS if electron_mcp_enabled and agent_type in ("qa_reviewer", "qa_fixer") else []), + ], + }, + } + + # Write settings to a file in the project directory + settings_file = project_dir / ".claude_settings.json" + with open(settings_file, "w") as f: + json.dump(security_settings, f, indent=2) + + print(f"Security settings: {settings_file}") + print(" - Sandbox enabled (OS-level bash isolation)") + print(f" - Filesystem restricted to: {project_dir.resolve()}") + print(" - Bash commands restricted to allowlist") + + mcp_servers_list = ["puppeteer (browser automation)", "context7 (documentation)"] + if linear_enabled: + mcp_servers_list.append("linear (project management)") + if graphiti_mcp_enabled: + mcp_servers_list.append("graphiti-memory (knowledge graph)") + if electron_mcp_enabled: + mcp_servers_list.append(f"electron (desktop automation, port {get_electron_debug_port()})") + if auto_claude_tools_enabled: + mcp_servers_list.append(f"auto-claude ({agent_type} tools)") + print(f" - MCP servers: {', '.join(mcp_servers_list)}") + print() + + # Configure MCP servers + mcp_servers = { + "puppeteer": {"command": "npx", "args": ["puppeteer-mcp-server"]}, + "context7": {"command": "npx", "args": ["-y", "@upstash/context7-mcp"]}, + } + + # Add Linear MCP server if enabled + if linear_enabled: + mcp_servers["linear"] = { + "type": "http", + "url": "https://mcp.linear.app/mcp", + "headers": {"Authorization": f"Bearer {linear_api_key}"}, + } + + # Add Graphiti MCP server if enabled + # Requires running: docker run -d -p 8000:8000 falkordb/graphiti-knowledge-graph-mcp + if graphiti_mcp_enabled: + mcp_servers["graphiti-memory"] = { + "type": "http", + "url": get_graphiti_mcp_url(), + } + + # Add Electron MCP server if enabled + # Electron app must be started with --remote-debugging-port= for connection + # Uses puppeteer-mcp-server to connect to Electron via Chrome DevTools Protocol + if electron_mcp_enabled: + electron_port = get_electron_debug_port() + mcp_servers["electron"] = { + "command": "npx", + "args": [ + "-y", + "@anthropic/puppeteer-mcp-server", + "--debug-port", + str(electron_port), + ], + } + + # Add custom auto-claude MCP server if available + auto_claude_mcp_server = None + if auto_claude_tools_enabled: + auto_claude_mcp_server = create_auto_claude_mcp_server(spec_dir, project_dir) + if auto_claude_mcp_server: + mcp_servers["auto-claude"] = auto_claude_mcp_server + + return ClaudeSDKClient( + options=ClaudeAgentOptions( + model=model, + system_prompt=( + f"You are an expert full-stack developer building production-quality software. " + f"Your working directory is: {project_dir.resolve()}\n" + f"Your filesystem access is RESTRICTED to this directory only. " + f"Use relative paths (starting with ./) for all file operations. " + f"Never use absolute paths or try to access files outside your working directory.\n\n" + f"You follow existing code patterns, write clean maintainable code, and verify " + f"your work through thorough testing. You communicate progress through Git commits " + f"and build-progress.txt updates." + ), + allowed_tools=allowed_tools_list, + mcp_servers=mcp_servers, + hooks={ + "PreToolUse": [ + HookMatcher(matcher="Bash", hooks=[bash_security_hook]), + ], + }, + max_turns=1000, + cwd=str(project_dir.resolve()), + settings=str(settings_file.resolve()), + ) + ) diff --git a/auto-claude/core/debug.py b/auto-claude/core/debug.py new file mode 100644 index 00000000..9bef363d --- /dev/null +++ b/auto-claude/core/debug.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +""" +Debug Logging Utility +===================== + +Centralized debug logging for the Auto-Claude framework. +Controlled via environment variables: + - DEBUG=true Enable debug mode + - DEBUG_LEVEL=1|2|3 Log verbosity (1=basic, 2=detailed, 3=verbose) + - DEBUG_LOG_FILE=path Optional file output + +Usage: + from debug import debug, debug_detailed, debug_verbose, is_debug_enabled + + debug("run.py", "Starting task execution", task_id="001") + debug_detailed("agent", "Agent response received", response_length=1234) + debug_verbose("client", "Full request payload", payload=data) +""" + +import json +import os +import sys +import time +from datetime import datetime +from functools import wraps +from pathlib import Path +from typing import Any + + +# ANSI color codes for terminal output +class Colors: + RESET = "\033[0m" + BOLD = "\033[1m" + DIM = "\033[2m" + + # Debug colors + DEBUG = "\033[36m" # Cyan + DEBUG_DIM = "\033[96m" # Light cyan + TIMESTAMP = "\033[90m" # Gray + MODULE = "\033[33m" # Yellow + KEY = "\033[35m" # Magenta + VALUE = "\033[37m" # White + SUCCESS = "\033[32m" # Green + WARNING = "\033[33m" # Yellow + ERROR = "\033[31m" # Red + + +def _get_debug_enabled() -> bool: + """Check if debug mode is enabled via environment variable.""" + return os.environ.get("DEBUG", "").lower() in ("true", "1", "yes", "on") + + +def _get_debug_level() -> int: + """Get debug verbosity level (1-3).""" + try: + level = int(os.environ.get("DEBUG_LEVEL", "1")) + return max(1, min(3, level)) # Clamp to 1-3 + except ValueError: + return 1 + + +def _get_log_file() -> Path | None: + """Get optional log file path.""" + log_file = os.environ.get("DEBUG_LOG_FILE") + if log_file: + return Path(log_file) + return None + + +def is_debug_enabled() -> bool: + """Check if debug mode is enabled.""" + return _get_debug_enabled() + + +def get_debug_level() -> int: + """Get current debug level.""" + return _get_debug_level() + + +def _format_value(value: Any, max_length: int = 200) -> str: + """Format a value for debug output, truncating if necessary.""" + if value is None: + return "None" + + if isinstance(value, (dict, list)): + try: + formatted = json.dumps(value, indent=2, default=str) + if len(formatted) > max_length: + formatted = formatted[:max_length] + "..." + return formatted + except (TypeError, ValueError): + return str(value)[:max_length] + + str_value = str(value) + if len(str_value) > max_length: + return str_value[:max_length] + "..." + return str_value + + +def _write_log(message: str, to_file: bool = True) -> None: + """Write log message to stdout and optionally to file.""" + print(message, file=sys.stderr) + + if to_file: + log_file = _get_log_file() + if log_file: + try: + log_file.parent.mkdir(parents=True, exist_ok=True) + # Strip ANSI codes for file output + import re + + clean_message = re.sub(r"\033\[[0-9;]*m", "", message) + with open(log_file, "a") as f: + f.write(clean_message + "\n") + except Exception: + pass # Silently fail file logging + + +def debug(module: str, message: str, level: int = 1, **kwargs) -> None: + """ + Log a debug message. + + Args: + module: Source module name (e.g., "run.py", "ideation_runner") + message: Debug message + level: Required debug level (1=basic, 2=detailed, 3=verbose) + **kwargs: Additional key-value pairs to log + """ + if not _get_debug_enabled(): + return + + if _get_debug_level() < level: + return + + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + + # Build the log line + parts = [ + f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET}", + f"{Colors.DEBUG}[DEBUG]{Colors.RESET}", + f"{Colors.MODULE}[{module}]{Colors.RESET}", + f"{Colors.DEBUG_DIM}{message}{Colors.RESET}", + ] + + log_line = " ".join(parts) + + # Add kwargs on separate lines if present + if kwargs: + for key, value in kwargs.items(): + formatted_value = _format_value(value) + if "\n" in formatted_value: + # Multi-line value + log_line += f"\n {Colors.KEY}{key}{Colors.RESET}:" + for line in formatted_value.split("\n"): + log_line += f"\n {Colors.VALUE}{line}{Colors.RESET}" + else: + log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{formatted_value}{Colors.RESET}" + + _write_log(log_line) + + +def debug_detailed(module: str, message: str, **kwargs) -> None: + """Log a detailed debug message (level 2).""" + debug(module, message, level=2, **kwargs) + + +def debug_verbose(module: str, message: str, **kwargs) -> None: + """Log a verbose debug message (level 3).""" + debug(module, message, level=3, **kwargs) + + +def debug_success(module: str, message: str, **kwargs) -> None: + """Log a success debug message.""" + if not _get_debug_enabled(): + return + + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.SUCCESS}[OK]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}" + + if kwargs: + for key, value in kwargs.items(): + log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}" + + _write_log(log_line) + + +def debug_info(module: str, message: str, **kwargs) -> None: + """Log an info debug message.""" + if not _get_debug_enabled(): + return + + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}[INFO]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}" + + if kwargs: + for key, value in kwargs.items(): + log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}" + + _write_log(log_line) + + +def debug_error(module: str, message: str, **kwargs) -> None: + """Log an error debug message (always shown if debug enabled).""" + if not _get_debug_enabled(): + return + + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.ERROR}[ERROR]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.ERROR}{message}{Colors.RESET}" + + if kwargs: + for key, value in kwargs.items(): + log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}" + + _write_log(log_line) + + +def debug_warning(module: str, message: str, **kwargs) -> None: + """Log a warning debug message.""" + if not _get_debug_enabled(): + return + + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.WARNING}[WARN]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.WARNING}{message}{Colors.RESET}" + + if kwargs: + for key, value in kwargs.items(): + log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}" + + _write_log(log_line) + + +def debug_section(module: str, title: str) -> None: + """Log a section header for organizing debug output.""" + if not _get_debug_enabled(): + return + + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + separator = "─" * 60 + log_line = f"\n{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}{Colors.BOLD}┌{separator}┐{Colors.RESET}" + log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}│ {module}: {title}{' ' * (58 - len(module) - len(title) - 2)}│{Colors.RESET}" + log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}└{separator}┘{Colors.RESET}" + + _write_log(log_line) + + +def debug_timer(module: str): + """ + Decorator to time function execution. + + Usage: + @debug_timer("run.py") + def my_function(): + ... + """ + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + if not _get_debug_enabled(): + return func(*args, **kwargs) + + start = time.time() + debug_detailed(module, f"Starting {func.__name__}()") + + try: + result = func(*args, **kwargs) + elapsed = time.time() - start + debug_success( + module, + f"Completed {func.__name__}()", + elapsed_ms=f"{elapsed * 1000:.1f}ms", + ) + return result + except Exception as e: + elapsed = time.time() - start + debug_error( + module, + f"Failed {func.__name__}()", + error=str(e), + elapsed_ms=f"{elapsed * 1000:.1f}ms", + ) + raise + + return wrapper + + return decorator + + +def debug_async_timer(module: str): + """ + Decorator to time async function execution. + + Usage: + @debug_async_timer("ideation_runner") + async def my_async_function(): + ... + """ + + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + if not _get_debug_enabled(): + return await func(*args, **kwargs) + + start = time.time() + debug_detailed(module, f"Starting {func.__name__}()") + + try: + result = await func(*args, **kwargs) + elapsed = time.time() - start + debug_success( + module, + f"Completed {func.__name__}()", + elapsed_ms=f"{elapsed * 1000:.1f}ms", + ) + return result + except Exception as e: + elapsed = time.time() - start + debug_error( + module, + f"Failed {func.__name__}()", + error=str(e), + elapsed_ms=f"{elapsed * 1000:.1f}ms", + ) + raise + + return wrapper + + return decorator + + +def debug_env_status() -> None: + """Print debug environment status on startup.""" + if not _get_debug_enabled(): + return + + debug_section("debug", "Debug Mode Enabled") + debug( + "debug", + "Environment configuration", + DEBUG=os.environ.get("DEBUG", "not set"), + DEBUG_LEVEL=_get_debug_level(), + DEBUG_LOG_FILE=os.environ.get("DEBUG_LOG_FILE", "not set"), + ) + + +# Print status on import if debug is enabled +if _get_debug_enabled(): + debug_env_status() diff --git a/auto-claude/core/progress.py b/auto-claude/core/progress.py new file mode 100644 index 00000000..1e416045 --- /dev/null +++ b/auto-claude/core/progress.py @@ -0,0 +1,467 @@ +""" +Progress Tracking Utilities +=========================== + +Functions for tracking and displaying progress of the autonomous coding agent. +Uses subtask-based implementation plans (implementation_plan.json). + +Enhanced with colored output, icons, and better visual formatting. +""" + +import json +from pathlib import Path + +from ui import ( + Icons, + bold, + box, + highlight, + icon, + muted, + print_phase_status, + print_status, + progress_bar, + success, + warning, +) + + +def count_subtasks(spec_dir: Path) -> tuple[int, int]: + """ + Count completed and total subtasks in implementation_plan.json. + + Args: + spec_dir: Directory containing implementation_plan.json + + Returns: + (completed_count, total_count) + """ + plan_file = spec_dir / "implementation_plan.json" + + if not plan_file.exists(): + return 0, 0 + + try: + with open(plan_file) as f: + plan = json.load(f) + + total = 0 + completed = 0 + + for phase in plan.get("phases", []): + for subtask in phase.get("subtasks", []): + total += 1 + if subtask.get("status") == "completed": + completed += 1 + + return completed, total + except (OSError, json.JSONDecodeError): + return 0, 0 + + +def count_subtasks_detailed(spec_dir: Path) -> dict: + """ + Count subtasks by status. + + Returns: + Dict with completed, in_progress, pending, failed counts + """ + plan_file = spec_dir / "implementation_plan.json" + + result = { + "completed": 0, + "in_progress": 0, + "pending": 0, + "failed": 0, + "total": 0, + } + + if not plan_file.exists(): + return result + + try: + with open(plan_file) as f: + plan = json.load(f) + + for phase in plan.get("phases", []): + for subtask in phase.get("subtasks", []): + result["total"] += 1 + status = subtask.get("status", "pending") + if status in result: + result[status] += 1 + else: + result["pending"] += 1 + + return result + except (OSError, json.JSONDecodeError): + return result + + +def is_build_complete(spec_dir: Path) -> bool: + """ + Check if all subtasks are completed. + + Args: + spec_dir: Directory containing implementation_plan.json + + Returns: + True if all subtasks complete, False otherwise + """ + completed, total = count_subtasks(spec_dir) + return total > 0 and completed == total + + +def get_progress_percentage(spec_dir: Path) -> float: + """ + Get the progress as a percentage. + + Args: + spec_dir: Directory containing implementation_plan.json + + Returns: + Percentage of subtasks completed (0-100) + """ + completed, total = count_subtasks(spec_dir) + if total == 0: + return 0.0 + return (completed / total) * 100 + + +def print_session_header( + session_num: int, + is_planner: bool, + subtask_id: str = None, + subtask_desc: str = None, + phase_name: str = None, + attempt: int = 1, +) -> None: + """Print a formatted header for the session.""" + session_type = "PLANNER AGENT" if is_planner else "CODING AGENT" + session_icon = Icons.GEAR if is_planner else Icons.LIGHTNING + + content = [ + bold(f"{icon(session_icon)} SESSION {session_num}: {session_type}"), + ] + + if subtask_id: + content.append("") + subtask_line = f"{icon(Icons.SUBTASK)} Subtask: {highlight(subtask_id)}" + if subtask_desc: + # Truncate long descriptions + desc = subtask_desc[:50] + "..." if len(subtask_desc) > 50 else subtask_desc + subtask_line += f" - {desc}" + content.append(subtask_line) + + if phase_name: + content.append(f"{icon(Icons.PHASE)} Phase: {phase_name}") + + if attempt > 1: + content.append(warning(f"{icon(Icons.WARNING)} Attempt: {attempt}")) + + print() + print(box(content, width=70, style="heavy")) + print() + + +def print_progress_summary(spec_dir: Path, show_next: bool = True) -> None: + """Print a summary of current progress with enhanced formatting.""" + completed, total = count_subtasks(spec_dir) + + if total > 0: + print() + # Progress bar + print(f"Progress: {progress_bar(completed, total, width=40)}") + + # Status message + if completed == total: + print_status("BUILD COMPLETE - All subtasks completed!", "success") + else: + remaining = total - completed + print_status(f"{remaining} subtasks remaining", "info") + + # Phase summary + try: + with open(spec_dir / "implementation_plan.json") as f: + plan = json.load(f) + + print("\nPhases:") + for phase in plan.get("phases", []): + phase_subtasks = phase.get("subtasks", []) + phase_completed = sum( + 1 for s in phase_subtasks if s.get("status") == "completed" + ) + phase_total = len(phase_subtasks) + phase_name = phase.get("name", phase.get("id", "Unknown")) + + if phase_completed == phase_total: + status = "complete" + elif phase_completed > 0 or any( + s.get("status") == "in_progress" for s in phase_subtasks + ): + status = "in_progress" + else: + # Check if blocked by dependencies + deps = phase.get("depends_on", []) + all_deps_complete = True + for dep_id in deps: + for p in plan.get("phases", []): + if p.get("id") == dep_id or p.get("phase") == dep_id: + p_subtasks = p.get("subtasks", []) + if not all( + s.get("status") == "completed" for s in p_subtasks + ): + all_deps_complete = False + break + status = "pending" if all_deps_complete else "blocked" + + print_phase_status(phase_name, phase_completed, phase_total, status) + + # Show next subtask if requested + if show_next and completed < total: + next_subtask = get_next_subtask(spec_dir) + if next_subtask: + print() + next_id = next_subtask.get("id", "unknown") + next_desc = next_subtask.get("description", "") + if len(next_desc) > 60: + next_desc = next_desc[:57] + "..." + print( + f" {icon(Icons.ARROW_RIGHT)} Next: {highlight(next_id)} - {next_desc}" + ) + + except (OSError, json.JSONDecodeError): + pass + else: + print() + print_status("No implementation subtasks yet - planner needs to run", "pending") + + +def print_build_complete_banner(spec_dir: Path) -> None: + """Print a completion banner.""" + content = [ + success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"), + "", + "All subtasks have been implemented successfully.", + "", + muted("Next steps:"), + f" 1. Review the {highlight('auto-claude/*')} branch", + " 2. Run manual tests", + " 3. Create a PR and merge to main", + ] + + print() + print(box(content, width=70, style="heavy")) + print() + + +def print_paused_banner( + spec_dir: Path, + spec_name: str, + has_worktree: bool = False, +) -> None: + """Print a paused banner with resume instructions.""" + completed, total = count_subtasks(spec_dir) + + content = [ + warning(f"{icon(Icons.PAUSE)} BUILD PAUSED"), + "", + f"Progress saved: {completed}/{total} subtasks complete", + ] + + if has_worktree: + content.append("") + content.append(muted("Your build is in a separate workspace and is safe.")) + + print() + print(box(content, width=70, style="heavy")) + + +def get_plan_summary(spec_dir: Path) -> dict: + """ + Get a detailed summary of implementation plan status. + + Args: + spec_dir: Directory containing implementation_plan.json + + Returns: + Dictionary with plan statistics + """ + plan_file = spec_dir / "implementation_plan.json" + + if not plan_file.exists(): + return { + "workflow_type": None, + "total_phases": 0, + "total_subtasks": 0, + "completed_subtasks": 0, + "pending_subtasks": 0, + "in_progress_subtasks": 0, + "failed_subtasks": 0, + "phases": [], + } + + try: + with open(plan_file) as f: + plan = json.load(f) + + summary = { + "workflow_type": plan.get("workflow_type"), + "total_phases": len(plan.get("phases", [])), + "total_subtasks": 0, + "completed_subtasks": 0, + "pending_subtasks": 0, + "in_progress_subtasks": 0, + "failed_subtasks": 0, + "phases": [], + } + + for phase in plan.get("phases", []): + phase_info = { + "id": phase.get("id"), + "phase": phase.get("phase"), + "name": phase.get("name"), + "depends_on": phase.get("depends_on", []), + "subtasks": [], + "completed": 0, + "total": 0, + } + + for subtask in phase.get("subtasks", []): + status = subtask.get("status", "pending") + summary["total_subtasks"] += 1 + phase_info["total"] += 1 + + if status == "completed": + summary["completed_subtasks"] += 1 + phase_info["completed"] += 1 + elif status == "in_progress": + summary["in_progress_subtasks"] += 1 + elif status == "failed": + summary["failed_subtasks"] += 1 + else: + summary["pending_subtasks"] += 1 + + phase_info["subtasks"].append( + { + "id": subtask.get("id"), + "description": subtask.get("description"), + "status": status, + "service": subtask.get("service"), + } + ) + + summary["phases"].append(phase_info) + + return summary + + except (OSError, json.JSONDecodeError): + return { + "workflow_type": None, + "total_phases": 0, + "total_subtasks": 0, + "completed_subtasks": 0, + "pending_subtasks": 0, + "in_progress_subtasks": 0, + "failed_subtasks": 0, + "phases": [], + } + + +def get_current_phase(spec_dir: Path) -> dict | None: + """Get the current phase being worked on.""" + plan_file = spec_dir / "implementation_plan.json" + + if not plan_file.exists(): + return None + + try: + with open(plan_file) as f: + plan = json.load(f) + + for phase in plan.get("phases", []): + subtasks = phase.get("subtasks", []) + # Phase is current if it has incomplete subtasks and dependencies are met + has_incomplete = any(s.get("status") != "completed" for s in subtasks) + if has_incomplete: + return { + "id": phase.get("id"), + "phase": phase.get("phase"), + "name": phase.get("name"), + "completed": sum( + 1 for s in subtasks if s.get("status") == "completed" + ), + "total": len(subtasks), + } + + return None + + except (OSError, json.JSONDecodeError): + return None + + +def get_next_subtask(spec_dir: Path) -> dict | None: + """ + Find the next subtask to work on, respecting phase dependencies. + + Args: + spec_dir: Directory containing implementation_plan.json + + Returns: + The next subtask dict to work on, or None if all complete + """ + plan_file = spec_dir / "implementation_plan.json" + + if not plan_file.exists(): + return None + + try: + with open(plan_file) as f: + plan = json.load(f) + + phases = plan.get("phases", []) + + # Build a map of phase completion + phase_complete = {} + for phase in phases: + phase_id = phase.get("id") or phase.get("phase") + subtasks = phase.get("subtasks", []) + phase_complete[phase_id] = all( + s.get("status") == "completed" for s in subtasks + ) + + # Find next available subtask + for phase in phases: + phase_id = phase.get("id") or phase.get("phase") + depends_on = phase.get("depends_on", []) + + # Check if dependencies are satisfied + deps_satisfied = all(phase_complete.get(dep, False) for dep in depends_on) + if not deps_satisfied: + continue + + # Find first pending subtask in this phase + for subtask in phase.get("subtasks", []): + if subtask.get("status") == "pending": + return { + "phase_id": phase_id, + "phase_name": phase.get("name"), + "phase_num": phase.get("phase"), + **subtask, + } + + return None + + except (OSError, json.JSONDecodeError): + return None + + +def format_duration(seconds: float) -> str: + """Format a duration in human-readable form.""" + if seconds < 60: + return f"{seconds:.0f}s" + elif seconds < 3600: + minutes = seconds / 60 + return f"{minutes:.1f}m" + else: + hours = seconds / 3600 + return f"{hours:.1f}h" diff --git a/auto-claude/workspace_original.py b/auto-claude/core/workspace.py similarity index 78% rename from auto-claude/workspace_original.py rename to auto-claude/core/workspace.py index 3a9baf78..a3672dcc 100644 --- a/auto-claude/workspace_original.py +++ b/auto-claude/core/workspace.py @@ -6,19 +6,15 @@ Workspace Management - Per-Spec Architecture Handles workspace isolation through Git worktrees, where each spec gets its own isolated worktree in .worktrees/{spec-name}/. -Key changes from old design: -- Each spec has its own worktree (not shared) -- Worktree path: .worktrees/{spec-name}/ -- Branch name: auto-claude/{spec-name} -- Fixed: get_existing_build_worktree() now properly checks spec_name -- Fixed: finalize_workspace() skips prompts in auto_continue mode +This module has been refactored for better maintainability: +- Models and enums: workspace/models.py +- Git utilities: workspace/git_utils.py +- Setup functions: workspace/setup.py +- Display functions: workspace/display.py +- Finalization: workspace/finalization.py +- Complex merge operations: remain here (workspace.py) -Terminology mapping (technical -> user-friendly): -- worktree -> "separate workspace" -- branch -> "version of your project" -- uncommitted changes -> "unsaved work" -- merge -> "add to your project" -- working directory -> "your project" +Public API is exported via workspace/__init__.py for backward compatibility. """ import asyncio @@ -27,7 +23,6 @@ import shutil import subprocess import sys from dataclasses import dataclass -from enum import Enum from pathlib import Path from typing import Optional @@ -69,645 +64,81 @@ from merge import ( FileTimelineTracker, ) -# Track if we've already tried to install the git hook this session -_git_hook_check_done = False +# Import from refactored modules in core/workspace/ +from core.workspace.models import ( + WorkspaceMode, + WorkspaceChoice, + ParallelMergeTask, + ParallelMergeResult, + MergeLock, + MergeLockError, +) + +from core.workspace.git_utils import ( + has_uncommitted_changes, + get_current_branch, + get_existing_build_worktree, + get_file_content_from_ref as _get_file_content_from_ref, + get_changed_files_from_branch as _get_changed_files_from_branch, + is_process_running as _is_process_running, + is_binary_file as _is_binary_file, + is_lock_file as _is_lock_file, + validate_merged_syntax as _validate_merged_syntax, + create_conflict_file_with_git as _create_conflict_file_with_git, + MAX_FILE_LINES_FOR_AI, + MAX_PARALLEL_AI_MERGES, + MAX_SYNTAX_FIX_RETRIES, + BINARY_EXTENSIONS, + LOCK_FILES, + MERGE_LOCK_TIMEOUT, +) + +from core.workspace.setup import ( + choose_workspace, + copy_spec_to_worktree, + setup_workspace, + ensure_timeline_hook_installed as _ensure_timeline_hook_installed, + initialize_timeline_tracking as _initialize_timeline_tracking, +) + +from core.workspace.display import ( + show_build_summary, + show_changed_files, + print_merge_success as _print_merge_success, + print_conflict_info as _print_conflict_info, +) + +from core.workspace.finalization import ( + finalize_workspace, + handle_workspace_choice, + review_existing_build, + discard_existing_build, + check_existing_build, + list_all_worktrees, + cleanup_all_worktrees, +) MODULE = "workspace" - -class WorkspaceMode(Enum): - """How auto-claude should work.""" - - ISOLATED = "isolated" # Work in a separate worktree (safe) - DIRECT = "direct" # Work directly in user's project - - -class WorkspaceChoice(Enum): - """User's choice after build completes.""" - - MERGE = "merge" # Add changes to project - REVIEW = "review" # Show what changed - TEST = "test" # Test the feature in the staging worktree - LATER = "later" # Decide later - - -def has_uncommitted_changes(project_dir: Path) -> bool: - """Check if user has unsaved work.""" - result = subprocess.run( - ["git", "status", "--porcelain"], - cwd=project_dir, - capture_output=True, - text=True, - ) - return bool(result.stdout.strip()) - - -def get_current_branch(project_dir: Path) -> str: - """Get the current branch name.""" - result = subprocess.run( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], - cwd=project_dir, - capture_output=True, - text=True, - ) - return result.stdout.strip() - - -def get_existing_build_worktree(project_dir: Path, spec_name: str) -> Path | None: - """ - Check if there's an existing worktree for this specific spec. - - Args: - project_dir: The main project directory - spec_name: The spec folder name (e.g., "001-feature-name") - - Returns: - Path to the worktree if it exists for this spec, None otherwise - """ - # Per-spec worktree path: .worktrees/{spec-name}/ - worktree_path = project_dir / ".worktrees" / spec_name - if worktree_path.exists(): - return worktree_path - return None - - -def choose_workspace( - project_dir: Path, - spec_name: str, - force_isolated: bool = False, - force_direct: bool = False, - auto_continue: bool = False, -) -> WorkspaceMode: - """ - Let user choose where auto-claude should work. - - Uses simple, non-technical language. Safe defaults. - - Args: - project_dir: The project directory - spec_name: Name of the spec being built - force_isolated: Skip prompts and use isolated mode - force_direct: Skip prompts and use direct mode - auto_continue: Non-interactive mode (for UI integration) - skip all prompts - - Returns: - WorkspaceMode indicating where to work - """ - # Handle forced modes - if force_isolated: - return WorkspaceMode.ISOLATED - if force_direct: - return WorkspaceMode.DIRECT - - # Non-interactive mode: default to isolated for safety - if auto_continue: - print("Auto-continue: Using isolated workspace for safety.") - return WorkspaceMode.ISOLATED - - # Check for unsaved work - has_unsaved = has_uncommitted_changes(project_dir) - - if has_unsaved: - # Unsaved work detected - use isolated mode for safety - content = [ - success(f"{icon(Icons.SHIELD)} YOUR WORK IS PROTECTED"), - "", - "You have unsaved work in your project.", - "", - "To keep your work safe, the AI will build in a", - "separate workspace. Your current files won't be", - "touched until you're ready.", - ] - print() - print(box(content, width=60, style="heavy")) - print() - - try: - input("Press Enter to continue...") - except KeyboardInterrupt: - print() - print_status("Cancelled.", "info") - sys.exit(0) - - return WorkspaceMode.ISOLATED - - # Clean working directory - give choice with enhanced menu - options = [ - MenuOption( - key="isolated", - label="Separate workspace (Recommended)", - icon=Icons.SHIELD, - description="Your current files stay untouched. Easy to review and undo.", - ), - MenuOption( - key="direct", - label="Right here in your project", - icon=Icons.LIGHTNING, - description="Changes happen directly. Best if you're not working on anything else.", - ), - ] - - choice = select_menu( - title="Where should the AI build your feature?", - options=options, - allow_quit=True, - ) - - if choice is None: - print() - print_status("Cancelled.", "info") - sys.exit(0) - - if choice == "direct": - print() - print_status("Working directly in your project.", "info") - return WorkspaceMode.DIRECT - else: - print() - print_status("Using a separate workspace for safety.", "success") - return WorkspaceMode.ISOLATED - - -def copy_spec_to_worktree( - source_spec_dir: Path, - worktree_path: Path, - spec_name: str, -) -> Path: - """ - Copy spec files into the worktree so the AI can access them. - - The AI's filesystem is restricted to the worktree, so spec files - must be copied inside for access. - - Args: - source_spec_dir: Original spec directory (may be outside worktree) - worktree_path: Path to the worktree - spec_name: Name of the spec folder - - Returns: - Path to the spec directory inside the worktree - """ - # Determine target location inside worktree - # Use .auto-claude/specs/{spec_name}/ as the standard location - # Note: auto-claude/ is source code, .auto-claude/ is the installed instance - target_spec_dir = worktree_path / ".auto-claude" / "specs" / spec_name - - # Create parent directories if needed - target_spec_dir.parent.mkdir(parents=True, exist_ok=True) - - # Copy spec files (overwrite if exists to get latest) - if target_spec_dir.exists(): - shutil.rmtree(target_spec_dir) - - shutil.copytree(source_spec_dir, target_spec_dir) - - return target_spec_dir - - -def setup_workspace( - project_dir: Path, - spec_name: str, - mode: WorkspaceMode, - source_spec_dir: Path | None = None, -) -> tuple[Path, WorktreeManager | None, Path | None]: - """ - Set up the workspace based on user's choice. - - Uses per-spec worktrees - each spec gets its own isolated worktree. - - Args: - project_dir: The project directory - spec_name: Name of the spec being built (e.g., "001-feature-name") - mode: The workspace mode to use - source_spec_dir: Optional source spec directory to copy to worktree - - Returns: - Tuple of (working_directory, worktree_manager or None, localized_spec_dir or None) - - When using isolated mode with source_spec_dir: - - working_directory: Path to the worktree - - worktree_manager: Manager for the worktree - - localized_spec_dir: Path to spec files INSIDE the worktree (accessible to AI) - """ - if mode == WorkspaceMode.DIRECT: - # Work directly in project - spec_dir stays as-is - return project_dir, None, source_spec_dir - - # Create isolated workspace using per-spec worktree - print() - print_status("Setting up separate workspace...", "progress") - - # Ensure timeline tracking hook is installed (once per session) - _ensure_timeline_hook_installed(project_dir) - - manager = WorktreeManager(project_dir) - manager.setup() - - # Get or create worktree for THIS SPECIFIC SPEC - worktree_info = manager.get_or_create_worktree(spec_name) - - # Copy spec files to worktree if provided - localized_spec_dir = None - if source_spec_dir and source_spec_dir.exists(): - localized_spec_dir = copy_spec_to_worktree( - source_spec_dir, worktree_info.path, spec_name - ) - print_status("Spec files copied to workspace", "success") - - print_status(f"Workspace ready: {worktree_info.path.name}", "success") - print() - - # Initialize FileTimelineTracker for this task - _initialize_timeline_tracking( - project_dir=project_dir, - spec_name=spec_name, - worktree_path=worktree_info.path, - source_spec_dir=localized_spec_dir or source_spec_dir, - ) - - return worktree_info.path, manager, localized_spec_dir - - -def _ensure_timeline_hook_installed(project_dir: Path) -> None: - """ - Ensure the FileTimelineTracker git post-commit hook is installed. - - This enables tracking human commits to main branch for drift detection. - Called once per session during first workspace setup. - """ - global _git_hook_check_done - if _git_hook_check_done: - return - - _git_hook_check_done = True - - try: - git_dir = project_dir / ".git" - if not git_dir.exists(): - return # Not a git repo - - # Handle worktrees (where .git is a file, not directory) - if git_dir.is_file(): - content = git_dir.read_text().strip() - if content.startswith("gitdir:"): - git_dir = Path(content.split(":", 1)[1].strip()) - else: - return - - hook_path = git_dir / "hooks" / "post-commit" - - # Check if hook already installed - if hook_path.exists(): - if "FileTimelineTracker" in hook_path.read_text(): - debug(MODULE, "FileTimelineTracker hook already installed") - return - - # Auto-install the hook (silent, non-intrusive) - from merge.install_hook import install_hook - install_hook(project_dir) - debug(MODULE, "Auto-installed FileTimelineTracker git hook") - - except Exception as e: - # Non-fatal - hook installation is optional - debug_warning(MODULE, f"Could not auto-install timeline hook: {e}") - - -def _initialize_timeline_tracking( - project_dir: Path, - spec_name: str, - worktree_path: Path, - source_spec_dir: Path | None = None, -) -> None: - """ - Initialize FileTimelineTracker for a new task. - - This registers the task's branch point and the files it intends to modify, - enabling intent-aware merge conflict resolution later. - """ - try: - tracker = FileTimelineTracker(project_dir) - - # Get task intent from implementation plan - task_intent = "" - task_title = spec_name - files_to_modify = [] - - if source_spec_dir: - plan_path = source_spec_dir / "implementation_plan.json" - if plan_path.exists(): - import json - with open(plan_path) as f: - plan = json.load(f) - task_title = plan.get("title", spec_name) - task_intent = plan.get("description", "") - - # Extract files from phases/subtasks - for phase in plan.get("phases", []): - for subtask in phase.get("subtasks", []): - files_to_modify.extend(subtask.get("files", [])) - - # Get the current branch point commit - import subprocess - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=project_dir, - capture_output=True, - text=True, - ) - branch_point = result.stdout.strip() if result.returncode == 0 else None - - if files_to_modify and branch_point: - # Register the task with known files - tracker.on_task_start( - task_id=spec_name, - files_to_modify=list(set(files_to_modify)), # Dedupe - branch_point_commit=branch_point, - task_intent=task_intent, - task_title=task_title, - ) - debug(MODULE, f"Timeline tracking initialized for {spec_name}", - files_tracked=len(files_to_modify), - branch_point=branch_point[:8] if branch_point else None) - else: - # Initialize retroactively from worktree if no plan - tracker.initialize_from_worktree( - task_id=spec_name, - worktree_path=worktree_path, - task_intent=task_intent, - task_title=task_title, - ) - - except Exception as e: - # Non-fatal - timeline tracking is supplementary - debug_warning(MODULE, f"Could not initialize timeline tracking: {e}") - print(muted(f" Note: Timeline tracking could not be initialized: {e}")) - - -def show_build_summary(manager: WorktreeManager, spec_name: str) -> None: - """Show a summary of what was built.""" - summary = manager.get_change_summary(spec_name) - files = manager.get_changed_files(spec_name) - - total = summary["new_files"] + summary["modified_files"] + summary["deleted_files"] - - if total == 0: - print_status("No changes were made.", "info") - return - - print() - print(bold("What was built:")) - if summary["new_files"] > 0: - print( - success( - f" + {summary['new_files']} new file{'s' if summary['new_files'] != 1 else ''}" - ) - ) - if summary["modified_files"] > 0: - print( - info( - f" ~ {summary['modified_files']} modified file{'s' if summary['modified_files'] != 1 else ''}" - ) - ) - if summary["deleted_files"] > 0: - print( - error( - f" - {summary['deleted_files']} deleted file{'s' if summary['deleted_files'] != 1 else ''}" - ) - ) - - -def show_changed_files(manager: WorktreeManager, spec_name: str) -> None: - """Show detailed list of changed files.""" - files = manager.get_changed_files(spec_name) - - if not files: - print_status("No changes.", "info") - return - - print() - print(bold("Changed files:")) - for status, filepath in files: - if status == "A": - print(success(f" + {filepath}")) - elif status == "M": - print(info(f" ~ {filepath}")) - elif status == "D": - print(error(f" - {filepath}")) - else: - print(f" {status} {filepath}") - - -def finalize_workspace( - project_dir: Path, - spec_name: str, - manager: WorktreeManager | None, - auto_continue: bool = False, -) -> WorkspaceChoice: - """ - Handle post-build workflow - let user decide what to do with changes. - - Safe design: - - No "discard" option (requires separate --discard command) - - Default is "test" - encourages testing before merging - - Everything is preserved until user explicitly merges or discards - - Args: - project_dir: The project directory - spec_name: Name of the spec that was built - manager: The worktree manager (None if direct mode was used) - auto_continue: If True, skip interactive prompts (UI mode) - - Returns: - WorkspaceChoice indicating what user wants to do - """ - if manager is None: - # Direct mode - nothing to finalize - content = [ - success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"), - "", - "Changes were made directly to your project.", - muted("Use 'git status' to see what changed."), - ] - print() - print(box(content, width=60, style="heavy")) - return WorkspaceChoice.MERGE # Already merged - - # In auto_continue mode (UI), skip interactive prompts - # The worktree stays for the UI to manage - if auto_continue: - worktree_info = manager.get_worktree_info(spec_name) - if worktree_info: - print() - print(success(f"Build complete in worktree: {worktree_info.path}")) - print(muted("Worktree preserved for UI review.")) - return WorkspaceChoice.LATER - - # Isolated mode - show options with testing as the recommended path - content = [ - success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"), - "", - "The AI built your feature in a separate workspace.", - ] - print() - print(box(content, width=60, style="heavy")) - - show_build_summary(manager, spec_name) - - # Get the worktree path for test instructions - worktree_info = manager.get_worktree_info(spec_name) - staging_path = worktree_info.path if worktree_info else None - - # Enhanced menu for post-build options - options = [ - MenuOption( - key="test", - label="Test the feature (Recommended)", - icon=Icons.PLAY, - description="Run the app and try it out before adding to your project", - ), - MenuOption( - key="merge", - label="Add to my project now", - icon=Icons.SUCCESS, - description="Merge the changes into your files immediately", - ), - MenuOption( - key="review", - label="Review what changed", - icon=Icons.FILE, - description="See exactly what files were modified", - ), - MenuOption( - key="later", - label="Decide later", - icon=Icons.PAUSE, - description="Your build is saved - you can come back anytime", - ), - ] - - print() - choice = select_menu( - title="What would you like to do?", - options=options, - allow_quit=False, - ) - - if choice == "test": - return WorkspaceChoice.TEST - elif choice == "merge": - return WorkspaceChoice.MERGE - elif choice == "review": - return WorkspaceChoice.REVIEW - else: - return WorkspaceChoice.LATER - - -def handle_workspace_choice( - choice: WorkspaceChoice, - project_dir: Path, - spec_name: str, - manager: WorktreeManager, -) -> None: - """ - Execute the user's choice. - - Args: - choice: What the user wants to do - project_dir: The project directory - spec_name: Name of the spec - manager: The worktree manager - """ - worktree_info = manager.get_worktree_info(spec_name) - staging_path = worktree_info.path if worktree_info else None - - if choice == WorkspaceChoice.TEST: - # Show testing instructions - content = [ - bold(f"{icon(Icons.PLAY)} TEST YOUR FEATURE"), - "", - "Your feature is ready to test in a separate workspace.", - ] - print() - print(box(content, width=60, style="heavy")) - - print() - print("To test it, open a NEW terminal and run:") - print() - if staging_path: - print(highlight(f" cd {staging_path}")) - else: - print(highlight(f" cd {project_dir}/.worktrees/{spec_name}")) - - # Show likely test/run commands - if staging_path: - commands = manager.get_test_commands(spec_name) - print() - print("Then run your project:") - for cmd in commands[:2]: # Show top 2 commands - print(f" {cmd}") - - print() - print(muted("-" * 60)) - print() - print("When you're done testing:") - print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge")) - print() - print("To discard (if you don't like it):") - print(muted(f" python auto-claude/run.py --spec {spec_name} --discard")) - print() - - elif choice == WorkspaceChoice.MERGE: - print() - print_status("Adding changes to your project...", "progress") - success_result = manager.merge_worktree(spec_name, delete_after=True) - - if success_result: - print() - print_status("Your feature has been added to your project.", "success") - else: - print() - print_status("There was a conflict merging the changes.", "error") - print(muted("Your build is still saved in the separate workspace.")) - print(muted("You may need to merge manually or ask for help.")) - - elif choice == WorkspaceChoice.REVIEW: - show_changed_files(manager, spec_name) - print() - print(muted("-" * 60)) - print() - print("To see full details of changes:") - if worktree_info: - print( - muted( - f" git diff {worktree_info.base_branch}...{worktree_info.branch}" - ) - ) - print() - print("To test the feature:") - if staging_path: - print(highlight(f" cd {staging_path}")) - print() - print("To add these changes to your project:") - print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge")) - print() - - else: # LATER - print() - print_status("No problem! Your build is saved.", "success") - print() - print("To test the feature:") - if staging_path: - print(highlight(f" cd {staging_path}")) - else: - print(highlight(f" cd {project_dir}/.worktrees/{spec_name}")) - print() - print("When you're ready to add it:") - print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge")) - print() - print("To see what was built:") - print(muted(f" python auto-claude/run.py --spec {spec_name} --review")) - print() - +# The following functions are now imported from refactored modules above. +# They are kept here only to avoid breaking the existing code that still needs +# the complex merge operations below. + +# Remaining complex merge operations that reference each other: +# - merge_existing_build +# - _try_smart_merge +# - _try_smart_merge_inner +# - _check_git_conflicts +# - _resolve_git_conflicts_with_ai +# - _create_async_claude_client +# - _async_ai_call +# - _merge_file_with_ai_async +# - _run_parallel_merges +# - _record_merge_completion +# - _get_task_intent +# - _get_recent_merges_context +# - _merge_file_with_ai +# - _heuristic_merge def merge_existing_build( project_dir: Path, @@ -1236,15 +667,22 @@ def _resolve_git_conflicts_with_ai( simple_merges.append((file_path, None)) # None = delete debug(MODULE, f" {file_path}: deleted (no AI needed)") else: - # File exists in both - needs AI merge - files_needing_ai_merge.append(ParallelMergeTask( - file_path=file_path, - main_content=main_content, - worktree_content=worktree_content, - base_content=base_content, - spec_name=spec_name, - )) - debug(MODULE, f" {file_path}: needs AI merge") + # File exists in both - check if it's a lock file + if _is_lock_file(file_path): + # Lock files should never go through AI - just take worktree version + # User can run package manager install to regenerate if needed + simple_merges.append((file_path, worktree_content)) + debug(MODULE, f" {file_path}: lock file (taking worktree version)") + else: + # Regular file - needs AI merge + files_needing_ai_merge.append(ParallelMergeTask( + file_path=file_path, + main_content=main_content, + worktree_content=worktree_content, + base_content=base_content, + spec_name=spec_name, + )) + debug(MODULE, f" {file_path}: needs AI merge") except Exception as e: print(error(f" ✗ Failed to categorize {file_path}: {e}")) @@ -1265,7 +703,11 @@ def _resolve_git_conflicts_with_ai( target_path.write_text(merged_content, encoding="utf-8") subprocess.run(["git", "add", file_path], cwd=project_dir, capture_output=True) resolved_files.append(file_path) - print(success(f" ✓ {file_path} (new file)")) + # Determine the type for display + if _is_lock_file(file_path): + print(success(f" ✓ {file_path} (lock file - took worktree version)")) + else: + print(success(f" ✓ {file_path} (new file)")) else: # Delete the file target_path = project_dir / file_path @@ -1330,14 +772,8 @@ def _resolve_git_conflicts_with_ai( if remaining_conflicts: print(muted(f" Failed: {len(remaining_conflicts)}")) - if remaining_conflicts: - return { - "success": False, - "resolved_files": resolved_files, - "remaining_conflicts": remaining_conflicts, - } - - # All conflicts resolved - now merge remaining non-conflicting files + # ALWAYS process non-conflicting files, even if some conflicts failed + # This ensures we get as much of the build as possible # (New files were already copied at the start) print(muted(" Merging remaining files...")) @@ -1371,18 +807,31 @@ def _resolve_git_conflicts_with_ai( if resolved_files: _record_merge_completion(project_dir, spec_name, resolved_files) - return { - "success": True, + # Build result - partial success if some files failed but we got others + result = { + "success": len(remaining_conflicts) == 0, "resolved_files": resolved_files, "stats": { "files_merged": len(resolved_files), - "conflicts_resolved": len(conflicting_files), + "conflicts_resolved": len(conflicting_files) - len(remaining_conflicts), "ai_assisted": ai_merged_count, "auto_merged": auto_merged_count, "parallel_ai_merges": len(files_needing_ai_merge), }, } + # Add remaining conflicts if any (for UI to show what needs manual attention) + if remaining_conflicts: + result["remaining_conflicts"] = remaining_conflicts + result["partial_success"] = len(resolved_files) > 0 + print() + print(warning(f" ⚠ {len(remaining_conflicts)} file(s) could not be auto-merged:")) + for conflict in remaining_conflicts: + print(muted(f" - {conflict['file']}: {conflict['reason']}")) + print(muted(" These files may need manual review.")) + + return result + def _get_file_content_from_ref(project_dir: Path, ref: str, file_path: str) -> Optional[str]: """Get file content from a git ref (branch, commit, etc.).""" @@ -1702,14 +1151,55 @@ async def _merge_file_with_ai_async( resolutions = extract_conflict_resolutions(response, conflicts, language) if resolutions: merged = reassemble_with_resolutions(merged_content, conflicts, resolutions) - is_valid, _ = _validate_merged_syntax(file_path, merged, project_dir) + is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) + + # Retry loop: if syntax is invalid, give AI feedback to fix it + retry_count = 0 + while not is_valid and retry_count < MAX_SYNTAX_FIX_RETRIES: + retry_count += 1 + debug(MODULE, f"[PARALLEL] Syntax error in merge, retry {retry_count}/{MAX_SYNTAX_FIX_RETRIES}: {file_path}") + + # Build fix prompt with error feedback + fix_prompt = f"""The merged code you produced has a syntax error: + +{error_msg} + +Here is your merged code that has the error: +```{language} +{merged} +``` + +Please fix the syntax error and output the corrected code. Make sure: +1. All lines are properly separated (no concatenated lines) +2. All brackets/braces are properly matched +3. All statements end correctly + +Output ONLY the fixed code, no explanations.""" + + fix_response = await _async_ai_call( + client, + "You are an expert code fixer. Fix the syntax error in the code.", + fix_prompt, + ) + + if fix_response: + fixed = resolver._extract_code_block(fix_response, language) + if not fixed and resolver._looks_like_code(fix_response, language): + fixed = fix_response.strip() + if fixed: + merged = fixed + is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) + if is_valid: - debug_success(MODULE, f"[PARALLEL] Conflict-only merge succeeded: {file_path}") + debug_success(MODULE, f"[PARALLEL] Conflict-only merge succeeded: {file_path}" + + (f" (after {retry_count} fix retries)" if retry_count > 0 else "")) return ParallelMergeResult( file_path=file_path, merged_content=merged, success=True ) + else: + debug(MODULE, f"[PARALLEL] Conflict-only merge failed validation after {retry_count} retries, falling back to full-file: {file_path}") # Case 3: Full-file AI merge (fallback) debug(MODULE, f"[PARALLEL] Full-file merge for: {file_path}") @@ -1736,21 +1226,62 @@ async def _merge_file_with_ai_async( merged = response.strip() if merged: - is_valid, _ = _validate_merged_syntax(file_path, merged, project_dir) + is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) + + # Retry loop: if syntax is invalid, give AI feedback to fix it + retry_count = 0 + while not is_valid and retry_count < MAX_SYNTAX_FIX_RETRIES: + retry_count += 1 + debug(MODULE, f"[PARALLEL] Syntax error in full-file merge, retry {retry_count}/{MAX_SYNTAX_FIX_RETRIES}: {file_path}") + + # Build fix prompt with error feedback + fix_prompt = f"""The merged code you produced has a syntax error: + +{error_msg} + +Here is your merged code that has the error: +```{language} +{merged} +``` + +Please fix the syntax error and output the corrected code. Make sure: +1. All lines are properly separated (no concatenated lines) +2. All brackets/braces are properly matched +3. All statements end correctly + +Output ONLY the fixed code, no explanations.""" + + fix_response = await _async_ai_call( + client, + "You are an expert code fixer. Fix the syntax error in the code.", + fix_prompt, + ) + + if fix_response: + fixed = resolver._extract_code_block(fix_response, language) + if not fixed and resolver._looks_like_code(fix_response, language): + fixed = fix_response.strip() + if fixed: + merged = fixed + is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) + if is_valid: - debug_success(MODULE, f"[PARALLEL] Full-file merge succeeded: {file_path}") + debug_success(MODULE, f"[PARALLEL] Full-file merge succeeded: {file_path}" + + (f" (after {retry_count} fix retries)" if retry_count > 0 else "")) return ParallelMergeResult( file_path=file_path, merged_content=merged, success=True ) + else: + debug_error(MODULE, f"[PARALLEL] Full-file merge failed validation after {retry_count} retries: {file_path}") # AI couldn't merge return ParallelMergeResult( file_path=file_path, merged_content=None, success=False, - error="AI could not merge file" + error="AI could not merge file - syntax validation failed after retries" ) except Exception as e: diff --git a/auto-claude/workspace/README.md b/auto-claude/core/workspace/README.md similarity index 100% rename from auto-claude/workspace/README.md rename to auto-claude/core/workspace/README.md diff --git a/auto-claude/workspace/__init__.py b/auto-claude/core/workspace/__init__.py similarity index 98% rename from auto-claude/workspace/__init__.py rename to auto-claude/core/workspace/__init__.py index f30483ae..c088c313 100644 --- a/auto-claude/workspace/__init__.py +++ b/auto-claude/core/workspace/__init__.py @@ -104,6 +104,7 @@ from .finalization import ( __all__ = [ # Merge Operations (from workspace.py) 'merge_existing_build', + '_run_parallel_merges', # Private but used by tests # Models 'WorkspaceMode', 'WorkspaceChoice', diff --git a/auto-claude/workspace/display.py b/auto-claude/core/workspace/display.py similarity index 100% rename from auto-claude/workspace/display.py rename to auto-claude/core/workspace/display.py diff --git a/auto-claude/workspace/finalization.py b/auto-claude/core/workspace/finalization.py similarity index 100% rename from auto-claude/workspace/finalization.py rename to auto-claude/core/workspace/finalization.py diff --git a/auto-claude/workspace/git_utils.py b/auto-claude/core/workspace/git_utils.py similarity index 100% rename from auto-claude/workspace/git_utils.py rename to auto-claude/core/workspace/git_utils.py diff --git a/auto-claude/workspace/models.py b/auto-claude/core/workspace/models.py similarity index 100% rename from auto-claude/workspace/models.py rename to auto-claude/core/workspace/models.py diff --git a/auto-claude/workspace/setup.py b/auto-claude/core/workspace/setup.py similarity index 100% rename from auto-claude/workspace/setup.py rename to auto-claude/core/workspace/setup.py diff --git a/auto-claude/core/worktree.py b/auto-claude/core/worktree.py new file mode 100644 index 00000000..2aaae53b --- /dev/null +++ b/auto-claude/core/worktree.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +""" +Git Worktree Manager - Per-Spec Architecture +============================================= + +Each spec gets its own worktree: +- Worktree path: .worktrees/{spec-name}/ +- Branch name: auto-claude/{spec-name} + +This allows: +1. Multiple specs to be worked on simultaneously +2. Each spec's changes are isolated +3. Branches persist until explicitly merged +4. Clear 1:1:1 mapping: spec → worktree → branch +""" + +import asyncio +import re +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + + +class WorktreeError(Exception): + """Error during worktree operations.""" + + pass + + +@dataclass +class WorktreeInfo: + """Information about a spec's worktree.""" + + path: Path + branch: str + spec_name: str + base_branch: str + is_active: bool = True + commit_count: int = 0 + files_changed: int = 0 + additions: int = 0 + deletions: int = 0 + + +class WorktreeManager: + """ + Manages per-spec Git worktrees. + + Each spec gets its own worktree in .worktrees/{spec-name}/ with + a corresponding branch auto-claude/{spec-name}. + """ + + def __init__(self, project_dir: Path, base_branch: str | None = None): + self.project_dir = project_dir + self.base_branch = base_branch or self._get_current_branch() + self.worktrees_dir = project_dir / ".worktrees" + self._merge_lock = asyncio.Lock() + + def _get_current_branch(self) -> str: + """Get the current git branch.""" + result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + cwd=self.project_dir, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise WorktreeError(f"Failed to get current branch: {result.stderr}") + return result.stdout.strip() + + def _run_git( + self, args: list[str], cwd: Path | None = None + ) -> subprocess.CompletedProcess: + """Run a git command and return the result.""" + return subprocess.run( + ["git"] + args, + cwd=cwd or self.project_dir, + capture_output=True, + text=True, + ) + + def _unstage_gitignored_files(self) -> None: + """ + Unstage any staged files that are gitignored in the current branch. + + This is needed after a --no-commit merge because files that exist in the + source branch (like spec files in auto-claude/specs/) get staged even if + they're gitignored in the target branch. + """ + # Get list of staged files + result = self._run_git(["diff", "--cached", "--name-only"]) + if result.returncode != 0 or not result.stdout.strip(): + return + + staged_files = result.stdout.strip().split("\n") + + # Check which staged files are gitignored + # git check-ignore returns the files that ARE ignored + result = self._run_git(["check-ignore", "--stdin"], cwd=self.project_dir) + # We need to pass the files via stdin + result = subprocess.run( + ["git", "check-ignore", "--stdin"], + cwd=self.project_dir, + input="\n".join(staged_files), + capture_output=True, + text=True, + ) + + if not result.stdout.strip(): + return + + ignored_files = result.stdout.strip().split("\n") + + if ignored_files: + print(f"Unstaging {len(ignored_files)} gitignored file(s)...") + # Unstage each ignored file + for file in ignored_files: + if file.strip(): + self._run_git(["reset", "HEAD", "--", file.strip()]) + + def setup(self) -> None: + """Create worktrees directory if needed.""" + self.worktrees_dir.mkdir(exist_ok=True) + + # ==================== Per-Spec Worktree Methods ==================== + + def get_worktree_path(self, spec_name: str) -> Path: + """Get the worktree path for a spec.""" + return self.worktrees_dir / spec_name + + def get_branch_name(self, spec_name: str) -> str: + """Get the branch name for a spec.""" + return f"auto-claude/{spec_name}" + + def worktree_exists(self, spec_name: str) -> bool: + """Check if a worktree exists for a spec.""" + return self.get_worktree_path(spec_name).exists() + + def get_worktree_info(self, spec_name: str) -> WorktreeInfo | None: + """Get info about a spec's worktree.""" + worktree_path = self.get_worktree_path(spec_name) + if not worktree_path.exists(): + return None + + # Verify the branch exists in the worktree + result = self._run_git(["rev-parse", "--abbrev-ref", "HEAD"], cwd=worktree_path) + if result.returncode != 0: + return None + + actual_branch = result.stdout.strip() + + # Get statistics + stats = self._get_worktree_stats(spec_name) + + return WorktreeInfo( + path=worktree_path, + branch=actual_branch, + spec_name=spec_name, + base_branch=self.base_branch, + is_active=True, + **stats, + ) + + def _get_worktree_stats(self, spec_name: str) -> dict: + """Get diff statistics for a worktree.""" + worktree_path = self.get_worktree_path(spec_name) + + stats = { + "commit_count": 0, + "files_changed": 0, + "additions": 0, + "deletions": 0, + } + + if not worktree_path.exists(): + return stats + + # Commit count + result = self._run_git( + ["rev-list", "--count", f"{self.base_branch}..HEAD"], cwd=worktree_path + ) + if result.returncode == 0: + stats["commit_count"] = int(result.stdout.strip() or "0") + + # Diff stats + result = self._run_git( + ["diff", "--shortstat", f"{self.base_branch}...HEAD"], cwd=worktree_path + ) + if result.returncode == 0 and result.stdout.strip(): + # Parse: "3 files changed, 50 insertions(+), 10 deletions(-)" + match = re.search(r"(\d+) files? changed", result.stdout) + if match: + stats["files_changed"] = int(match.group(1)) + match = re.search(r"(\d+) insertions?", result.stdout) + if match: + stats["additions"] = int(match.group(1)) + match = re.search(r"(\d+) deletions?", result.stdout) + if match: + stats["deletions"] = int(match.group(1)) + + return stats + + def create_worktree(self, spec_name: str) -> WorktreeInfo: + """ + Create a worktree for a spec. + + Args: + spec_name: The spec folder name (e.g., "002-implement-memory") + + Returns: + WorktreeInfo for the created worktree + """ + worktree_path = self.get_worktree_path(spec_name) + branch_name = self.get_branch_name(spec_name) + + # Remove existing if present (from crashed previous run) + if worktree_path.exists(): + self._run_git(["worktree", "remove", "--force", str(worktree_path)]) + + # Delete branch if it exists (from previous attempt) + self._run_git(["branch", "-D", branch_name]) + + # Create worktree with new branch from base + result = self._run_git( + ["worktree", "add", "-b", branch_name, str(worktree_path), self.base_branch] + ) + + if result.returncode != 0: + raise WorktreeError( + f"Failed to create worktree for {spec_name}: {result.stderr}" + ) + + print(f"Created worktree: {worktree_path.name} on branch {branch_name}") + + return WorktreeInfo( + path=worktree_path, + branch=branch_name, + spec_name=spec_name, + base_branch=self.base_branch, + is_active=True, + ) + + def get_or_create_worktree(self, spec_name: str) -> WorktreeInfo: + """ + Get existing worktree or create a new one for a spec. + + Args: + spec_name: The spec folder name + + Returns: + WorktreeInfo for the worktree + """ + existing = self.get_worktree_info(spec_name) + if existing: + print(f"Using existing worktree: {existing.path}") + return existing + + return self.create_worktree(spec_name) + + def remove_worktree(self, spec_name: str, delete_branch: bool = False) -> None: + """ + Remove a spec's worktree. + + Args: + spec_name: The spec folder name + delete_branch: Whether to also delete the branch + """ + worktree_path = self.get_worktree_path(spec_name) + branch_name = self.get_branch_name(spec_name) + + if worktree_path.exists(): + result = self._run_git( + ["worktree", "remove", "--force", str(worktree_path)] + ) + if result.returncode == 0: + print(f"Removed worktree: {worktree_path.name}") + else: + print(f"Warning: Could not remove worktree: {result.stderr}") + shutil.rmtree(worktree_path, ignore_errors=True) + + if delete_branch: + self._run_git(["branch", "-D", branch_name]) + print(f"Deleted branch: {branch_name}") + + self._run_git(["worktree", "prune"]) + + def merge_worktree( + self, spec_name: str, delete_after: bool = False, no_commit: bool = False + ) -> bool: + """ + Merge a spec's worktree branch back to base branch. + + Args: + spec_name: The spec folder name + delete_after: Whether to remove worktree and branch after merge + no_commit: If True, merge changes but don't commit (stage only for review) + + Returns: + True if merge succeeded + """ + info = self.get_worktree_info(spec_name) + if not info: + print(f"No worktree found for spec: {spec_name}") + return False + + if no_commit: + print( + f"Merging {info.branch} into {self.base_branch} (staged, not committed)..." + ) + else: + print(f"Merging {info.branch} into {self.base_branch}...") + + # Switch to base branch in main project + result = self._run_git(["checkout", self.base_branch]) + if result.returncode != 0: + print(f"Error: Could not checkout base branch: {result.stderr}") + return False + + # Merge the spec branch + merge_args = ["merge", "--no-ff", info.branch] + if no_commit: + # --no-commit stages the merge but doesn't create the commit + merge_args.append("--no-commit") + else: + merge_args.extend(["-m", f"auto-claude: Merge {info.branch}"]) + + result = self._run_git(merge_args) + + if result.returncode != 0: + print("Merge conflict! Aborting merge...") + self._run_git(["merge", "--abort"]) + return False + + if no_commit: + # Unstage any files that are gitignored in the main branch + # These get staged during merge because they exist in the worktree branch + self._unstage_gitignored_files() + print( + f"Changes from {info.branch} are now staged in your working directory." + ) + print("Review the changes, then commit when ready:") + print(" git commit -m 'your commit message'") + else: + print(f"Successfully merged {info.branch}") + + if delete_after: + self.remove_worktree(spec_name, delete_branch=True) + + return True + + def commit_in_worktree(self, spec_name: str, message: str) -> bool: + """Commit all changes in a spec's worktree.""" + worktree_path = self.get_worktree_path(spec_name) + if not worktree_path.exists(): + return False + + self._run_git(["add", "."], cwd=worktree_path) + result = self._run_git(["commit", "-m", message], cwd=worktree_path) + + if result.returncode == 0: + return True + elif "nothing to commit" in result.stdout + result.stderr: + return True + else: + print(f"Commit failed: {result.stderr}") + return False + + # ==================== Listing & Discovery ==================== + + def list_all_worktrees(self) -> list[WorktreeInfo]: + """List all spec worktrees.""" + worktrees = [] + + if not self.worktrees_dir.exists(): + return worktrees + + for item in self.worktrees_dir.iterdir(): + if item.is_dir(): + info = self.get_worktree_info(item.name) + if info: + worktrees.append(info) + + return worktrees + + def list_all_spec_branches(self) -> list[str]: + """List all auto-claude branches (even if worktree removed).""" + result = self._run_git(["branch", "--list", "auto-claude/*"]) + if result.returncode != 0: + return [] + + branches = [] + for line in result.stdout.strip().split("\n"): + branch = line.strip().lstrip("* ") + if branch: + branches.append(branch) + + return branches + + def get_changed_files(self, spec_name: str) -> list[tuple[str, str]]: + """Get list of changed files in a spec's worktree.""" + worktree_path = self.get_worktree_path(spec_name) + if not worktree_path.exists(): + return [] + + result = self._run_git( + ["diff", "--name-status", f"{self.base_branch}...HEAD"], cwd=worktree_path + ) + + files = [] + for line in result.stdout.strip().split("\n"): + if not line: + continue + parts = line.split("\t", 1) + if len(parts) == 2: + files.append((parts[0], parts[1])) + + return files + + def get_change_summary(self, spec_name: str) -> dict: + """Get a summary of changes in a worktree.""" + files = self.get_changed_files(spec_name) + + new_files = sum(1 for status, _ in files if status == "A") + modified_files = sum(1 for status, _ in files if status == "M") + deleted_files = sum(1 for status, _ in files if status == "D") + + return { + "new_files": new_files, + "modified_files": modified_files, + "deleted_files": deleted_files, + } + + def cleanup_all(self) -> None: + """Remove all worktrees and their branches.""" + for worktree in self.list_all_worktrees(): + self.remove_worktree(worktree.spec_name, delete_branch=True) + + def cleanup_stale_worktrees(self) -> None: + """Remove worktrees that aren't registered with git.""" + if not self.worktrees_dir.exists(): + return + + # Get list of registered worktrees + result = self._run_git(["worktree", "list", "--porcelain"]) + registered_paths = set() + for line in result.stdout.split("\n"): + if line.startswith("worktree "): + registered_paths.add(Path(line.split(" ", 1)[1])) + + # Remove unregistered directories + for item in self.worktrees_dir.iterdir(): + if item.is_dir() and item not in registered_paths: + print(f"Removing stale worktree directory: {item.name}") + shutil.rmtree(item, ignore_errors=True) + + self._run_git(["worktree", "prune"]) + + def get_test_commands(self, spec_name: str) -> list[str]: + """Detect likely test/run commands for the project.""" + worktree_path = self.get_worktree_path(spec_name) + commands = [] + + if (worktree_path / "package.json").exists(): + commands.append("npm install && npm run dev") + commands.append("npm test") + + if (worktree_path / "requirements.txt").exists(): + commands.append("pip install -r requirements.txt") + + if (worktree_path / "Cargo.toml").exists(): + commands.append("cargo run") + commands.append("cargo test") + + if (worktree_path / "go.mod").exists(): + commands.append("go run .") + commands.append("go test ./...") + + if not commands: + commands.append("# Check the project's README for run instructions") + + return commands + + # ==================== Backward Compatibility ==================== + # These methods provide backward compatibility with the old single-worktree API + + def get_staging_path(self) -> Path | None: + """ + Backward compatibility: Get path to any existing spec worktree. + Prefer using get_worktree_path(spec_name) instead. + """ + worktrees = self.list_all_worktrees() + if worktrees: + return worktrees[0].path + return None + + def get_staging_info(self) -> WorktreeInfo | None: + """ + Backward compatibility: Get info about any existing spec worktree. + Prefer using get_worktree_info(spec_name) instead. + """ + worktrees = self.list_all_worktrees() + if worktrees: + return worktrees[0] + return None + + def merge_staging(self, delete_after: bool = True) -> bool: + """ + Backward compatibility: Merge first found worktree. + Prefer using merge_worktree(spec_name) instead. + """ + worktrees = self.list_all_worktrees() + if worktrees: + return self.merge_worktree(worktrees[0].spec_name, delete_after) + return False + + def remove_staging(self, delete_branch: bool = True) -> None: + """ + Backward compatibility: Remove first found worktree. + Prefer using remove_worktree(spec_name) instead. + """ + worktrees = self.list_all_worktrees() + if worktrees: + self.remove_worktree(worktrees[0].spec_name, delete_branch) + + def get_or_create_staging(self, spec_name: str) -> WorktreeInfo: + """ + Backward compatibility: Alias for get_or_create_worktree. + """ + return self.get_or_create_worktree(spec_name) + + def staging_exists(self) -> bool: + """ + Backward compatibility: Check if any spec worktree exists. + Prefer using worktree_exists(spec_name) instead. + """ + return len(self.list_all_worktrees()) > 0 + + def commit_in_staging(self, message: str) -> bool: + """ + Backward compatibility: Commit in first found worktree. + Prefer using commit_in_worktree(spec_name, message) instead. + """ + worktrees = self.list_all_worktrees() + if worktrees: + return self.commit_in_worktree(worktrees[0].spec_name, message) + return False + + def has_uncommitted_changes(self, in_staging: bool = False) -> bool: + """Check if there are uncommitted changes.""" + worktrees = self.list_all_worktrees() + if in_staging and worktrees: + cwd = worktrees[0].path + else: + cwd = None + result = self._run_git(["status", "--porcelain"], cwd=cwd) + return bool(result.stdout.strip()) + + +# Keep STAGING_WORKTREE_NAME for backward compatibility in imports +STAGING_WORKTREE_NAME = "auto-claude" diff --git a/auto-claude/critique.py b/auto-claude/critique.py index 3308db84..68fc3b57 100644 --- a/auto-claude/critique.py +++ b/auto-claude/critique.py @@ -1,369 +1,2 @@ -#!/usr/bin/env python3 -""" -Self-Critique System -==================== - -Implements a self-critique loop that agents must run before marking subtasks complete. -This helps catch quality issues early, before verification stage. - -The critique system ensures: -- Code follows patterns from reference files -- All required files were modified/created -- Error handling is present -- No debugging artifacts left behind -- Implementation matches subtask requirements -""" - -import re -from dataclasses import dataclass, field - - -@dataclass -class CritiqueResult: - """Result of a self-critique evaluation.""" - - passes: bool - issues: list[str] = field(default_factory=list) - improvements_made: list[str] = field(default_factory=list) - recommendations: list[str] = field(default_factory=list) - - def to_dict(self) -> dict: - """Convert to dictionary for storage.""" - return { - "passes": self.passes, - "issues": self.issues, - "improvements_made": self.improvements_made, - "recommendations": self.recommendations, - } - - @classmethod - def from_dict(cls, data: dict) -> "CritiqueResult": - """Load from dictionary.""" - return cls( - passes=data.get("passes", False), - issues=data.get("issues", []), - improvements_made=data.get("improvements_made", []), - recommendations=data.get("recommendations", []), - ) - - -def generate_critique_prompt( - subtask: dict, files_modified: list[str], patterns_from: list[str] -) -> str: - """ - Generate a critique prompt for the agent to self-evaluate. - - Args: - subtask: The subtask being implemented - files_modified: List of files actually modified - patterns_from: List of pattern files to compare against - - Returns: - Formatted prompt for self-critique - """ - subtask_id = subtask.get("id", "unknown") - subtask_desc = subtask.get("description", "No description") - service = subtask.get("service", "all services") - files_to_modify = subtask.get("files_to_modify", []) - files_to_create = subtask.get("files_to_create", []) - - prompt = f"""## MANDATORY Self-Critique: {subtask_id} - -**Subtask Description:** {subtask_desc} -**Service:** {service} - -Before marking this subtask as complete, you MUST perform a thorough self-critique. -This is NOT optional - it's a required quality gate. - -### STEP 1: Code Quality Checklist - -Review your implementation against these criteria: - -**Pattern Adherence:** -- [ ] Follows patterns from reference files exactly: {", ".join(patterns_from) if patterns_from else "N/A"} -- [ ] Variable naming matches codebase conventions -- [ ] Imports organized correctly (grouped, sorted) -- [ ] Code style consistent with existing files - -**Error Handling:** -- [ ] Try-catch blocks where operations can fail -- [ ] Meaningful error messages -- [ ] Proper error propagation -- [ ] Edge cases considered - -**Code Cleanliness:** -- [ ] No console.log/print statements for debugging -- [ ] No commented-out code blocks -- [ ] No TODO comments without context -- [ ] No hardcoded values that should be configurable - -**Best Practices:** -- [ ] Functions are focused and single-purpose -- [ ] No code duplication -- [ ] Appropriate use of constants -- [ ] Documentation/comments where needed - -### STEP 2: Implementation Completeness - -**Files Modified:** -Expected: {", ".join(files_to_modify) if files_to_modify else "None"} -Actual: {", ".join(files_modified) if files_modified else "None"} -- [ ] All files_to_modify were actually modified -- [ ] No unexpected files were modified - -**Files Created:** -Expected: {", ".join(files_to_create) if files_to_create else "None"} -- [ ] All files_to_create were actually created -- [ ] Files follow naming conventions - -**Requirements:** -- [ ] Subtask description requirements fully met -- [ ] All acceptance criteria from spec considered -- [ ] No scope creep - stayed within subtask boundaries - -### STEP 3: Potential Issues Analysis - -List any concerns, limitations, or potential problems with your implementation: - -1. [Issue 1, or "None identified"] -2. [Issue 2, if any] -3. [Issue 3, if any] - -Be honest. Finding issues now is better than discovering them during verification. - -### STEP 4: Improvements Made - -If you identified issues in your critique, list what you fixed: - -1. [Improvement 1, or "No fixes needed"] -2. [Improvement 2, if applicable] -3. [Improvement 3, if applicable] - -### STEP 5: Final Verdict - -**PROCEED:** [YES/NO - Only YES if all critical items pass] - -**REASON:** [Brief explanation of your decision] - -**CONFIDENCE:** [High/Medium/Low - How confident are you in this implementation?] - ---- - -## Instructions for Agent - -1. Work through each section methodically -2. Check each box honestly - don't skip items -3. If you find issues, FIX THEM before continuing -4. Re-run this critique after fixes -5. Only mark the subtask complete when verdict is YES with High confidence -6. Document your critique results in your response - -Remember: The next session has no context. Quality issues you miss now will be harder to fix later. -""" - - return prompt - - -def parse_critique_response(response: str) -> CritiqueResult: - """ - Parse the agent's critique response into structured data. - - Args: - response: The agent's response to the critique prompt - - Returns: - CritiqueResult with parsed information - """ - issues = [] - improvements = [] - recommendations = [] - passes = False - - # Extract PROCEED verdict - proceed_match = re.search( - r"\*\*PROCEED:\*\*\s*\[?\s*(YES|NO)", response, re.IGNORECASE - ) - if proceed_match: - passes = proceed_match.group(1).upper() == "YES" - - # Extract issues from Step 3 - issues_section = re.search( - r"### STEP 3:.*?Potential Issues.*?\n\n(.*?)(?=###|\Z)", - response, - re.DOTALL | re.IGNORECASE, - ) - if issues_section: - issue_lines = issues_section.group(1).strip().split("\n") - for line in issue_lines: - line = line.strip() - if not line or line.startswith("---"): - continue - # Remove list markers - issue = re.sub(r"^\d+\.\s*|\*\s*|-\s*", "", line).strip() - # Skip if it's a placeholder or indicates no issues - if ( - issue - and issue.lower() - not in ["none", "none identified", "no issues", "no concerns"] - and issue - not in [ - '[Issue 1, or "None identified"]', - "[Issue 2, if any]", - "[Issue 3, if any]", - ] - ): - issues.append(issue) - - # Extract improvements from Step 4 - improvements_section = re.search( - r"### STEP 4:.*?Improvements Made.*?\n\n(.*?)(?=###|\Z)", - response, - re.DOTALL | re.IGNORECASE, - ) - if improvements_section: - improvement_lines = improvements_section.group(1).strip().split("\n") - for line in improvement_lines: - line = line.strip() - if not line or line.startswith("---"): - continue - # Remove list markers - improvement = re.sub(r"^\d+\.\s*|\*\s*|-\s*", "", line).strip() - # Skip if it's a placeholder or indicates no improvements - if ( - improvement - and improvement.lower() - not in ["none", "no fixes needed", "no improvements", "n/a"] - and improvement - not in [ - '[Improvement 1, or "No fixes needed"]', - "[Improvement 2, if applicable]", - "[Improvement 3, if applicable]", - ] - ): - improvements.append(improvement) - - # Extract confidence level as recommendation - confidence_match = re.search( - r"\*\*CONFIDENCE:\*\*\s*\[?\s*(High|Medium|Low)", response, re.IGNORECASE - ) - if confidence_match: - confidence = confidence_match.group(1) - if confidence.lower() != "high": - recommendations.append( - f"Confidence level: {confidence} - consider additional review" - ) - - return CritiqueResult( - passes=passes, - issues=issues, - improvements_made=improvements, - recommendations=recommendations, - ) - - -def should_proceed(result: CritiqueResult) -> bool: - """ - Determine if the subtask should be marked complete based on critique. - - Args: - result: The critique result - - Returns: - True if subtask can be marked complete, False otherwise - """ - # Must pass the critique - if not result.passes: - return False - - # If there are unresolved issues, don't proceed - if result.issues: - return False - - return True - - -def format_critique_summary(result: CritiqueResult) -> str: - """ - Format a critique result as a human-readable summary. - - Args: - result: The critique result - - Returns: - Formatted summary string - """ - lines = ["## Critique Summary"] - lines.append("") - lines.append(f"**Status:** {'PASSED ✓' if result.passes else 'FAILED ✗'}") - lines.append("") - - if result.issues: - lines.append("**Issues Identified:**") - for i, issue in enumerate(result.issues, 1): - lines.append(f"{i}. {issue}") - lines.append("") - - if result.improvements_made: - lines.append("**Improvements Made:**") - for i, improvement in enumerate(result.improvements_made, 1): - lines.append(f"{i}. {improvement}") - lines.append("") - - if result.recommendations: - lines.append("**Recommendations:**") - for i, rec in enumerate(result.recommendations, 1): - lines.append(f"{i}. {rec}") - lines.append("") - - if should_proceed(result): - lines.append("**Decision:** Subtask is ready to be marked complete.") - else: - lines.append("**Decision:** Subtask needs more work before completion.") - - return "\n".join(lines) - - -# Example usage for testing -if __name__ == "__main__": - # Demo subtask - subtask = { - "id": "auth-middleware", - "description": "Add JWT authentication middleware", - "service": "backend", - "files_to_modify": ["app/middleware/auth.py"], - "patterns_from": ["app/middleware/cors.py"], - } - - files_modified = ["app/middleware/auth.py"] - - # Generate prompt - prompt = generate_critique_prompt(subtask, files_modified, subtask["patterns_from"]) - print(prompt) - print("\n" + "=" * 80 + "\n") - - # Simulate a critique response - sample_response = """ -### STEP 3: Potential Issues Analysis - -1. Token expiration edge case not fully tested -2. None - -### STEP 4: Improvements Made - -1. Added comprehensive error handling for invalid tokens -2. Improved logging for debugging -3. Added input validation for JWT format - -### STEP 5: Final Verdict - -**PROCEED:** YES - -**REASON:** All critical items verified, patterns followed, error handling complete - -**CONFIDENCE:** High -""" - - # Parse response - result = parse_critique_response(sample_response) - print(format_critique_summary(result)) - print(f"\nShould proceed: {should_proceed(result)}") +"""Backward compatibility shim - import from spec.critique instead.""" +from spec.critique import * diff --git a/auto-claude/debug.py b/auto-claude/debug.py index 9bef363d..ee2df503 100644 --- a/auto-claude/debug.py +++ b/auto-claude/debug.py @@ -1,349 +1,2 @@ -#!/usr/bin/env python3 -""" -Debug Logging Utility -===================== - -Centralized debug logging for the Auto-Claude framework. -Controlled via environment variables: - - DEBUG=true Enable debug mode - - DEBUG_LEVEL=1|2|3 Log verbosity (1=basic, 2=detailed, 3=verbose) - - DEBUG_LOG_FILE=path Optional file output - -Usage: - from debug import debug, debug_detailed, debug_verbose, is_debug_enabled - - debug("run.py", "Starting task execution", task_id="001") - debug_detailed("agent", "Agent response received", response_length=1234) - debug_verbose("client", "Full request payload", payload=data) -""" - -import json -import os -import sys -import time -from datetime import datetime -from functools import wraps -from pathlib import Path -from typing import Any - - -# ANSI color codes for terminal output -class Colors: - RESET = "\033[0m" - BOLD = "\033[1m" - DIM = "\033[2m" - - # Debug colors - DEBUG = "\033[36m" # Cyan - DEBUG_DIM = "\033[96m" # Light cyan - TIMESTAMP = "\033[90m" # Gray - MODULE = "\033[33m" # Yellow - KEY = "\033[35m" # Magenta - VALUE = "\033[37m" # White - SUCCESS = "\033[32m" # Green - WARNING = "\033[33m" # Yellow - ERROR = "\033[31m" # Red - - -def _get_debug_enabled() -> bool: - """Check if debug mode is enabled via environment variable.""" - return os.environ.get("DEBUG", "").lower() in ("true", "1", "yes", "on") - - -def _get_debug_level() -> int: - """Get debug verbosity level (1-3).""" - try: - level = int(os.environ.get("DEBUG_LEVEL", "1")) - return max(1, min(3, level)) # Clamp to 1-3 - except ValueError: - return 1 - - -def _get_log_file() -> Path | None: - """Get optional log file path.""" - log_file = os.environ.get("DEBUG_LOG_FILE") - if log_file: - return Path(log_file) - return None - - -def is_debug_enabled() -> bool: - """Check if debug mode is enabled.""" - return _get_debug_enabled() - - -def get_debug_level() -> int: - """Get current debug level.""" - return _get_debug_level() - - -def _format_value(value: Any, max_length: int = 200) -> str: - """Format a value for debug output, truncating if necessary.""" - if value is None: - return "None" - - if isinstance(value, (dict, list)): - try: - formatted = json.dumps(value, indent=2, default=str) - if len(formatted) > max_length: - formatted = formatted[:max_length] + "..." - return formatted - except (TypeError, ValueError): - return str(value)[:max_length] - - str_value = str(value) - if len(str_value) > max_length: - return str_value[:max_length] + "..." - return str_value - - -def _write_log(message: str, to_file: bool = True) -> None: - """Write log message to stdout and optionally to file.""" - print(message, file=sys.stderr) - - if to_file: - log_file = _get_log_file() - if log_file: - try: - log_file.parent.mkdir(parents=True, exist_ok=True) - # Strip ANSI codes for file output - import re - - clean_message = re.sub(r"\033\[[0-9;]*m", "", message) - with open(log_file, "a") as f: - f.write(clean_message + "\n") - except Exception: - pass # Silently fail file logging - - -def debug(module: str, message: str, level: int = 1, **kwargs) -> None: - """ - Log a debug message. - - Args: - module: Source module name (e.g., "run.py", "ideation_runner") - message: Debug message - level: Required debug level (1=basic, 2=detailed, 3=verbose) - **kwargs: Additional key-value pairs to log - """ - if not _get_debug_enabled(): - return - - if _get_debug_level() < level: - return - - timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - - # Build the log line - parts = [ - f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET}", - f"{Colors.DEBUG}[DEBUG]{Colors.RESET}", - f"{Colors.MODULE}[{module}]{Colors.RESET}", - f"{Colors.DEBUG_DIM}{message}{Colors.RESET}", - ] - - log_line = " ".join(parts) - - # Add kwargs on separate lines if present - if kwargs: - for key, value in kwargs.items(): - formatted_value = _format_value(value) - if "\n" in formatted_value: - # Multi-line value - log_line += f"\n {Colors.KEY}{key}{Colors.RESET}:" - for line in formatted_value.split("\n"): - log_line += f"\n {Colors.VALUE}{line}{Colors.RESET}" - else: - log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{formatted_value}{Colors.RESET}" - - _write_log(log_line) - - -def debug_detailed(module: str, message: str, **kwargs) -> None: - """Log a detailed debug message (level 2).""" - debug(module, message, level=2, **kwargs) - - -def debug_verbose(module: str, message: str, **kwargs) -> None: - """Log a verbose debug message (level 3).""" - debug(module, message, level=3, **kwargs) - - -def debug_success(module: str, message: str, **kwargs) -> None: - """Log a success debug message.""" - if not _get_debug_enabled(): - return - - timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.SUCCESS}[OK]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}" - - if kwargs: - for key, value in kwargs.items(): - log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}" - - _write_log(log_line) - - -def debug_info(module: str, message: str, **kwargs) -> None: - """Log an info debug message.""" - if not _get_debug_enabled(): - return - - timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}[INFO]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {message}" - - if kwargs: - for key, value in kwargs.items(): - log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}" - - _write_log(log_line) - - -def debug_error(module: str, message: str, **kwargs) -> None: - """Log an error debug message (always shown if debug enabled).""" - if not _get_debug_enabled(): - return - - timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.ERROR}[ERROR]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.ERROR}{message}{Colors.RESET}" - - if kwargs: - for key, value in kwargs.items(): - log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}" - - _write_log(log_line) - - -def debug_warning(module: str, message: str, **kwargs) -> None: - """Log a warning debug message.""" - if not _get_debug_enabled(): - return - - timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - log_line = f"{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.WARNING}[WARN]{Colors.RESET} {Colors.MODULE}[{module}]{Colors.RESET} {Colors.WARNING}{message}{Colors.RESET}" - - if kwargs: - for key, value in kwargs.items(): - log_line += f"\n {Colors.KEY}{key}{Colors.RESET}: {Colors.VALUE}{_format_value(value)}{Colors.RESET}" - - _write_log(log_line) - - -def debug_section(module: str, title: str) -> None: - """Log a section header for organizing debug output.""" - if not _get_debug_enabled(): - return - - timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] - separator = "─" * 60 - log_line = f"\n{Colors.TIMESTAMP}[{timestamp}]{Colors.RESET} {Colors.DEBUG}{Colors.BOLD}┌{separator}┐{Colors.RESET}" - log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}│ {module}: {title}{' ' * (58 - len(module) - len(title) - 2)}│{Colors.RESET}" - log_line += f"\n{Colors.TIMESTAMP} {Colors.RESET} {Colors.DEBUG}{Colors.BOLD}└{separator}┘{Colors.RESET}" - - _write_log(log_line) - - -def debug_timer(module: str): - """ - Decorator to time function execution. - - Usage: - @debug_timer("run.py") - def my_function(): - ... - """ - - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - if not _get_debug_enabled(): - return func(*args, **kwargs) - - start = time.time() - debug_detailed(module, f"Starting {func.__name__}()") - - try: - result = func(*args, **kwargs) - elapsed = time.time() - start - debug_success( - module, - f"Completed {func.__name__}()", - elapsed_ms=f"{elapsed * 1000:.1f}ms", - ) - return result - except Exception as e: - elapsed = time.time() - start - debug_error( - module, - f"Failed {func.__name__}()", - error=str(e), - elapsed_ms=f"{elapsed * 1000:.1f}ms", - ) - raise - - return wrapper - - return decorator - - -def debug_async_timer(module: str): - """ - Decorator to time async function execution. - - Usage: - @debug_async_timer("ideation_runner") - async def my_async_function(): - ... - """ - - def decorator(func): - @wraps(func) - async def wrapper(*args, **kwargs): - if not _get_debug_enabled(): - return await func(*args, **kwargs) - - start = time.time() - debug_detailed(module, f"Starting {func.__name__}()") - - try: - result = await func(*args, **kwargs) - elapsed = time.time() - start - debug_success( - module, - f"Completed {func.__name__}()", - elapsed_ms=f"{elapsed * 1000:.1f}ms", - ) - return result - except Exception as e: - elapsed = time.time() - start - debug_error( - module, - f"Failed {func.__name__}()", - error=str(e), - elapsed_ms=f"{elapsed * 1000:.1f}ms", - ) - raise - - return wrapper - - return decorator - - -def debug_env_status() -> None: - """Print debug environment status on startup.""" - if not _get_debug_enabled(): - return - - debug_section("debug", "Debug Mode Enabled") - debug( - "debug", - "Environment configuration", - DEBUG=os.environ.get("DEBUG", "not set"), - DEBUG_LEVEL=_get_debug_level(), - DEBUG_LOG_FILE=os.environ.get("DEBUG_LOG_FILE", "not set"), - ) - - -# Print status on import if debug is enabled -if _get_debug_enabled(): - debug_env_status() +"""Backward compatibility shim - import from core.debug instead.""" +from core.debug import * diff --git a/auto-claude/graphiti_config.py b/auto-claude/graphiti_config.py index 7b0decc0..6b632227 100644 --- a/auto-claude/graphiti_config.py +++ b/auto-claude/graphiti_config.py @@ -1,529 +1,2 @@ -""" -Graphiti Integration Configuration -================================== - -Constants, status mappings, and configuration helpers for Graphiti memory integration. -Follows the same patterns as linear_config.py for consistency. - -Multi-Provider Support (V2): -- LLM Providers: OpenAI, Anthropic, Azure OpenAI, Ollama -- Embedder Providers: OpenAI, Voyage AI, Azure OpenAI, Ollama - -Environment Variables: - # Core - GRAPHITI_ENABLED: Set to "true" to enable Graphiti integration - GRAPHITI_LLM_PROVIDER: openai|anthropic|azure_openai|ollama (default: openai) - GRAPHITI_EMBEDDER_PROVIDER: openai|voyage|azure_openai|ollama (default: openai) - - # OpenAI - OPENAI_API_KEY: Required for OpenAI provider - OPENAI_MODEL: Model for LLM (default: gpt-5-mini) - OPENAI_EMBEDDING_MODEL: Model for embeddings (default: text-embedding-3-small) - - # Anthropic (LLM only - needs separate embedder) - ANTHROPIC_API_KEY: Required for Anthropic provider - GRAPHITI_ANTHROPIC_MODEL: Model for LLM (default: claude-sonnet-4-5-latest) - - # Azure OpenAI - AZURE_OPENAI_API_KEY: Required for Azure provider - AZURE_OPENAI_BASE_URL: Azure endpoint URL - AZURE_OPENAI_LLM_DEPLOYMENT: Deployment name for LLM - AZURE_OPENAI_EMBEDDING_DEPLOYMENT: Deployment name for embeddings - - # Voyage AI (embeddings only - commonly used with Anthropic) - VOYAGE_API_KEY: Required for Voyage embedder - VOYAGE_EMBEDDING_MODEL: Model (default: voyage-3) - - # Ollama (local) - OLLAMA_BASE_URL: Ollama server URL (default: http://localhost:11434) - OLLAMA_LLM_MODEL: Model for LLM (e.g., deepseek-r1:7b) - OLLAMA_EMBEDDING_MODEL: Model for embeddings (e.g., nomic-embed-text) - OLLAMA_EMBEDDING_DIM: Embedding dimension (required for Ollama, e.g., 768) - - # FalkorDB - GRAPHITI_FALKORDB_HOST: FalkorDB host (default: localhost) - GRAPHITI_FALKORDB_PORT: FalkorDB port (default: 6380) - GRAPHITI_FALKORDB_PASSWORD: FalkorDB password (default: empty) - GRAPHITI_DATABASE: Graph database name (default: auto_claude_memory) - GRAPHITI_TELEMETRY_ENABLED: Set to "false" to disable telemetry (default: true) -""" - -import json -import os -from dataclasses import dataclass, field -from datetime import datetime -from enum import Enum -from pathlib import Path -from typing import Optional - -# Default configuration values -DEFAULT_FALKORDB_HOST = "localhost" -DEFAULT_FALKORDB_PORT = 6380 -DEFAULT_DATABASE = "auto_claude_memory" -DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434" - -# Graphiti state marker file (stores connection info and status) -GRAPHITI_STATE_MARKER = ".graphiti_state.json" - -# Episode types for different memory categories -EPISODE_TYPE_SESSION_INSIGHT = "session_insight" -EPISODE_TYPE_CODEBASE_DISCOVERY = "codebase_discovery" -EPISODE_TYPE_PATTERN = "pattern" -EPISODE_TYPE_GOTCHA = "gotcha" -EPISODE_TYPE_TASK_OUTCOME = "task_outcome" -EPISODE_TYPE_QA_RESULT = "qa_result" -EPISODE_TYPE_HISTORICAL_CONTEXT = "historical_context" - - -class LLMProvider(str, Enum): - """Supported LLM providers for Graphiti.""" - - OPENAI = "openai" - ANTHROPIC = "anthropic" - AZURE_OPENAI = "azure_openai" - OLLAMA = "ollama" - - -class EmbedderProvider(str, Enum): - """Supported embedder providers for Graphiti.""" - - OPENAI = "openai" - VOYAGE = "voyage" - AZURE_OPENAI = "azure_openai" - OLLAMA = "ollama" - - -@dataclass -class GraphitiConfig: - """Configuration for Graphiti memory integration with multi-provider support.""" - - # Core settings - enabled: bool = False - llm_provider: str = "openai" - embedder_provider: str = "openai" - - # FalkorDB connection - falkordb_host: str = DEFAULT_FALKORDB_HOST - falkordb_port: int = DEFAULT_FALKORDB_PORT - falkordb_password: str = "" - database: str = DEFAULT_DATABASE - telemetry_enabled: bool = True - - # OpenAI settings - openai_api_key: str = "" - openai_model: str = "gpt-5-mini" - openai_embedding_model: str = "text-embedding-3-small" - - # Anthropic settings (LLM only) - anthropic_api_key: str = "" - anthropic_model: str = "claude-sonnet-4-5-latest" - - # Azure OpenAI settings - azure_openai_api_key: str = "" - azure_openai_base_url: str = "" - azure_openai_llm_deployment: str = "" - azure_openai_embedding_deployment: str = "" - - # Voyage AI settings (embeddings only) - voyage_api_key: str = "" - voyage_embedding_model: str = "voyage-3" - - # Ollama settings (local) - ollama_base_url: str = DEFAULT_OLLAMA_BASE_URL - ollama_llm_model: str = "" - ollama_embedding_model: str = "" - ollama_embedding_dim: int = 0 # Required for Ollama embeddings - - @classmethod - def from_env(cls) -> "GraphitiConfig": - """Create config from environment variables.""" - # Check if Graphiti is explicitly enabled - enabled_str = os.environ.get("GRAPHITI_ENABLED", "").lower() - enabled = enabled_str in ("true", "1", "yes") - - # Provider selection - llm_provider = os.environ.get("GRAPHITI_LLM_PROVIDER", "openai").lower() - embedder_provider = os.environ.get( - "GRAPHITI_EMBEDDER_PROVIDER", "openai" - ).lower() - - # FalkorDB connection settings - falkordb_host = os.environ.get("GRAPHITI_FALKORDB_HOST", DEFAULT_FALKORDB_HOST) - - try: - falkordb_port = int( - os.environ.get("GRAPHITI_FALKORDB_PORT", str(DEFAULT_FALKORDB_PORT)) - ) - except ValueError: - falkordb_port = DEFAULT_FALKORDB_PORT - - falkordb_password = os.environ.get("GRAPHITI_FALKORDB_PASSWORD", "") - database = os.environ.get("GRAPHITI_DATABASE", DEFAULT_DATABASE) - - # Telemetry setting - telemetry_str = os.environ.get("GRAPHITI_TELEMETRY_ENABLED", "true").lower() - telemetry_enabled = telemetry_str not in ("false", "0", "no") - - # OpenAI settings - openai_api_key = os.environ.get("OPENAI_API_KEY", "") - openai_model = os.environ.get("OPENAI_MODEL", "gpt-5-mini") - openai_embedding_model = os.environ.get( - "OPENAI_EMBEDDING_MODEL", "text-embedding-3-small" - ) - - # Anthropic settings - anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY", "") - anthropic_model = os.environ.get( - "GRAPHITI_ANTHROPIC_MODEL", "claude-sonnet-4-5-latest" - ) - - # Azure OpenAI settings - azure_openai_api_key = os.environ.get("AZURE_OPENAI_API_KEY", "") - azure_openai_base_url = os.environ.get("AZURE_OPENAI_BASE_URL", "") - azure_openai_llm_deployment = os.environ.get("AZURE_OPENAI_LLM_DEPLOYMENT", "") - azure_openai_embedding_deployment = os.environ.get( - "AZURE_OPENAI_EMBEDDING_DEPLOYMENT", "" - ) - - # Voyage AI settings - voyage_api_key = os.environ.get("VOYAGE_API_KEY", "") - voyage_embedding_model = os.environ.get("VOYAGE_EMBEDDING_MODEL", "voyage-3") - - # Ollama settings - ollama_base_url = os.environ.get("OLLAMA_BASE_URL", DEFAULT_OLLAMA_BASE_URL) - ollama_llm_model = os.environ.get("OLLAMA_LLM_MODEL", "") - ollama_embedding_model = os.environ.get("OLLAMA_EMBEDDING_MODEL", "") - - # Ollama embedding dimension (required for Ollama) - try: - ollama_embedding_dim = int(os.environ.get("OLLAMA_EMBEDDING_DIM", "0")) - except ValueError: - ollama_embedding_dim = 0 - - return cls( - enabled=enabled, - llm_provider=llm_provider, - embedder_provider=embedder_provider, - falkordb_host=falkordb_host, - falkordb_port=falkordb_port, - falkordb_password=falkordb_password, - database=database, - telemetry_enabled=telemetry_enabled, - openai_api_key=openai_api_key, - openai_model=openai_model, - openai_embedding_model=openai_embedding_model, - anthropic_api_key=anthropic_api_key, - anthropic_model=anthropic_model, - azure_openai_api_key=azure_openai_api_key, - azure_openai_base_url=azure_openai_base_url, - azure_openai_llm_deployment=azure_openai_llm_deployment, - azure_openai_embedding_deployment=azure_openai_embedding_deployment, - voyage_api_key=voyage_api_key, - voyage_embedding_model=voyage_embedding_model, - ollama_base_url=ollama_base_url, - ollama_llm_model=ollama_llm_model, - ollama_embedding_model=ollama_embedding_model, - ollama_embedding_dim=ollama_embedding_dim, - ) - - def is_valid(self) -> bool: - """ - Check if config has minimum required values for operation. - - Returns True if: - - GRAPHITI_ENABLED is true - - LLM provider is configured correctly - - Embedder provider is configured correctly - """ - if not self.enabled: - return False - - # Validate LLM provider - if not self._validate_llm_provider(): - return False - - # Validate embedder provider - if not self._validate_embedder_provider(): - return False - - return True - - def _validate_llm_provider(self) -> bool: - """Validate LLM provider configuration.""" - if self.llm_provider == "openai": - return bool(self.openai_api_key) - elif self.llm_provider == "anthropic": - return bool(self.anthropic_api_key) - elif self.llm_provider == "azure_openai": - return bool( - self.azure_openai_api_key - and self.azure_openai_base_url - and self.azure_openai_llm_deployment - ) - elif self.llm_provider == "ollama": - return bool(self.ollama_llm_model) - return False - - def _validate_embedder_provider(self) -> bool: - """Validate embedder provider configuration.""" - if self.embedder_provider == "openai": - return bool(self.openai_api_key) - elif self.embedder_provider == "voyage": - return bool(self.voyage_api_key) - elif self.embedder_provider == "azure_openai": - return bool( - self.azure_openai_api_key - and self.azure_openai_base_url - and self.azure_openai_embedding_deployment - ) - elif self.embedder_provider == "ollama": - return bool(self.ollama_embedding_model and self.ollama_embedding_dim) - return False - - def get_validation_errors(self) -> list[str]: - """Get list of validation errors for current configuration.""" - errors = [] - - if not self.enabled: - errors.append("GRAPHITI_ENABLED must be set to true") - return errors - - # LLM provider validation - if self.llm_provider == "openai": - if not self.openai_api_key: - errors.append("OpenAI LLM provider requires OPENAI_API_KEY") - elif self.llm_provider == "anthropic": - if not self.anthropic_api_key: - errors.append("Anthropic LLM provider requires ANTHROPIC_API_KEY") - elif self.llm_provider == "azure_openai": - if not self.azure_openai_api_key: - errors.append("Azure OpenAI LLM provider requires AZURE_OPENAI_API_KEY") - if not self.azure_openai_base_url: - errors.append( - "Azure OpenAI LLM provider requires AZURE_OPENAI_BASE_URL" - ) - if not self.azure_openai_llm_deployment: - errors.append( - "Azure OpenAI LLM provider requires AZURE_OPENAI_LLM_DEPLOYMENT" - ) - elif self.llm_provider == "ollama": - if not self.ollama_llm_model: - errors.append("Ollama LLM provider requires OLLAMA_LLM_MODEL") - else: - errors.append(f"Unknown LLM provider: {self.llm_provider}") - - # Embedder provider validation - if self.embedder_provider == "openai": - if not self.openai_api_key: - errors.append("OpenAI embedder provider requires OPENAI_API_KEY") - elif self.embedder_provider == "voyage": - if not self.voyage_api_key: - errors.append("Voyage embedder provider requires VOYAGE_API_KEY") - elif self.embedder_provider == "azure_openai": - if not self.azure_openai_api_key: - errors.append( - "Azure OpenAI embedder provider requires AZURE_OPENAI_API_KEY" - ) - if not self.azure_openai_base_url: - errors.append( - "Azure OpenAI embedder provider requires AZURE_OPENAI_BASE_URL" - ) - if not self.azure_openai_embedding_deployment: - errors.append( - "Azure OpenAI embedder provider requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT" - ) - elif self.embedder_provider == "ollama": - if not self.ollama_embedding_model: - errors.append( - "Ollama embedder provider requires OLLAMA_EMBEDDING_MODEL" - ) - if not self.ollama_embedding_dim: - errors.append("Ollama embedder provider requires OLLAMA_EMBEDDING_DIM") - else: - errors.append(f"Unknown embedder provider: {self.embedder_provider}") - - return errors - - def get_connection_uri(self) -> str: - """Get the FalkorDB connection URI.""" - if self.falkordb_password: - return f"redis://:{self.falkordb_password}@{self.falkordb_host}:{self.falkordb_port}" - return f"redis://{self.falkordb_host}:{self.falkordb_port}" - - def get_provider_summary(self) -> str: - """Get a summary of configured providers.""" - return f"LLM: {self.llm_provider}, Embedder: {self.embedder_provider}" - - -@dataclass -class GraphitiState: - """State of Graphiti integration for an auto-claude spec.""" - - initialized: bool = False - database: str | None = None - indices_built: bool = False - created_at: str | None = None - last_session: int | None = None - episode_count: int = 0 - error_log: list = field(default_factory=list) - # V2 additions - llm_provider: str | None = None - embedder_provider: str | None = None - - def to_dict(self) -> dict: - return { - "initialized": self.initialized, - "database": self.database, - "indices_built": self.indices_built, - "created_at": self.created_at, - "last_session": self.last_session, - "episode_count": self.episode_count, - "error_log": self.error_log[-10:], # Keep last 10 errors - "llm_provider": self.llm_provider, - "embedder_provider": self.embedder_provider, - } - - @classmethod - def from_dict(cls, data: dict) -> "GraphitiState": - return cls( - initialized=data.get("initialized", False), - database=data.get("database"), - indices_built=data.get("indices_built", False), - created_at=data.get("created_at"), - last_session=data.get("last_session"), - episode_count=data.get("episode_count", 0), - error_log=data.get("error_log", []), - llm_provider=data.get("llm_provider"), - embedder_provider=data.get("embedder_provider"), - ) - - def save(self, spec_dir: Path) -> None: - """Save state to the spec directory.""" - marker_file = spec_dir / GRAPHITI_STATE_MARKER - with open(marker_file, "w") as f: - json.dump(self.to_dict(), f, indent=2) - - @classmethod - def load(cls, spec_dir: Path) -> Optional["GraphitiState"]: - """Load state from the spec directory.""" - marker_file = spec_dir / GRAPHITI_STATE_MARKER - if not marker_file.exists(): - return None - - try: - with open(marker_file) as f: - return cls.from_dict(json.load(f)) - except (OSError, json.JSONDecodeError): - return None - - def record_error(self, error_msg: str) -> None: - """Record an error in the state.""" - self.error_log.append( - { - "timestamp": datetime.now().isoformat(), - "error": error_msg[:500], # Limit error message length - } - ) - # Keep only last 10 errors - self.error_log = self.error_log[-10:] - - -def is_graphiti_enabled() -> bool: - """ - Quick check if Graphiti integration is available. - - Returns True if: - - GRAPHITI_ENABLED is set to true/1/yes - - Required provider credentials are configured - """ - config = GraphitiConfig.from_env() - return config.is_valid() - - -def get_graphiti_status() -> dict: - """ - Get the current Graphiti integration status. - - Returns: - Dict with status information: - - enabled: bool - - available: bool (has required dependencies) - - host: str - - port: int - - database: str - - llm_provider: str - - embedder_provider: str - - reason: str (why unavailable if not available) - - errors: list (validation errors if any) - """ - config = GraphitiConfig.from_env() - - status = { - "enabled": config.enabled, - "available": False, - "host": config.falkordb_host, - "port": config.falkordb_port, - "database": config.database, - "llm_provider": config.llm_provider, - "embedder_provider": config.embedder_provider, - "reason": "", - "errors": [], - } - - if not config.enabled: - status["reason"] = "GRAPHITI_ENABLED not set to true" - return status - - # Get validation errors - errors = config.get_validation_errors() - if errors: - status["errors"] = errors - status["reason"] = errors[0] # First error as primary reason - return status - - status["available"] = True - return status - - -def get_available_providers() -> dict: - """ - Get list of available providers based on current environment. - - Returns: - Dict with lists of available LLM and embedder providers - """ - config = GraphitiConfig.from_env() - - available_llm = [] - available_embedder = [] - - # Check OpenAI - if config.openai_api_key: - available_llm.append("openai") - available_embedder.append("openai") - - # Check Anthropic - if config.anthropic_api_key: - available_llm.append("anthropic") - - # Check Azure OpenAI - if config.azure_openai_api_key and config.azure_openai_base_url: - if config.azure_openai_llm_deployment: - available_llm.append("azure_openai") - if config.azure_openai_embedding_deployment: - available_embedder.append("azure_openai") - - # Check Voyage - if config.voyage_api_key: - available_embedder.append("voyage") - - # Check Ollama - if config.ollama_llm_model: - available_llm.append("ollama") - if config.ollama_embedding_model and config.ollama_embedding_dim: - available_embedder.append("ollama") - - return { - "llm_providers": available_llm, - "embedder_providers": available_embedder, - } +"""Backward compatibility shim - import from integrations.graphiti.config instead.""" +from integrations.graphiti.config import * diff --git a/auto-claude/graphiti_providers.py b/auto-claude/graphiti_providers.py index 87affdec..6ca2cdba 100644 --- a/auto-claude/graphiti_providers.py +++ b/auto-claude/graphiti_providers.py @@ -1,70 +1,2 @@ -""" -Graphiti Multi-Provider Entry Point -==================================== - -Main entry point for Graphiti provider functionality. -This module re-exports all functionality from the graphiti_providers package. - -The actual implementation has been refactored into a package structure: -- graphiti_providers/exceptions.py - Provider exceptions -- graphiti_providers/models.py - Embedding dimensions and constants -- graphiti_providers/llm_providers/ - LLM provider implementations -- graphiti_providers/embedder_providers/ - Embedder provider implementations -- graphiti_providers/cross_encoder.py - Cross-encoder/reranker -- graphiti_providers/validators.py - Validation and health checks -- graphiti_providers/utils.py - Utility functions -- graphiti_providers/factory.py - Factory functions - -For backward compatibility, this module re-exports all public APIs. - -Usage: - from graphiti_providers import create_llm_client, create_embedder - from graphiti_config import GraphitiConfig - - config = GraphitiConfig.from_env() - llm_client = create_llm_client(config) - embedder = create_embedder(config) -""" - -# Re-export all public APIs from the package -from graphiti_providers import ( - # Exceptions - ProviderError, - ProviderNotInstalled, - # Factory functions - create_llm_client, - create_embedder, - create_cross_encoder, - # Models - EMBEDDING_DIMENSIONS, - get_expected_embedding_dim, - # Validators - validate_embedding_config, - test_llm_connection, - test_embedder_connection, - test_ollama_connection, - # Utilities - is_graphiti_enabled, - get_graph_hints, -) - -__all__ = [ - # Exceptions - "ProviderError", - "ProviderNotInstalled", - # Factory functions - "create_llm_client", - "create_embedder", - "create_cross_encoder", - # Models - "EMBEDDING_DIMENSIONS", - "get_expected_embedding_dim", - # Validators - "validate_embedding_config", - "test_llm_connection", - "test_embedder_connection", - "test_ollama_connection", - # Utilities - "is_graphiti_enabled", - "get_graph_hints", -] +"""Backward compatibility shim - import from integrations.graphiti.providers_pkg instead.""" +from integrations.graphiti.providers_pkg import * diff --git a/auto-claude/graphiti_providers.py.bak b/auto-claude/graphiti_providers.py.bak deleted file mode 100644 index f97a166e..00000000 --- a/auto-claude/graphiti_providers.py.bak +++ /dev/null @@ -1,705 +0,0 @@ -""" -Graphiti Multi-Provider Factory -================================ - -Factory functions for creating LLM clients and embedders for Graphiti. -Supports multiple providers: OpenAI, Anthropic, Azure OpenAI, and Ollama. - -This module provides: -- Lazy imports to avoid ImportError when provider packages not installed -- Factory functions that create the correct client based on provider selection -- Provider-specific configuration validation -- Graceful error handling with helpful messages - -Usage: - from graphiti_providers import create_llm_client, create_embedder - from graphiti_config import GraphitiConfig - - config = GraphitiConfig.from_env() - llm_client = create_llm_client(config) - embedder = create_embedder(config) -""" - -import logging -from typing import TYPE_CHECKING, Any, Optional - -if TYPE_CHECKING: - from pathlib import Path - - from graphiti_config import GraphitiConfig - -logger = logging.getLogger(__name__) - - -class ProviderError(Exception): - """Raised when a provider cannot be initialized.""" - - pass - - -class ProviderNotInstalled(ProviderError): - """Raised when required packages for a provider are not installed.""" - - pass - - -# ============================================================================ -# LLM Client Factory -# ============================================================================ - - -def create_llm_client(config: "GraphitiConfig") -> Any: - """ - Create an LLM client based on the configured provider. - - Args: - config: GraphitiConfig with provider settings - - Returns: - LLM client instance for Graphiti - - Raises: - ProviderNotInstalled: If required packages are missing - ProviderError: If client creation fails - """ - provider = config.llm_provider - - logger.info(f"Creating LLM client for provider: {provider}") - - if provider == "openai": - return _create_openai_llm_client(config) - elif provider == "anthropic": - return _create_anthropic_llm_client(config) - elif provider == "azure_openai": - return _create_azure_openai_llm_client(config) - elif provider == "ollama": - return _create_ollama_llm_client(config) - else: - raise ProviderError(f"Unknown LLM provider: {provider}") - - -def _create_openai_llm_client(config: "GraphitiConfig") -> Any: - """Create OpenAI LLM client.""" - try: - from graphiti_core.llm_client.config import LLMConfig - from graphiti_core.llm_client.openai_client import OpenAIClient - except ImportError as e: - raise ProviderNotInstalled( - f"OpenAI provider requires graphiti-core. " - f"Install with: pip install graphiti-core\n" - f"Error: {e}" - ) - - if not config.openai_api_key: - raise ProviderError("OpenAI provider requires OPENAI_API_KEY") - - llm_config = LLMConfig( - api_key=config.openai_api_key, - model=config.openai_model, - ) - - # GPT-5 family and o1/o3 models support reasoning/verbosity params - model_lower = config.openai_model.lower() - supports_reasoning = ( - model_lower.startswith("gpt-5") - or model_lower.startswith("o1") - or model_lower.startswith("o3") - ) - - if supports_reasoning: - # Use defaults for models that support reasoning params - return OpenAIClient(config=llm_config) - else: - # Disable reasoning/verbosity for older models that don't support them - return OpenAIClient(config=llm_config, reasoning=None, verbosity=None) - - -def _create_anthropic_llm_client(config: "GraphitiConfig") -> Any: - """Create Anthropic LLM client.""" - try: - from graphiti_core.llm_client.anthropic_client import AnthropicClient - from graphiti_core.llm_client.config import LLMConfig - except ImportError as e: - raise ProviderNotInstalled( - f"Anthropic provider requires graphiti-core[anthropic]. " - f"Install with: pip install graphiti-core[anthropic]\n" - f"Error: {e}" - ) - - if not config.anthropic_api_key: - raise ProviderError("Anthropic provider requires ANTHROPIC_API_KEY") - - llm_config = LLMConfig( - api_key=config.anthropic_api_key, - model=config.anthropic_model, - ) - - return AnthropicClient(config=llm_config) - - -def _create_azure_openai_llm_client(config: "GraphitiConfig") -> Any: - """Create Azure OpenAI LLM client.""" - try: - from graphiti_core.llm_client.azure_openai_client import AzureOpenAILLMClient - from graphiti_core.llm_client.config import LLMConfig - from openai import AsyncOpenAI - except ImportError as e: - raise ProviderNotInstalled( - f"Azure OpenAI provider requires graphiti-core and openai. " - f"Install with: pip install graphiti-core openai\n" - f"Error: {e}" - ) - - if not config.azure_openai_api_key: - raise ProviderError("Azure OpenAI provider requires AZURE_OPENAI_API_KEY") - if not config.azure_openai_base_url: - raise ProviderError("Azure OpenAI provider requires AZURE_OPENAI_BASE_URL") - if not config.azure_openai_llm_deployment: - raise ProviderError( - "Azure OpenAI provider requires AZURE_OPENAI_LLM_DEPLOYMENT" - ) - - azure_client = AsyncOpenAI( - base_url=config.azure_openai_base_url, - api_key=config.azure_openai_api_key, - ) - - llm_config = LLMConfig( - model=config.azure_openai_llm_deployment, - small_model=config.azure_openai_llm_deployment, - ) - - return AzureOpenAILLMClient(azure_client=azure_client, config=llm_config) - - -def _create_ollama_llm_client(config: "GraphitiConfig") -> Any: - """Create Ollama LLM client (using OpenAI-compatible interface).""" - try: - from graphiti_core.llm_client.config import LLMConfig - from graphiti_core.llm_client.openai_generic_client import OpenAIGenericClient - except ImportError as e: - raise ProviderNotInstalled( - f"Ollama provider requires graphiti-core. " - f"Install with: pip install graphiti-core\n" - f"Error: {e}" - ) - - if not config.ollama_llm_model: - raise ProviderError("Ollama provider requires OLLAMA_LLM_MODEL") - - # Ensure Ollama base URL ends with /v1 for OpenAI compatibility - base_url = config.ollama_base_url - if not base_url.endswith("/v1"): - base_url = base_url.rstrip("/") + "/v1" - - llm_config = LLMConfig( - api_key="ollama", # Ollama requires a dummy API key - model=config.ollama_llm_model, - small_model=config.ollama_llm_model, - base_url=base_url, - ) - - return OpenAIGenericClient(config=llm_config) - - -# ============================================================================ -# Embedder Factory -# ============================================================================ - - -def create_embedder(config: "GraphitiConfig") -> Any: - """ - Create an embedder based on the configured provider. - - Args: - config: GraphitiConfig with provider settings - - Returns: - Embedder instance for Graphiti - - Raises: - ProviderNotInstalled: If required packages are missing - ProviderError: If embedder creation fails - """ - provider = config.embedder_provider - - logger.info(f"Creating embedder for provider: {provider}") - - if provider == "openai": - return _create_openai_embedder(config) - elif provider == "voyage": - return _create_voyage_embedder(config) - elif provider == "azure_openai": - return _create_azure_openai_embedder(config) - elif provider == "ollama": - return _create_ollama_embedder(config) - else: - raise ProviderError(f"Unknown embedder provider: {provider}") - - -def _create_openai_embedder(config: "GraphitiConfig") -> Any: - """Create OpenAI embedder.""" - try: - from graphiti_core.embedder.openai import OpenAIEmbedder, OpenAIEmbedderConfig - except ImportError as e: - raise ProviderNotInstalled( - f"OpenAI embedder requires graphiti-core. " - f"Install with: pip install graphiti-core\n" - f"Error: {e}" - ) - - if not config.openai_api_key: - raise ProviderError("OpenAI embedder requires OPENAI_API_KEY") - - embedder_config = OpenAIEmbedderConfig( - api_key=config.openai_api_key, - embedding_model=config.openai_embedding_model, - ) - - return OpenAIEmbedder(config=embedder_config) - - -def _create_voyage_embedder(config: "GraphitiConfig") -> Any: - """Create Voyage AI embedder (commonly used with Anthropic LLM).""" - try: - from graphiti_core.embedder.voyage import VoyageAIConfig, VoyageEmbedder - except ImportError as e: - raise ProviderNotInstalled( - f"Voyage embedder requires graphiti-core[voyage]. " - f"Install with: pip install graphiti-core[voyage]\n" - f"Error: {e}" - ) - - if not config.voyage_api_key: - raise ProviderError("Voyage embedder requires VOYAGE_API_KEY") - - voyage_config = VoyageAIConfig( - api_key=config.voyage_api_key, - embedding_model=config.voyage_embedding_model, - ) - - return VoyageEmbedder(config=voyage_config) - - -def _create_azure_openai_embedder(config: "GraphitiConfig") -> Any: - """Create Azure OpenAI embedder.""" - try: - from graphiti_core.embedder.azure_openai import AzureOpenAIEmbedderClient - from openai import AsyncOpenAI - except ImportError as e: - raise ProviderNotInstalled( - f"Azure OpenAI embedder requires graphiti-core and openai. " - f"Install with: pip install graphiti-core openai\n" - f"Error: {e}" - ) - - if not config.azure_openai_api_key: - raise ProviderError("Azure OpenAI embedder requires AZURE_OPENAI_API_KEY") - if not config.azure_openai_base_url: - raise ProviderError("Azure OpenAI embedder requires AZURE_OPENAI_BASE_URL") - if not config.azure_openai_embedding_deployment: - raise ProviderError( - "Azure OpenAI embedder requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT" - ) - - azure_client = AsyncOpenAI( - base_url=config.azure_openai_base_url, - api_key=config.azure_openai_api_key, - ) - - return AzureOpenAIEmbedderClient( - azure_client=azure_client, - model=config.azure_openai_embedding_deployment, - ) - - -def _create_ollama_embedder(config: "GraphitiConfig") -> Any: - """Create Ollama embedder (using OpenAI-compatible interface).""" - try: - from graphiti_core.embedder.openai import OpenAIEmbedder, OpenAIEmbedderConfig - except ImportError as e: - raise ProviderNotInstalled( - f"Ollama embedder requires graphiti-core. " - f"Install with: pip install graphiti-core\n" - f"Error: {e}" - ) - - if not config.ollama_embedding_model: - raise ProviderError("Ollama embedder requires OLLAMA_EMBEDDING_MODEL") - - # Ensure Ollama base URL ends with /v1 for OpenAI compatibility - base_url = config.ollama_base_url - if not base_url.endswith("/v1"): - base_url = base_url.rstrip("/") + "/v1" - - embedder_config = OpenAIEmbedderConfig( - api_key="ollama", # Ollama requires a dummy API key - embedding_model=config.ollama_embedding_model, - embedding_dim=config.ollama_embedding_dim, - base_url=base_url, - ) - - return OpenAIEmbedder(config=embedder_config) - - -# ============================================================================ -# Cross-Encoder / Reranker Factory (Optional) -# ============================================================================ - - -def create_cross_encoder( - config: "GraphitiConfig", llm_client: Any = None -) -> Any | None: - """ - Create a cross-encoder/reranker for improved search quality. - - This is optional and primarily useful for Ollama setups. - Other providers typically have built-in reranking. - - Args: - config: GraphitiConfig with provider settings - llm_client: Optional LLM client for reranking - - Returns: - Cross-encoder instance, or None if not applicable - """ - # Only create for Ollama provider currently - if config.llm_provider != "ollama": - return None - - if llm_client is None: - return None - - try: - from graphiti_core.cross_encoder.openai_reranker_client import ( - OpenAIRerankerClient, - ) - from graphiti_core.llm_client.config import LLMConfig - except ImportError: - logger.debug("Cross-encoder not available (optional)") - return None - - try: - # Create LLM config for reranker - base_url = config.ollama_base_url - if not base_url.endswith("/v1"): - base_url = base_url.rstrip("/") + "/v1" - - llm_config = LLMConfig( - api_key="ollama", - model=config.ollama_llm_model, - base_url=base_url, - ) - - return OpenAIRerankerClient(client=llm_client, config=llm_config) - except Exception as e: - logger.warning(f"Could not create cross-encoder: {e}") - return None - - -# ============================================================================ -# Embedding Dimension Validation -# ============================================================================ - -# Known embedding dimensions by provider and model -EMBEDDING_DIMENSIONS = { - # OpenAI - "text-embedding-3-small": 1536, - "text-embedding-3-large": 3072, - "text-embedding-ada-002": 1536, - # Voyage AI - "voyage-3": 1024, - "voyage-3.5": 1024, - "voyage-3-lite": 512, - "voyage-3.5-lite": 512, - "voyage-2": 1024, - "voyage-large-2": 1536, - # Ollama (common models) - "nomic-embed-text": 768, - "mxbai-embed-large": 1024, - "all-minilm": 384, - "snowflake-arctic-embed": 1024, -} - - -def get_expected_embedding_dim(model: str) -> int | None: - """ - Get the expected embedding dimension for a known model. - - Args: - model: Embedding model name - - Returns: - Expected dimension, or None if unknown - """ - # Try exact match first - if model in EMBEDDING_DIMENSIONS: - return EMBEDDING_DIMENSIONS[model] - - # Try partial match (model name might have version suffix) - model_lower = model.lower() - for known_model, dim in EMBEDDING_DIMENSIONS.items(): - if known_model.lower() in model_lower or model_lower in known_model.lower(): - return dim - - return None - - -def validate_embedding_config(config: "GraphitiConfig") -> tuple[bool, str]: - """ - Validate embedding configuration for consistency. - - Checks that embedding dimensions are correctly configured, - especially important for Ollama where explicit dimension is required. - - Args: - config: GraphitiConfig to validate - - Returns: - Tuple of (is_valid, message) - """ - provider = config.embedder_provider - - if provider == "ollama": - # Ollama requires explicit embedding dimension - if not config.ollama_embedding_dim: - expected = get_expected_embedding_dim(config.ollama_embedding_model) - if expected: - return False, ( - f"Ollama embedder requires OLLAMA_EMBEDDING_DIM. " - f"For model '{config.ollama_embedding_model}', " - f"expected dimension is {expected}." - ) - else: - return False, ( - "Ollama embedder requires OLLAMA_EMBEDDING_DIM. " - "Check your model's documentation for the correct dimension." - ) - - # Check for known dimension mismatches - if provider == "openai": - expected = get_expected_embedding_dim(config.openai_embedding_model) - # OpenAI handles this automatically, just log info - if expected: - logger.debug( - f"OpenAI embedding model '{config.openai_embedding_model}' has dimension {expected}" - ) - - elif provider == "voyage": - expected = get_expected_embedding_dim(config.voyage_embedding_model) - if expected: - logger.debug( - f"Voyage embedding model '{config.voyage_embedding_model}' has dimension {expected}" - ) - - return True, "Embedding configuration valid" - - -# ============================================================================ -# Provider Health Checks -# ============================================================================ - - -async def test_llm_connection(config: "GraphitiConfig") -> tuple[bool, str]: - """ - Test if LLM provider is reachable. - - Args: - config: GraphitiConfig with provider settings - - Returns: - Tuple of (success, message) - """ - try: - llm_client = create_llm_client(config) - # Most clients don't have a ping method, so just verify creation succeeded - return ( - True, - f"LLM client created successfully for provider: {config.llm_provider}", - ) - except ProviderNotInstalled as e: - return False, str(e) - except ProviderError as e: - return False, str(e) - except Exception as e: - return False, f"Failed to create LLM client: {e}" - - -async def test_embedder_connection(config: "GraphitiConfig") -> tuple[bool, str]: - """ - Test if embedder provider is reachable. - - Args: - config: GraphitiConfig with provider settings - - Returns: - Tuple of (success, message) - """ - # First validate config - valid, msg = validate_embedding_config(config) - if not valid: - return False, msg - - try: - embedder = create_embedder(config) - return ( - True, - f"Embedder created successfully for provider: {config.embedder_provider}", - ) - except ProviderNotInstalled as e: - return False, str(e) - except ProviderError as e: - return False, str(e) - except Exception as e: - return False, f"Failed to create embedder: {e}" - - -async def test_ollama_connection( - base_url: str = "http://localhost:11434", -) -> tuple[bool, str]: - """ - Test if Ollama server is running and reachable. - - Args: - base_url: Ollama server URL - - Returns: - Tuple of (success, message) - """ - import asyncio - - try: - import aiohttp - except ImportError: - # Fall back to sync request - import urllib.error - import urllib.request - - try: - # Normalize URL (remove /v1 suffix if present) - url = base_url.rstrip("/") - if url.endswith("/v1"): - url = url[:-3] - - req = urllib.request.Request(f"{url}/api/tags", method="GET") - with urllib.request.urlopen(req, timeout=5) as response: - if response.status == 200: - return True, f"Ollama is running at {url}" - return False, f"Ollama returned status {response.status}" - except urllib.error.URLError as e: - return False, f"Cannot connect to Ollama at {url}: {e.reason}" - except Exception as e: - return False, f"Ollama connection error: {e}" - - # Use aiohttp if available - try: - # Normalize URL - url = base_url.rstrip("/") - if url.endswith("/v1"): - url = url[:-3] - - async with aiohttp.ClientSession() as session: - async with session.get( - f"{url}/api/tags", timeout=aiohttp.ClientTimeout(total=5) - ) as response: - if response.status == 200: - return True, f"Ollama is running at {url}" - return False, f"Ollama returned status {response.status}" - except asyncio.TimeoutError: - return False, f"Ollama connection timed out at {url}" - except aiohttp.ClientError as e: - return False, f"Cannot connect to Ollama at {url}: {e}" - except Exception as e: - return False, f"Ollama connection error: {e}" - - -# ============================================================================ -# Re-exports and Convenience Functions -# ============================================================================ - - -def is_graphiti_enabled() -> bool: - """ - Check if Graphiti memory integration is available and configured. - - This is a convenience re-export from graphiti_config. - Returns True if GRAPHITI_ENABLED=true and provider credentials are valid. - """ - from graphiti_config import is_graphiti_enabled as _is_graphiti_enabled - - return _is_graphiti_enabled() - - -async def get_graph_hints( - query: str, - project_id: str, - max_results: int = 10, - spec_dir: Optional["Path"] = None, -) -> list[dict]: - """ - Get relevant hints from the Graphiti knowledge graph. - - This is a convenience function for querying historical context - from the memory system. Used by spec_runner, ideation_runner, - and roadmap_runner to inject historical insights. - - Args: - query: Search query (e.g., "authentication patterns", "API design") - project_id: Project identifier for scoping results - max_results: Maximum number of hints to return - spec_dir: Optional spec directory for loading memory instance - - Returns: - List of hint dictionaries with keys: - - content: str - The hint content - - score: float - Relevance score - - type: str - Type of hint (pattern, gotcha, outcome, etc.) - - Note: - Returns empty list if Graphiti is not enabled or unavailable. - This function never raises - it always fails gracefully. - """ - if not is_graphiti_enabled(): - logger.debug("Graphiti not enabled, returning empty hints") - return [] - - try: - from pathlib import Path - - from graphiti_memory import GraphitiMemory, GroupIdMode - - # Determine project directory from project_id or use current dir - project_dir = Path.cwd() - - # Use spec_dir if provided, otherwise create a temp context - if spec_dir is None: - # Create a temporary spec dir for the query - import tempfile - - spec_dir = Path(tempfile.mkdtemp(prefix="graphiti_query_")) - - # Create memory instance with project-level scope for cross-spec hints - memory = GraphitiMemory( - spec_dir=spec_dir, - project_dir=project_dir, - group_id_mode=GroupIdMode.PROJECT, - ) - - # Query for relevant context - hints = await memory.get_relevant_context( - query=query, - num_results=max_results, - include_project_context=True, - ) - - await memory.close() - - logger.info(f"Retrieved {len(hints)} graph hints for query: {query[:50]}...") - return hints - - except ImportError as e: - logger.debug(f"Graphiti packages not available: {e}") - return [] - except Exception as e: - logger.warning(f"Failed to get graph hints: {e}") - return [] diff --git a/auto-claude/ideation_runner_old.py.bak b/auto-claude/ideation_runner_old.py.bak deleted file mode 100644 index 56eb47c7..00000000 --- a/auto-claude/ideation_runner_old.py.bak +++ /dev/null @@ -1,1122 +0,0 @@ -#!/usr/bin/env python3 -""" -Ideation Creation Orchestrator -============================== - -AI-powered ideation generation for projects. -Analyzes project context, existing features, and generates three types of ideas: -1. Low-Hanging Fruit - Quick wins building on existing patterns -2. UI/UX Improvements - Visual and interaction enhancements -3. High-Value Features - Strategic features for target users - -Usage: - python auto-claude/ideation_runner.py --project /path/to/project - python auto-claude/ideation_runner.py --project /path/to/project --types low_hanging_fruit,high_value_features - python auto-claude/ideation_runner.py --project /path/to/project --refresh -""" - -import asyncio -import json -import os -import subprocess -import sys -from dataclasses import dataclass, field -from datetime import datetime -from pathlib import Path -from typing import Optional, List - -# Add auto-claude to path -sys.path.insert(0, str(Path(__file__).parent)) - -# Load .env file -from dotenv import load_dotenv -env_file = Path(__file__).parent / ".env" -if env_file.exists(): - load_dotenv(env_file) - -from client import create_client -from ui import ( - Icons, - icon, - box, - success, - error, - warning, - info, - muted, - highlight, - bold, - print_status, - print_key_value, - print_section, -) -from debug import ( - debug, - debug_detailed, - debug_verbose, - debug_success, - debug_error, - debug_warning, - debug_section, -) -from graphiti_providers import get_graph_hints, is_graphiti_enabled -from init import init_auto_claude_dir - - -# Configuration -MAX_RETRIES = 3 -PROMPTS_DIR = Path(__file__).parent / "prompts" - -# Ideation types -# Note: high_value_features removed - strategic features belong to Roadmap -# low_hanging_fruit renamed to code_improvements to cover all code-revealed opportunities -IDEATION_TYPES = [ - "code_improvements", - "ui_ux_improvements", - "documentation_gaps", - "security_hardening", - "performance_optimizations", - "code_quality", -] - -IDEATION_TYPE_LABELS = { - "code_improvements": "Code Improvements", - "ui_ux_improvements": "UI/UX Improvements", - "documentation_gaps": "Documentation Gaps", - "security_hardening": "Security Hardening", - "performance_optimizations": "Performance Optimizations", - "code_quality": "Code Quality & Refactoring", -} - -IDEATION_TYPE_PROMPTS = { - "code_improvements": "ideation_code_improvements.md", - "ui_ux_improvements": "ideation_ui_ux.md", - "documentation_gaps": "ideation_documentation.md", - "security_hardening": "ideation_security.md", - "performance_optimizations": "ideation_performance.md", - "code_quality": "ideation_code_quality.md", -} - - -@dataclass -class IdeationPhaseResult: - """Result of an ideation phase execution.""" - phase: str - ideation_type: Optional[str] - success: bool - output_files: list[str] - ideas_count: int - errors: list[str] - retries: int - - -@dataclass -class IdeationConfig: - """Configuration for ideation generation.""" - project_dir: Path - output_dir: Path - enabled_types: List[str] = field(default_factory=lambda: IDEATION_TYPES.copy()) - include_roadmap_context: bool = True - include_kanban_context: bool = True - max_ideas_per_type: int = 5 - model: str = "claude-opus-4-5-20251101" - refresh: bool = False - append: bool = False # If True, preserve existing ideas when merging - - -class IdeationOrchestrator: - """Orchestrates the ideation creation process.""" - - def __init__( - self, - project_dir: Path, - output_dir: Optional[Path] = None, - enabled_types: Optional[List[str]] = None, - include_roadmap_context: bool = True, - include_kanban_context: bool = True, - max_ideas_per_type: int = 5, - model: str = "claude-opus-4-5-20251101", - refresh: bool = False, - append: bool = False, - ): - self.project_dir = Path(project_dir) - self.model = model - self.refresh = refresh - self.append = append # Preserve existing ideas when merging - self.enabled_types = enabled_types or IDEATION_TYPES.copy() - self.include_roadmap_context = include_roadmap_context - self.include_kanban_context = include_kanban_context - self.max_ideas_per_type = max_ideas_per_type - - # Default output to project's .auto-claude directory (installed instance) - # Note: auto-claude/ is source code, .auto-claude/ is the installed instance - if output_dir: - self.output_dir = Path(output_dir) - else: - # Initialize .auto-claude directory and ensure it's in .gitignore - init_auto_claude_dir(self.project_dir) - self.output_dir = self.project_dir / ".auto-claude" / "ideation" - - self.output_dir.mkdir(parents=True, exist_ok=True) - - # Create screenshots directory for UI/UX analysis - (self.output_dir / "screenshots").mkdir(exist_ok=True) - - def _run_script(self, script: str, args: list[str]) -> tuple[bool, str]: - """Run a Python script and return (success, output).""" - script_path = Path(__file__).parent / script - - if not script_path.exists(): - return False, f"Script not found: {script_path}" - - cmd = [sys.executable, str(script_path)] + args - - try: - result = subprocess.run( - cmd, - cwd=self.project_dir, - capture_output=True, - text=True, - timeout=300, - ) - - if result.returncode == 0: - return True, result.stdout - else: - return False, result.stderr or result.stdout - - except subprocess.TimeoutExpired: - return False, "Script timed out" - except Exception as e: - return False, str(e) - - async def _run_agent( - self, - prompt_file: str, - additional_context: str = "", - ) -> tuple[bool, str]: - """Run an agent with the given prompt.""" - prompt_path = PROMPTS_DIR / prompt_file - - if not prompt_path.exists(): - return False, f"Prompt not found: {prompt_path}" - - # Load prompt - prompt = prompt_path.read_text() - - # Add context - prompt += f"\n\n---\n\n**Output Directory**: {self.output_dir}\n" - prompt += f"**Project Directory**: {self.project_dir}\n" - prompt += f"**Max Ideas**: {self.max_ideas_per_type}\n" - - if additional_context: - prompt += f"\n{additional_context}\n" - - # Create client - client = create_client(self.project_dir, self.output_dir, self.model) - - try: - async with client: - await client.query(prompt) - - response_text = "" - async for msg in client.receive_response(): - msg_type = type(msg).__name__ - - if msg_type == "AssistantMessage" and hasattr(msg, "content"): - for block in msg.content: - block_type = type(block).__name__ - if block_type == "TextBlock" and hasattr(block, "text"): - response_text += block.text - print(block.text, end="", flush=True) - elif block_type == "ToolUseBlock" and hasattr(block, "name"): - print(f"\n[Tool: {block.name}]", flush=True) - - print() - return True, response_text - - except Exception as e: - return False, str(e) - - async def _get_graph_hints(self, ideation_type: str) -> list[dict]: - """Get graph hints for a specific ideation type from Graphiti. - - This runs in parallel with ideation agents to provide historical context. - """ - if not is_graphiti_enabled(): - return [] - - # Create a query based on ideation type - query_map = { - "code_improvements": "code patterns, quick wins, and improvement opportunities that worked well", - "ui_ux_improvements": "UI and UX improvements and user interface patterns", - "documentation_gaps": "documentation improvements and common user confusion points", - "security_hardening": "security vulnerabilities and hardening measures", - "performance_optimizations": "performance bottlenecks and optimization techniques", - "code_quality": "code quality improvements and refactoring patterns", - } - - query = query_map.get(ideation_type, f"ideas for {ideation_type}") - - try: - hints = await get_graph_hints( - query=query, - project_id=str(self.project_dir), - max_results=5, - ) - debug_success("ideation_runner", f"Got {len(hints)} graph hints for {ideation_type}") - return hints - except Exception as e: - debug_warning("ideation_runner", f"Graph hints failed for {ideation_type}: {e}") - return [] - - def _gather_context(self) -> dict: - """Gather context from project for ideation.""" - context = { - "existing_features": [], - "tech_stack": [], - "target_audience": None, - "planned_features": [], - } - - # Get project index (from .auto-claude - the installed instance) - project_index_path = self.project_dir / ".auto-claude" / "project_index.json" - if project_index_path.exists(): - try: - with open(project_index_path) as f: - index = json.load(f) - # Extract tech stack from services - for service_name, service_info in index.get("services", {}).items(): - if service_info.get("language"): - context["tech_stack"].append(service_info["language"]) - if service_info.get("framework"): - context["tech_stack"].append(service_info["framework"]) - context["tech_stack"] = list(set(context["tech_stack"])) - except (json.JSONDecodeError, KeyError): - pass - - # Get roadmap context if enabled - if self.include_roadmap_context: - roadmap_path = self.project_dir / ".auto-claude" / "roadmap" / "roadmap.json" - if roadmap_path.exists(): - try: - with open(roadmap_path) as f: - roadmap = json.load(f) - # Extract planned features - for feature in roadmap.get("features", []): - context["planned_features"].append(feature.get("title", "")) - # Get target audience - audience = roadmap.get("target_audience", {}) - context["target_audience"] = audience.get("primary") - except (json.JSONDecodeError, KeyError): - pass - - # Also check discovery for audience - discovery_path = self.project_dir / ".auto-claude" / "roadmap" / "roadmap_discovery.json" - if discovery_path.exists() and not context["target_audience"]: - try: - with open(discovery_path) as f: - discovery = json.load(f) - audience = discovery.get("target_audience", {}) - context["target_audience"] = audience.get("primary_persona") - - # Also get existing features - current_state = discovery.get("current_state", {}) - context["existing_features"] = current_state.get("existing_features", []) - except (json.JSONDecodeError, KeyError): - pass - - # Get kanban context if enabled - if self.include_kanban_context: - specs_dir = self.project_dir / ".auto-claude" / "specs" - if specs_dir.exists(): - for spec_dir in specs_dir.iterdir(): - if spec_dir.is_dir(): - spec_file = spec_dir / "spec.md" - if spec_file.exists(): - # Extract title from spec - content = spec_file.read_text() - lines = content.split("\n") - for line in lines: - if line.startswith("# "): - context["planned_features"].append(line[2:].strip()) - break - - # Remove duplicates from planned features - context["planned_features"] = list(set(context["planned_features"])) - - return context - - async def phase_graph_hints(self) -> IdeationPhaseResult: - """Retrieve graph hints for all enabled ideation types in parallel. - - This phase runs concurrently with context gathering to fetch - historical insights from Graphiti without slowing down the pipeline. - """ - hints_file = self.output_dir / "graph_hints.json" - - if hints_file.exists(): - print_status("graph_hints.json already exists", "success") - return IdeationPhaseResult( - phase="graph_hints", - ideation_type=None, - success=True, - output_files=[str(hints_file)], - ideas_count=0, - errors=[], - retries=0, - ) - - if not is_graphiti_enabled(): - print_status("Graphiti not enabled, skipping graph hints", "info") - with open(hints_file, "w") as f: - json.dump({ - "enabled": False, - "reason": "Graphiti not configured", - "hints_by_type": {}, - "created_at": datetime.now().isoformat(), - }, f, indent=2) - return IdeationPhaseResult( - phase="graph_hints", - ideation_type=None, - success=True, - output_files=[str(hints_file)], - ideas_count=0, - errors=[], - retries=0, - ) - - print_status("Querying Graphiti for ideation hints...", "progress") - - # Fetch hints for all enabled types in parallel - hint_tasks = [ - self._get_graph_hints(ideation_type) - for ideation_type in self.enabled_types - ] - - results = await asyncio.gather(*hint_tasks, return_exceptions=True) - - # Collect hints by type - hints_by_type = {} - total_hints = 0 - errors = [] - - for i, result in enumerate(results): - ideation_type = self.enabled_types[i] - if isinstance(result, Exception): - errors.append(f"{ideation_type}: {str(result)}") - hints_by_type[ideation_type] = [] - else: - hints_by_type[ideation_type] = result - total_hints += len(result) - - # Save hints - with open(hints_file, "w") as f: - json.dump({ - "enabled": True, - "hints_by_type": hints_by_type, - "total_hints": total_hints, - "created_at": datetime.now().isoformat(), - }, f, indent=2) - - if total_hints > 0: - print_status(f"Retrieved {total_hints} graph hints across {len(self.enabled_types)} types", "success") - else: - print_status("No relevant graph hints found", "info") - - return IdeationPhaseResult( - phase="graph_hints", - ideation_type=None, - success=True, - output_files=[str(hints_file)], - ideas_count=0, - errors=errors, - retries=0, - ) - - async def phase_context(self) -> IdeationPhaseResult: - """Create ideation context file.""" - - context_file = self.output_dir / "ideation_context.json" - - print_status("Gathering project context...", "progress") - - context = self._gather_context() - - # Check for graph hints and include them - hints_file = self.output_dir / "graph_hints.json" - graph_hints = {} - if hints_file.exists(): - try: - with open(hints_file) as f: - hints_data = json.load(f) - graph_hints = hints_data.get("hints_by_type", {}) - except (json.JSONDecodeError, IOError): - pass - - # Write context file - context_data = { - "existing_features": context["existing_features"], - "tech_stack": context["tech_stack"], - "target_audience": context["target_audience"], - "planned_features": context["planned_features"], - "graph_hints": graph_hints, # Include graph hints in context - "config": { - "enabled_types": self.enabled_types, - "include_roadmap_context": self.include_roadmap_context, - "include_kanban_context": self.include_kanban_context, - "max_ideas_per_type": self.max_ideas_per_type, - }, - "created_at": datetime.now().isoformat(), - } - - with open(context_file, "w") as f: - json.dump(context_data, f, indent=2) - - print_status(f"Created ideation_context.json", "success") - print_key_value("Tech Stack", ", ".join(context["tech_stack"][:5]) or "Unknown") - print_key_value("Planned Features", str(len(context["planned_features"]))) - print_key_value("Target Audience", context["target_audience"] or "Not specified") - if graph_hints: - total_hints = sum(len(h) for h in graph_hints.values()) - print_key_value("Graph Hints", str(total_hints)) - - return IdeationPhaseResult( - phase="context", - ideation_type=None, - success=True, - output_files=[str(context_file)], - ideas_count=0, - errors=[], - retries=0, - ) - - async def phase_project_index(self) -> IdeationPhaseResult: - """Ensure project index exists.""" - - project_index = self.output_dir / "project_index.json" - auto_build_index = self.project_dir / ".auto-claude" / "project_index.json" - - # Check if we can copy existing index - if auto_build_index.exists(): - import shutil - shutil.copy(auto_build_index, project_index) - print_status("Copied existing project_index.json", "success") - return IdeationPhaseResult("project_index", None, True, [str(project_index)], 0, [], 0) - - if project_index.exists() and not self.refresh: - print_status("project_index.json already exists", "success") - return IdeationPhaseResult("project_index", None, True, [str(project_index)], 0, [], 0) - - # Run analyzer - print_status("Running project analyzer...", "progress") - success, output = self._run_script( - "analyzer.py", - ["--output", str(project_index)] - ) - - if success and project_index.exists(): - print_status("Created project_index.json", "success") - return IdeationPhaseResult("project_index", None, True, [str(project_index)], 0, [], 0) - - return IdeationPhaseResult("project_index", None, False, [], 0, [output], 1) - - async def phase_ideation_type(self, ideation_type: str) -> IdeationPhaseResult: - """Run ideation for a specific type.""" - - prompt_file = IDEATION_TYPE_PROMPTS.get(ideation_type) - if not prompt_file: - return IdeationPhaseResult( - phase="ideation", - ideation_type=ideation_type, - success=False, - output_files=[], - ideas_count=0, - errors=[f"Unknown ideation type: {ideation_type}"], - retries=0, - ) - - output_file = self.output_dir / f"{ideation_type}_ideas.json" - - if output_file.exists() and not self.refresh: - # Load and validate existing ideas - only skip if we have valid ideas - try: - with open(output_file) as f: - data = json.load(f) - count = len(data.get(ideation_type, [])) - - if count >= 1: - # Valid ideas exist, skip regeneration - print_status(f"{ideation_type}_ideas.json already exists ({count} ideas)", "success") - return IdeationPhaseResult( - phase="ideation", - ideation_type=ideation_type, - success=True, - output_files=[str(output_file)], - ideas_count=count, - errors=[], - retries=0, - ) - else: - # File exists but has no valid ideas - needs regeneration - print_status(f"{ideation_type}_ideas.json exists but has 0 ideas, regenerating...", "warning") - except (json.JSONDecodeError, KeyError): - # Invalid file - will regenerate - print_status(f"{ideation_type}_ideas.json exists but is invalid, regenerating...", "warning") - - errors = [] - - # First attempt: run the full ideation agent - print_status(f"Running {IDEATION_TYPE_LABELS[ideation_type]} agent...", "progress") - - context = f""" -**Ideation Context**: {self.output_dir / "ideation_context.json"} -**Project Index**: {self.output_dir / "project_index.json"} -**Output File**: {output_file} -**Max Ideas**: {self.max_ideas_per_type} - -Generate up to {self.max_ideas_per_type} {IDEATION_TYPE_LABELS[ideation_type]} ideas. -Avoid duplicating features that are already planned (see ideation_context.json). -Output your ideas to {output_file.name}. -""" - success, output = await self._run_agent( - prompt_file, - additional_context=context, - ) - - # Validate the output - validation_result = self._validate_ideation_output(output_file, ideation_type) - - if validation_result["success"]: - print_status(f"Created {output_file.name} ({validation_result['count']} ideas)", "success") - return IdeationPhaseResult( - phase="ideation", - ideation_type=ideation_type, - success=True, - output_files=[str(output_file)], - ideas_count=validation_result["count"], - errors=[], - retries=0, - ) - - errors.append(validation_result["error"]) - - # Recovery attempts: show the current state and ask AI to fix it - for recovery_attempt in range(MAX_RETRIES - 1): - print_status(f"Running recovery agent (attempt {recovery_attempt + 1})...", "warning") - - recovery_success = await self._run_recovery_agent( - output_file, - ideation_type, - validation_result["error"], - validation_result.get("current_content", "") - ) - - if recovery_success: - # Re-validate after recovery - validation_result = self._validate_ideation_output(output_file, ideation_type) - - if validation_result["success"]: - print_status(f"Recovery successful: {output_file.name} ({validation_result['count']} ideas)", "success") - return IdeationPhaseResult( - phase="ideation", - ideation_type=ideation_type, - success=True, - output_files=[str(output_file)], - ideas_count=validation_result["count"], - errors=[], - retries=recovery_attempt + 1, - ) - else: - errors.append(f"Recovery {recovery_attempt + 1}: {validation_result['error']}") - else: - errors.append(f"Recovery {recovery_attempt + 1}: Agent failed to run") - - return IdeationPhaseResult( - phase="ideation", - ideation_type=ideation_type, - success=False, - output_files=[], - ideas_count=0, - errors=errors, - retries=MAX_RETRIES, - ) - - def _validate_ideation_output(self, output_file: Path, ideation_type: str) -> dict: - """Validate ideation output file and return validation result.""" - debug_detailed("ideation_runner", f"Validating output for {ideation_type}", - output_file=str(output_file)) - - if not output_file.exists(): - debug_warning("ideation_runner", "Output file does not exist", - output_file=str(output_file)) - return { - "success": False, - "error": "Output file does not exist", - "current_content": "", - "count": 0, - } - - try: - content = output_file.read_text() - data = json.loads(content) - debug_verbose("ideation_runner", "Parsed JSON successfully", - keys=list(data.keys())) - - # Check for correct key - ideas = data.get(ideation_type, []) - - # Also check for common incorrect key "ideas" - if not ideas and "ideas" in data: - debug_warning("ideation_runner", "Wrong JSON key detected", - expected=ideation_type, found="ideas") - return { - "success": False, - "error": f"Wrong JSON key: found 'ideas' but expected '{ideation_type}'", - "current_content": content, - "count": 0, - } - - if len(ideas) >= 1: - debug_success("ideation_runner", f"Validation passed for {ideation_type}", - ideas_count=len(ideas)) - return { - "success": True, - "error": None, - "current_content": content, - "count": len(ideas), - } - else: - debug_warning("ideation_runner", f"No ideas found for {ideation_type}") - return { - "success": False, - "error": f"No {ideation_type} ideas found in output", - "current_content": content, - "count": 0, - } - - except json.JSONDecodeError as e: - debug_error("ideation_runner", "JSON parse error", error=str(e)) - return { - "success": False, - "error": f"Invalid JSON: {e}", - "current_content": output_file.read_text() if output_file.exists() else "", - "count": 0, - } - - async def _run_recovery_agent( - self, - output_file: Path, - ideation_type: str, - error: str, - current_content: str, - ) -> bool: - """Run a recovery agent to fix validation errors in the output file.""" - - # Truncate content if too long - max_content_length = 8000 - if len(current_content) > max_content_length: - current_content = current_content[:max_content_length] + "\n... (truncated)" - - recovery_prompt = f"""# Ideation Output Recovery - -The ideation output file failed validation. Your task is to fix it. - -## Error -{error} - -## Expected Format -The output file must be valid JSON with the following structure: - -```json -{{ - "{ideation_type}": [ - {{ - "id": "...", - "type": "{ideation_type}", - "title": "...", - "description": "...", - ... other fields ... - }} - ] -}} -``` - -**CRITICAL**: The top-level key MUST be `"{ideation_type}"` (not "ideas" or anything else). - -## Current File Content -File: {output_file} - -```json -{current_content} -``` - -## Your Task -1. Read the current file content above -2. Identify what's wrong based on the error message -3. Fix the JSON structure to match the expected format -4. Write the corrected content to {output_file} - -Common fixes: -- If the key is "ideas", rename it to "{ideation_type}" -- If the JSON is invalid, fix the syntax errors -- If there are no ideas, ensure the array has at least one idea object - -Write the fixed JSON to the file now. -""" - - client = create_client(self.project_dir, self.output_dir, self.model) - - try: - async with client: - await client.query(recovery_prompt) - - async for msg in client.receive_response(): - msg_type = type(msg).__name__ - - if msg_type == "AssistantMessage" and hasattr(msg, "content"): - for block in msg.content: - block_type = type(block).__name__ - if block_type == "TextBlock" and hasattr(block, "text"): - print(block.text, end="", flush=True) - elif block_type == "ToolUseBlock" and hasattr(block, "name"): - print(f"\n[Recovery Tool: {block.name}]", flush=True) - - print() - return True - - except Exception as e: - print_status(f"Recovery agent error: {e}", "error") - return False - - async def phase_merge(self) -> IdeationPhaseResult: - """Merge all ideation outputs into a single ideation.json.""" - - ideation_file = self.output_dir / "ideation.json" - - # Load existing ideas if in append mode - existing_ideas = [] - existing_session = None - if self.append and ideation_file.exists(): - try: - with open(ideation_file) as f: - existing_session = json.load(f) - existing_ideas = existing_session.get("ideas", []) - print_status(f"Preserving {len(existing_ideas)} existing ideas", "info") - except json.JSONDecodeError: - pass - - # Collect new ideas from the enabled types - new_ideas = [] - output_files = [] - - for ideation_type in self.enabled_types: - type_file = self.output_dir / f"{ideation_type}_ideas.json" - if type_file.exists(): - try: - with open(type_file) as f: - data = json.load(f) - ideas = data.get(ideation_type, []) - new_ideas.extend(ideas) - output_files.append(str(type_file)) - except (json.JSONDecodeError, KeyError): - pass - - # In append mode, filter out ideas from types we're regenerating - # (to avoid duplicates) and keep ideas from other types - if self.append and existing_ideas: - # Keep existing ideas that are NOT from the types we just generated - preserved_ideas = [ - idea for idea in existing_ideas - if idea.get("type") not in self.enabled_types - ] - all_ideas = preserved_ideas + new_ideas - print_status(f"Merged: {len(preserved_ideas)} preserved + {len(new_ideas)} new = {len(all_ideas)} total", "info") - else: - all_ideas = new_ideas - - # Load context for metadata - context_file = self.output_dir / "ideation_context.json" - context_data = {} - if context_file.exists(): - try: - with open(context_file) as f: - context_data = json.load(f) - except json.JSONDecodeError: - pass - - # Create merged ideation session - # Preserve session ID and generated_at if appending - session_id = existing_session.get("id") if existing_session else f"ideation-{datetime.now().strftime('%Y%m%d-%H%M%S')}" - generated_at = existing_session.get("generated_at") if existing_session else datetime.now().isoformat() - - ideation_session = { - "id": session_id, - "project_id": str(self.project_dir), - "config": context_data.get("config", {}), - "ideas": all_ideas, - "project_context": { - "existing_features": context_data.get("existing_features", []), - "tech_stack": context_data.get("tech_stack", []), - "target_audience": context_data.get("target_audience"), - "planned_features": context_data.get("planned_features", []), - }, - "summary": { - "total_ideas": len(all_ideas), - "by_type": {}, - "by_status": {}, - }, - "generated_at": generated_at, - "updated_at": datetime.now().isoformat(), - } - - # Count by type and status - for idea in all_ideas: - idea_type = idea.get("type", "unknown") - idea_status = idea.get("status", "draft") - ideation_session["summary"]["by_type"][idea_type] = \ - ideation_session["summary"]["by_type"].get(idea_type, 0) + 1 - ideation_session["summary"]["by_status"][idea_status] = \ - ideation_session["summary"]["by_status"].get(idea_status, 0) + 1 - - with open(ideation_file, "w") as f: - json.dump(ideation_session, f, indent=2) - - action = "Updated" if self.append else "Created" - print_status(f"{action} ideation.json ({len(all_ideas)} total ideas)", "success") - - return IdeationPhaseResult( - phase="merge", - ideation_type=None, - success=True, - output_files=[str(ideation_file)], - ideas_count=len(all_ideas), - errors=[], - retries=0, - ) - - async def _run_ideation_type_with_streaming(self, ideation_type: str) -> IdeationPhaseResult: - """Run a single ideation type and stream results when complete.""" - result = await self.phase_ideation_type(ideation_type) - - if result.success: - # Signal that this type is complete - UI can now show these ideas - print(f"IDEATION_TYPE_COMPLETE:{ideation_type}:{result.ideas_count}") - sys.stdout.flush() - else: - print(f"IDEATION_TYPE_FAILED:{ideation_type}") - sys.stdout.flush() - - return result - - async def run(self) -> bool: - """Run the complete ideation generation process.""" - - debug_section("ideation_runner", "Starting Ideation Generation") - debug("ideation_runner", "Configuration", - project_dir=str(self.project_dir), - output_dir=str(self.output_dir), - model=self.model, - enabled_types=self.enabled_types, - refresh=self.refresh, - append=self.append) - - print(box( - f"Project: {self.project_dir}\n" - f"Output: {self.output_dir}\n" - f"Model: {self.model}\n" - f"Types: {', '.join(self.enabled_types)}", - title="IDEATION GENERATOR", - style="heavy" - )) - - results = [] - - # Phase 1: Project Index - debug("ideation_runner", "Starting Phase 1: Project Analysis") - print_section("PHASE 1: PROJECT ANALYSIS", Icons.FOLDER) - result = await self.phase_project_index() - results.append(result) - if not result.success: - print_status("Project analysis failed", "error") - return False - - # Phase 2: Context & Graph Hints (in parallel) - print_section("PHASE 2: CONTEXT & GRAPH HINTS (PARALLEL)", Icons.SEARCH) - - # Run context gathering and graph hints in parallel - context_task = self.phase_context() - hints_task = self.phase_graph_hints() - context_result, hints_result = await asyncio.gather(context_task, hints_task) - - results.append(hints_result) - results.append(context_result) - - if not context_result.success: - print_status("Context gathering failed", "error") - return False - # Note: hints_result.success is always True (graceful degradation) - - # Phase 3: Run all ideation types IN PARALLEL - debug("ideation_runner", "Starting Phase 3: Generating Ideas", - types=self.enabled_types, parallel=True) - print_section("PHASE 3: GENERATING IDEAS (PARALLEL)", Icons.SUBTASK) - print_status(f"Starting {len(self.enabled_types)} ideation agents in parallel...", "progress") - - # Create tasks for all enabled types - ideation_tasks = [ - self._run_ideation_type_with_streaming(ideation_type) - for ideation_type in self.enabled_types - ] - - # Run all ideation types concurrently - ideation_results = await asyncio.gather(*ideation_tasks, return_exceptions=True) - - # Process results - for i, result in enumerate(ideation_results): - ideation_type = self.enabled_types[i] - if isinstance(result, Exception): - print_status(f"{IDEATION_TYPE_LABELS[ideation_type]} ideation failed with exception: {result}", "error") - results.append(IdeationPhaseResult( - phase="ideation", - ideation_type=ideation_type, - success=False, - output_files=[], - ideas_count=0, - errors=[str(result)], - retries=0, - )) - else: - results.append(result) - if result.success: - print_status(f"{IDEATION_TYPE_LABELS[ideation_type]}: {result.ideas_count} ideas", "success") - else: - print_status(f"{IDEATION_TYPE_LABELS[ideation_type]} ideation failed", "warning") - for err in result.errors: - print(f" {muted('Error:')} {err}") - - # Final Phase: Merge - print_section("PHASE 4: MERGE & FINALIZE", Icons.SUCCESS) - result = await self.phase_merge() - results.append(result) - - # Summary - ideation_file = self.output_dir / "ideation.json" - if ideation_file.exists(): - with open(ideation_file) as f: - ideation = json.load(f) - - ideas = ideation.get("ideas", []) - summary = ideation.get("summary", {}) - by_type = summary.get("by_type", {}) - - print(box( - f"Total Ideas: {len(ideas)}\n\n" - f"By Type:\n" + - "\n".join(f" {icon(Icons.ARROW_RIGHT)} {IDEATION_TYPE_LABELS.get(t, t)}: {c}" - for t, c in by_type.items()) + - f"\n\nIdeation saved to: {ideation_file}", - title=f"{icon(Icons.SUCCESS)} IDEATION COMPLETE", - style="heavy" - )) - - return True - - -def main(): - """CLI entry point.""" - import argparse - - parser = argparse.ArgumentParser( - description="AI-powered ideation generation", - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "--project", - type=Path, - default=Path.cwd(), - help="Project directory (default: current directory)", - ) - parser.add_argument( - "--output", - type=Path, - help="Output directory for ideation files (default: project/auto-claude/ideation)", - ) - parser.add_argument( - "--types", - type=str, - help=f"Comma-separated ideation types to run (options: {','.join(IDEATION_TYPES)})", - ) - parser.add_argument( - "--no-roadmap", - action="store_true", - help="Don't include roadmap context", - ) - parser.add_argument( - "--no-kanban", - action="store_true", - help="Don't include kanban context", - ) - parser.add_argument( - "--max-ideas", - type=int, - default=5, - help="Maximum ideas per type (default: 5)", - ) - parser.add_argument( - "--model", - type=str, - default="claude-opus-4-5-20251101", - help="Model to use (default: claude-opus-4-5-20251101)", - ) - parser.add_argument( - "--refresh", - action="store_true", - help="Force regeneration even if ideation exists", - ) - parser.add_argument( - "--append", - action="store_true", - help="Append new ideas to existing session instead of replacing", - ) - - args = parser.parse_args() - - # Validate project directory - project_dir = args.project.resolve() - if not project_dir.exists(): - print(f"Error: Project directory does not exist: {project_dir}") - sys.exit(1) - - # Parse types - enabled_types = None - if args.types: - enabled_types = [t.strip() for t in args.types.split(",")] - invalid_types = [t for t in enabled_types if t not in IDEATION_TYPES] - if invalid_types: - print(f"Error: Invalid ideation types: {invalid_types}") - print(f"Valid types: {IDEATION_TYPES}") - sys.exit(1) - - orchestrator = IdeationOrchestrator( - project_dir=project_dir, - output_dir=args.output, - enabled_types=enabled_types, - include_roadmap_context=not args.no_roadmap, - include_kanban_context=not args.no_kanban, - max_ideas_per_type=args.max_ideas, - model=args.model, - refresh=args.refresh, - append=args.append, - ) - - try: - success = asyncio.run(orchestrator.run()) - sys.exit(0 if success else 1) - except KeyboardInterrupt: - print("\n\nIdeation generation interrupted.") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/auto-claude/implementation_plan.py b/auto-claude/implementation_plan.py index 733e78a4..64641781 100644 --- a/auto-claude/implementation_plan.py +++ b/auto-claude/implementation_plan.py @@ -1,168 +1,3 @@ -#!/usr/bin/env python3 -""" -Implementation Plan Manager -============================ - -DEPRECATED: This module is now a compatibility shim. The implementation has been -refactored into the implementation_plan/ package for better modularity. - -Please import from the package directly: - from implementation_plan import ImplementationPlan, Subtask, Phase, etc. - -This file re-exports all public APIs for backwards compatibility. - -Core data structures and utilities for subtask-based implementation plans. -Replaces the test-centric feature_list.json with implementation_plan.json. - -The key insight: Tests verify outcomes, but SUBTASKS define implementation steps. -For complex multi-service features, implementation order matters. - -Workflow Types: -- feature: Standard multi-service feature (phases = services) -- refactor: Migration/refactor work (phases = stages: add, migrate, remove) -- investigation: Bug hunting (phases = investigate, hypothesize, fix) -- migration: Data migration (phases = prepare, test, execute, cleanup) -- simple: Single-service enhancement (minimal overhead) -""" - -# Re-export everything from the implementation_plan package -from implementation_plan import ( - Chunk, - ChunkStatus, - ImplementationPlan, - Phase, - PhaseType, - Subtask, - SubtaskStatus, - Verification, - VerificationType, - WorkflowType, - create_feature_plan, - create_investigation_plan, - create_refactor_plan, -) - -__all__ = [ - # Enums - "WorkflowType", - "PhaseType", - "SubtaskStatus", - "VerificationType", - # Models - "Verification", - "Subtask", - "Phase", - "ImplementationPlan", - # Factories - "create_feature_plan", - "create_investigation_plan", - "create_refactor_plan", - # Backwards compatibility - "Chunk", - "ChunkStatus", -] - - -# CLI for testing -if __name__ == "__main__": - import json - import sys - from pathlib import Path - - if len(sys.argv) < 2: - print("Usage: python implementation_plan.py ") - print(" python implementation_plan.py --demo") - sys.exit(1) - - if sys.argv[1] == "--demo": - # Create a demo plan - plan = create_feature_plan( - feature="Avatar Upload with Processing", - services=["backend", "worker", "frontend"], - phases_config=[ - { - "name": "Backend Foundation", - "parallel_safe": True, - "subtasks": [ - { - "id": "avatar-model", - "service": "backend", - "description": "Add avatar fields to User model", - "files_to_modify": ["app/models/user.py"], - "files_to_create": ["migrations/add_avatar.py"], - "verification": { - "type": "command", - "run": "flask db upgrade", - }, - }, - { - "id": "avatar-endpoint", - "service": "backend", - "description": "POST /api/users/avatar endpoint", - "files_to_modify": ["app/routes/users.py"], - "patterns_from": ["app/routes/profile.py"], - "verification": { - "type": "api", - "method": "POST", - "url": "/api/users/avatar", - }, - }, - ], - }, - { - "name": "Worker Pipeline", - "depends_on": [1], - "subtasks": [ - { - "id": "image-task", - "service": "worker", - "description": "Celery task for image processing", - "files_to_create": ["app/tasks/images.py"], - "patterns_from": ["app/tasks/reports.py"], - }, - ], - }, - { - "name": "Frontend", - "depends_on": [1], - "subtasks": [ - { - "id": "avatar-component", - "service": "frontend", - "description": "AvatarUpload React component", - "files_to_create": ["src/components/AvatarUpload.tsx"], - "patterns_from": ["src/components/FileUpload.tsx"], - }, - ], - }, - { - "name": "Integration", - "depends_on": [2, 3], - "type": "integration", - "subtasks": [ - { - "id": "e2e-wiring", - "all_services": True, - "description": "Connect frontend → backend → worker", - "verification": { - "type": "browser", - "scenario": "Upload → Process → Display", - }, - }, - ], - }, - ], - ) - plan.final_acceptance = [ - "User can upload avatar from profile page", - "Avatar is automatically resized", - "Large/invalid files show error", - ] - - print(json.dumps(plan.to_dict(), indent=2)) - print("\n---\n") - print(plan.get_status_summary()) - else: - # Load and display existing plan - plan = ImplementationPlan.load(Path(sys.argv[1])) - print(plan.get_status_summary()) +"""Backward compatibility shim - import from implementation_plan package instead.""" +from implementation_plan.main import * +from implementation_plan import * diff --git a/auto-claude/implementation_plan/main.py b/auto-claude/implementation_plan/main.py new file mode 100644 index 00000000..733e78a4 --- /dev/null +++ b/auto-claude/implementation_plan/main.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +""" +Implementation Plan Manager +============================ + +DEPRECATED: This module is now a compatibility shim. The implementation has been +refactored into the implementation_plan/ package for better modularity. + +Please import from the package directly: + from implementation_plan import ImplementationPlan, Subtask, Phase, etc. + +This file re-exports all public APIs for backwards compatibility. + +Core data structures and utilities for subtask-based implementation plans. +Replaces the test-centric feature_list.json with implementation_plan.json. + +The key insight: Tests verify outcomes, but SUBTASKS define implementation steps. +For complex multi-service features, implementation order matters. + +Workflow Types: +- feature: Standard multi-service feature (phases = services) +- refactor: Migration/refactor work (phases = stages: add, migrate, remove) +- investigation: Bug hunting (phases = investigate, hypothesize, fix) +- migration: Data migration (phases = prepare, test, execute, cleanup) +- simple: Single-service enhancement (minimal overhead) +""" + +# Re-export everything from the implementation_plan package +from implementation_plan import ( + Chunk, + ChunkStatus, + ImplementationPlan, + Phase, + PhaseType, + Subtask, + SubtaskStatus, + Verification, + VerificationType, + WorkflowType, + create_feature_plan, + create_investigation_plan, + create_refactor_plan, +) + +__all__ = [ + # Enums + "WorkflowType", + "PhaseType", + "SubtaskStatus", + "VerificationType", + # Models + "Verification", + "Subtask", + "Phase", + "ImplementationPlan", + # Factories + "create_feature_plan", + "create_investigation_plan", + "create_refactor_plan", + # Backwards compatibility + "Chunk", + "ChunkStatus", +] + + +# CLI for testing +if __name__ == "__main__": + import json + import sys + from pathlib import Path + + if len(sys.argv) < 2: + print("Usage: python implementation_plan.py ") + print(" python implementation_plan.py --demo") + sys.exit(1) + + if sys.argv[1] == "--demo": + # Create a demo plan + plan = create_feature_plan( + feature="Avatar Upload with Processing", + services=["backend", "worker", "frontend"], + phases_config=[ + { + "name": "Backend Foundation", + "parallel_safe": True, + "subtasks": [ + { + "id": "avatar-model", + "service": "backend", + "description": "Add avatar fields to User model", + "files_to_modify": ["app/models/user.py"], + "files_to_create": ["migrations/add_avatar.py"], + "verification": { + "type": "command", + "run": "flask db upgrade", + }, + }, + { + "id": "avatar-endpoint", + "service": "backend", + "description": "POST /api/users/avatar endpoint", + "files_to_modify": ["app/routes/users.py"], + "patterns_from": ["app/routes/profile.py"], + "verification": { + "type": "api", + "method": "POST", + "url": "/api/users/avatar", + }, + }, + ], + }, + { + "name": "Worker Pipeline", + "depends_on": [1], + "subtasks": [ + { + "id": "image-task", + "service": "worker", + "description": "Celery task for image processing", + "files_to_create": ["app/tasks/images.py"], + "patterns_from": ["app/tasks/reports.py"], + }, + ], + }, + { + "name": "Frontend", + "depends_on": [1], + "subtasks": [ + { + "id": "avatar-component", + "service": "frontend", + "description": "AvatarUpload React component", + "files_to_create": ["src/components/AvatarUpload.tsx"], + "patterns_from": ["src/components/FileUpload.tsx"], + }, + ], + }, + { + "name": "Integration", + "depends_on": [2, 3], + "type": "integration", + "subtasks": [ + { + "id": "e2e-wiring", + "all_services": True, + "description": "Connect frontend → backend → worker", + "verification": { + "type": "browser", + "scenario": "Upload → Process → Display", + }, + }, + ], + }, + ], + ) + plan.final_acceptance = [ + "User can upload avatar from profile page", + "Avatar is automatically resized", + "Large/invalid files show error", + ] + + print(json.dumps(plan.to_dict(), indent=2)) + print("\n---\n") + print(plan.get_status_summary()) + else: + # Load and display existing plan + plan = ImplementationPlan.load(Path(sys.argv[1])) + print(plan.get_status_summary()) diff --git a/auto-claude/insight_extractor.py b/auto-claude/insight_extractor.py index 281fdb87..b7a650d2 100644 --- a/auto-claude/insight_extractor.py +++ b/auto-claude/insight_extractor.py @@ -1,584 +1,41 @@ """ -Insight Extractor -================= +Insight Extractor Re-export +=========================== -Automatically extracts structured insights from completed coding sessions. -Runs after each session to capture rich, actionable knowledge for Graphiti memory. - -Uses the Claude Agent SDK (same as the rest of the system) for extraction. -Falls back to generic insights if extraction fails (never blocks the build). +Re-exports the insight_extractor module from analysis/ for backwards compatibility. +Uses importlib to avoid triggering analysis/__init__.py imports. """ -import json -import logging -import os -import subprocess +import importlib.util +import sys from pathlib import Path -from typing import Any -logger = logging.getLogger(__name__) - -# Check for Claude SDK availability -try: - from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient - SDK_AVAILABLE = True -except ImportError: - SDK_AVAILABLE = False - ClaudeAgentOptions = None - ClaudeSDKClient = None - -# Default model for insight extraction (fast and cheap) -DEFAULT_EXTRACTION_MODEL = "claude-3-5-haiku-latest" - -# Maximum diff size to send to the LLM (avoid context limits) -MAX_DIFF_CHARS = 15000 - -# Maximum attempt history entries to include -MAX_ATTEMPTS_TO_INCLUDE = 3 - - -def is_extraction_enabled() -> bool: - """Check if insight extraction is enabled.""" - # Extraction requires Claude SDK and OAuth token - if not SDK_AVAILABLE: - return False - if not os.environ.get("CLAUDE_CODE_OAUTH_TOKEN"): - return False - enabled_str = os.environ.get("INSIGHT_EXTRACTION_ENABLED", "true").lower() - return enabled_str in ("true", "1", "yes") - - -def get_extraction_model() -> str: - """Get the model to use for insight extraction.""" - return os.environ.get("INSIGHT_EXTRACTOR_MODEL", DEFAULT_EXTRACTION_MODEL) - - -# ============================================================================= -# Git Helpers -# ============================================================================= - - -def get_session_diff( - project_dir: Path, - commit_before: str | None, - commit_after: str | None, -) -> str: - """ - Get the git diff between two commits. - - Args: - project_dir: Project root directory - commit_before: Commit hash before session (or None) - commit_after: Commit hash after session (or None) - - Returns: - Diff text (truncated if too large) - """ - if not commit_before or not commit_after: - return "(No commits to diff)" - - if commit_before == commit_after: - return "(No changes - same commit)" - - try: - result = subprocess.run( - ["git", "diff", commit_before, commit_after], - cwd=project_dir, - capture_output=True, - text=True, - timeout=30, - ) - diff = result.stdout - - if len(diff) > MAX_DIFF_CHARS: - # Truncate and add note - diff = ( - diff[:MAX_DIFF_CHARS] + f"\n\n... (truncated, {len(diff)} chars total)" - ) - - return diff if diff else "(Empty diff)" - - except subprocess.TimeoutExpired: - logger.warning("Git diff timed out") - return "(Git diff timed out)" - except Exception as e: - logger.warning(f"Failed to get git diff: {e}") - return f"(Failed to get diff: {e})" - - -def get_changed_files( - project_dir: Path, - commit_before: str | None, - commit_after: str | None, -) -> list[str]: - """ - Get list of files changed between two commits. - - Args: - project_dir: Project root directory - commit_before: Commit hash before session - commit_after: Commit hash after session - - Returns: - List of changed file paths - """ - if not commit_before or not commit_after or commit_before == commit_after: - return [] - - try: - result = subprocess.run( - ["git", "diff", "--name-only", commit_before, commit_after], - cwd=project_dir, - capture_output=True, - text=True, - timeout=10, - ) - files = [f.strip() for f in result.stdout.strip().split("\n") if f.strip()] - return files - - except Exception as e: - logger.warning(f"Failed to get changed files: {e}") - return [] - - -def get_commit_messages( - project_dir: Path, - commit_before: str | None, - commit_after: str | None, -) -> str: - """Get commit messages between two commits.""" - if not commit_before or not commit_after or commit_before == commit_after: - return "(No commits)" - - try: - result = subprocess.run( - ["git", "log", "--oneline", f"{commit_before}..{commit_after}"], - cwd=project_dir, - capture_output=True, - text=True, - timeout=10, - ) - return result.stdout.strip() if result.stdout.strip() else "(No commits)" - - except Exception as e: - logger.warning(f"Failed to get commit messages: {e}") - return f"(Failed: {e})" - - -# ============================================================================= -# Input Gathering -# ============================================================================= - - -def gather_extraction_inputs( - spec_dir: Path, - project_dir: Path, - subtask_id: str, - session_num: int, - commit_before: str | None, - commit_after: str | None, - success: bool, - recovery_manager: Any, -) -> dict: - """ - Gather all inputs needed for insight extraction. - - Args: - spec_dir: Spec directory - project_dir: Project root - subtask_id: The subtask that was worked on - session_num: Session number - commit_before: Commit before session - commit_after: Commit after session - success: Whether session succeeded - recovery_manager: Recovery manager with attempt history - - Returns: - Dict with all inputs for the extractor - """ - # Get subtask description from implementation plan - subtask_description = _get_subtask_description(spec_dir, subtask_id) - - # Get git diff - diff = get_session_diff(project_dir, commit_before, commit_after) - - # Get changed files - changed_files = get_changed_files(project_dir, commit_before, commit_after) - - # Get commit messages - commit_messages = get_commit_messages(project_dir, commit_before, commit_after) - - # Get attempt history - attempt_history = _get_attempt_history(recovery_manager, subtask_id) - - return { - "subtask_id": subtask_id, - "subtask_description": subtask_description, - "session_num": session_num, - "success": success, - "diff": diff, - "changed_files": changed_files, - "commit_messages": commit_messages, - "attempt_history": attempt_history, - } - - -def _get_subtask_description(spec_dir: Path, subtask_id: str) -> str: - """Get subtask description from implementation plan.""" - plan_file = spec_dir / "implementation_plan.json" - if not plan_file.exists(): - return f"Subtask: {subtask_id}" - - try: - with open(plan_file) as f: - plan = json.load(f) - - # Search through phases for the subtask - for phase in plan.get("phases", []): - for subtask in phase.get("subtasks", []): - if subtask.get("id") == subtask_id: - return subtask.get("description", f"Subtask: {subtask_id}") - - return f"Subtask: {subtask_id}" - - except Exception as e: - logger.warning(f"Failed to load subtask description: {e}") - return f"Subtask: {subtask_id}" - - -def _get_attempt_history(recovery_manager: Any, subtask_id: str) -> list[dict]: - """Get previous attempt history for this subtask.""" - if not recovery_manager: - return [] - - try: - history = recovery_manager.get_subtask_history(subtask_id) - attempts = history.get("attempts", []) - - # Limit to recent attempts - return attempts[-MAX_ATTEMPTS_TO_INCLUDE:] - - except Exception as e: - logger.warning(f"Failed to get attempt history: {e}") - return [] - - -# ============================================================================= -# LLM Extraction -# ============================================================================= - - -def _build_extraction_prompt(inputs: dict) -> str: - """Build the prompt for insight extraction.""" - prompt_file = Path(__file__).parent / "prompts" / "insight_extractor.md" - - if prompt_file.exists(): - base_prompt = prompt_file.read_text() - else: - # Fallback if prompt file missing - base_prompt = """Extract structured insights from this coding session. -Output ONLY valid JSON with: file_insights, patterns_discovered, gotchas_discovered, approach_outcome, recommendations""" - - # Build session context - session_context = f""" ---- - -## SESSION DATA - -### Subtask -- **ID**: {inputs["subtask_id"]} -- **Description**: {inputs["subtask_description"]} -- **Session Number**: {inputs["session_num"]} -- **Outcome**: {"SUCCESS" if inputs["success"] else "FAILED"} - -### Files Changed -{chr(10).join(f"- {f}" for f in inputs["changed_files"]) if inputs["changed_files"] else "(No files changed)"} - -### Commit Messages -{inputs["commit_messages"]} - -### Git Diff -```diff -{inputs["diff"]} -``` - -### Previous Attempts -{_format_attempt_history(inputs["attempt_history"])} - ---- - -Now analyze this session and output ONLY the JSON object. -""" - - return base_prompt + session_context - - -def _format_attempt_history(attempts: list[dict]) -> str: - """Format attempt history for the prompt.""" - if not attempts: - return "(First attempt - no previous history)" - - lines = [] - for i, attempt in enumerate(attempts, 1): - success = "SUCCESS" if attempt.get("success") else "FAILED" - approach = attempt.get("approach", "Unknown approach") - error = attempt.get("error", "") - lines.append(f"**Attempt {i}** ({success}): {approach}") - if error: - lines.append(f" Error: {error}") - - return "\n".join(lines) - - -async def run_insight_extraction(inputs: dict, project_dir: Path | None = None) -> dict | None: - """ - Run the insight extraction using Claude Agent SDK. - - Args: - inputs: Gathered session inputs - project_dir: Project directory for SDK context (optional) - - Returns: - Extracted insights dict or None if failed - """ - if not SDK_AVAILABLE: - logger.warning("Claude SDK not available, skipping insight extraction") - return None - - oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") - if not oauth_token: - logger.warning("CLAUDE_CODE_OAUTH_TOKEN not set, skipping insight extraction") - return None - - model = get_extraction_model() - prompt = _build_extraction_prompt(inputs) - - # Use current directory if project_dir not specified - cwd = str(project_dir.resolve()) if project_dir else os.getcwd() - - try: - # Create a minimal SDK client for insight extraction - # No tools needed - just text generation - client = ClaudeSDKClient( - options=ClaudeAgentOptions( - model=model, - system_prompt=( - "You are an expert code analyst. You extract structured insights from coding sessions. " - "Always respond with valid JSON only, no markdown formatting or explanations." - ), - allowed_tools=[], # No tools needed for extraction - max_turns=1, # Single turn extraction - cwd=cwd, - ) - ) - - # Use async context manager - async with client: - await client.query(prompt) - - # Collect the response - response_text = "" - async for msg in client.receive_response(): - msg_type = type(msg).__name__ - if msg_type == "AssistantMessage" and hasattr(msg, "content"): - for block in msg.content: - if hasattr(block, "text"): - response_text += block.text - - # Parse JSON from response - return parse_insights(response_text) - - except Exception as e: - logger.warning(f"Insight extraction failed: {e}") - return None - - -def parse_insights(response_text: str) -> dict | None: - """ - Parse the LLM response into structured insights. - - Args: - response_text: Raw LLM response - - Returns: - Parsed insights dict or None if parsing failed - """ - # Try to extract JSON from the response - text = response_text.strip() - - # Handle markdown code blocks - if text.startswith("```"): - # Remove code block markers - lines = text.split("\n") - # Remove first line (```json or ```) - if lines[0].startswith("```"): - lines = lines[1:] - # Remove last line if it's `` - if lines and lines[-1].strip() == "```": - lines = lines[:-1] - text = "\n".join(lines) - - try: - insights = json.loads(text) - - # Validate structure - if not isinstance(insights, dict): - logger.warning("Insights is not a dict") - return None - - # Ensure required keys exist with defaults - insights.setdefault("file_insights", []) - insights.setdefault("patterns_discovered", []) - insights.setdefault("gotchas_discovered", []) - insights.setdefault("approach_outcome", {}) - insights.setdefault("recommendations", []) - - return insights - - except json.JSONDecodeError as e: - logger.warning(f"Failed to parse insights JSON: {e}") - logger.debug(f"Response text was: {text[:500]}") - return None - - -# ============================================================================= -# Main Entry Point -# ============================================================================= - - -async def extract_session_insights( - spec_dir: Path, - project_dir: Path, - subtask_id: str, - session_num: int, - commit_before: str | None, - commit_after: str | None, - success: bool, - recovery_manager: Any, -) -> dict: - """ - Extract insights from a completed coding session. - - This is the main entry point called from post_session_processing(). - Falls back to generic insights if extraction fails. - - Args: - spec_dir: Spec directory - project_dir: Project root - subtask_id: Subtask that was worked on - session_num: Session number - commit_before: Commit before session - commit_after: Commit after session - success: Whether session succeeded - recovery_manager: Recovery manager with attempt history - - Returns: - Insights dict (rich if extraction succeeded, generic if failed) - """ - # Check if extraction is enabled - if not is_extraction_enabled(): - logger.info("Insight extraction disabled") - return _get_generic_insights(subtask_id, success) - - # Check for no changes - if commit_before == commit_after: - logger.info("No changes to extract insights from") - return _get_generic_insights(subtask_id, success) - - try: - # Gather inputs - inputs = gather_extraction_inputs( - spec_dir=spec_dir, - project_dir=project_dir, - subtask_id=subtask_id, - session_num=session_num, - commit_before=commit_before, - commit_after=commit_after, - success=success, - recovery_manager=recovery_manager, - ) - - # Run extraction - extracted = await run_insight_extraction(inputs, project_dir=project_dir) - - if extracted: - # Add metadata - extracted["subtask_id"] = subtask_id - extracted["session_num"] = session_num - extracted["success"] = success - extracted["changed_files"] = inputs["changed_files"] - - logger.info( - f"Extracted insights: {len(extracted.get('file_insights', []))} file insights, " - f"{len(extracted.get('patterns_discovered', []))} patterns, " - f"{len(extracted.get('gotchas_discovered', []))} gotchas" - ) - return extracted - else: - logger.warning("Extraction returned no results, using generic insights") - return _get_generic_insights(subtask_id, success) - - except Exception as e: - logger.warning(f"Insight extraction failed: {e}, using generic insights") - return _get_generic_insights(subtask_id, success) - - -def _get_generic_insights(subtask_id: str, success: bool) -> dict: - """Return generic insights when extraction fails or is disabled.""" - return { - "file_insights": [], - "patterns_discovered": [], - "gotchas_discovered": [], - "approach_outcome": { - "success": success, - "approach_used": f"Implemented subtask: {subtask_id}", - "why_it_worked": None, - "why_it_failed": None, - "alternatives_tried": [], - }, - "recommendations": [], - "subtask_id": subtask_id, - "success": success, - "changed_files": [], - } - - -# ============================================================================= -# CLI for Testing -# ============================================================================= - -if __name__ == "__main__": - import argparse - import asyncio - - parser = argparse.ArgumentParser(description="Test insight extraction") - parser.add_argument("--spec-dir", type=Path, required=True, help="Spec directory") - parser.add_argument( - "--project-dir", type=Path, required=True, help="Project directory" - ) - parser.add_argument( - "--commit-before", type=str, required=True, help="Commit before session" - ) - parser.add_argument( - "--commit-after", type=str, required=True, help="Commit after session" - ) - parser.add_argument( - "--subtask-id", type=str, default="test-subtask", help="Subtask ID" - ) - - args = parser.parse_args() - - async def main(): - insights = await extract_session_insights( - spec_dir=args.spec_dir, - project_dir=args.project_dir, - subtask_id=args.subtask_id, - session_num=1, - commit_before=args.commit_before, - commit_after=args.commit_after, - success=True, - recovery_manager=None, - ) - print(json.dumps(insights, indent=2)) - - asyncio.run(main()) +# Load the module directly without going through the package +_module_path = Path(__file__).parent / "analysis" / "insight_extractor.py" +_spec = importlib.util.spec_from_file_location("_insight_extractor_impl", _module_path) +_module = importlib.util.module_from_spec(_spec) +sys.modules["_insight_extractor_impl"] = _module +_spec.loader.exec_module(_module) + +# Re-export all public functions +extract_session_insights = _module.extract_session_insights +gather_extraction_inputs = _module.gather_extraction_inputs +get_changed_files = _module.get_changed_files +get_commit_messages = _module.get_commit_messages +get_extraction_model = _module.get_extraction_model +get_session_diff = _module.get_session_diff +is_extraction_enabled = _module.is_extraction_enabled +parse_insights = _module.parse_insights +run_insight_extraction = _module.run_insight_extraction + +__all__ = [ + "extract_session_insights", + "gather_extraction_inputs", + "get_changed_files", + "get_commit_messages", + "get_extraction_model", + "get_session_diff", + "is_extraction_enabled", + "parse_insights", + "run_insight_extraction", +] diff --git a/auto-claude/integrations/__init__.py b/auto-claude/integrations/__init__.py new file mode 100644 index 00000000..c6c06b34 --- /dev/null +++ b/auto-claude/integrations/__init__.py @@ -0,0 +1,11 @@ +""" +Integrations Module +=================== + +External service integrations for Auto Claude. +""" + +__all__ = [ + "linear", + "graphiti", +] diff --git a/auto-claude/integrations/graphiti/__init__.py b/auto-claude/integrations/graphiti/__init__.py new file mode 100644 index 00000000..eaa0b234 --- /dev/null +++ b/auto-claude/integrations/graphiti/__init__.py @@ -0,0 +1,35 @@ +""" +Graphiti Integration +==================== + +Integration with Graphiti knowledge graph for semantic memory. +""" + +# Config imports don't require graphiti package +from .config import GraphitiConfig, validate_graphiti_config + +# Lazy imports for components that require graphiti package +__all__ = [ + "GraphitiConfig", + "validate_graphiti_config", + "GraphitiMemory", + "create_llm_client", + "create_embedder", +] + + +def __getattr__(name): + """Lazy import to avoid requiring graphiti package for config-only imports.""" + if name == "GraphitiMemory": + from .memory import GraphitiMemory + + return GraphitiMemory + elif name == "create_llm_client": + from .providers import create_llm_client + + return create_llm_client + elif name == "create_embedder": + from .providers import create_embedder + + return create_embedder + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/auto-claude/integrations/graphiti/config.py b/auto-claude/integrations/graphiti/config.py new file mode 100644 index 00000000..54b435d0 --- /dev/null +++ b/auto-claude/integrations/graphiti/config.py @@ -0,0 +1,547 @@ +""" +Graphiti Integration Configuration +================================== + +Constants, status mappings, and configuration helpers for Graphiti memory integration. +Follows the same patterns as linear_config.py for consistency. + +Multi-Provider Support (V2): +- LLM Providers: OpenAI, Anthropic, Azure OpenAI, Ollama +- Embedder Providers: OpenAI, Voyage AI, Azure OpenAI, Ollama + +Environment Variables: + # Core + GRAPHITI_ENABLED: Set to "true" to enable Graphiti integration + GRAPHITI_LLM_PROVIDER: openai|anthropic|azure_openai|ollama (default: openai) + GRAPHITI_EMBEDDER_PROVIDER: openai|voyage|azure_openai|ollama (default: openai) + + # OpenAI + OPENAI_API_KEY: Required for OpenAI provider + OPENAI_MODEL: Model for LLM (default: gpt-5-mini) + OPENAI_EMBEDDING_MODEL: Model for embeddings (default: text-embedding-3-small) + + # Anthropic (LLM only - needs separate embedder) + ANTHROPIC_API_KEY: Required for Anthropic provider + GRAPHITI_ANTHROPIC_MODEL: Model for LLM (default: claude-sonnet-4-5-latest) + + # Azure OpenAI + AZURE_OPENAI_API_KEY: Required for Azure provider + AZURE_OPENAI_BASE_URL: Azure endpoint URL + AZURE_OPENAI_LLM_DEPLOYMENT: Deployment name for LLM + AZURE_OPENAI_EMBEDDING_DEPLOYMENT: Deployment name for embeddings + + # Voyage AI (embeddings only - commonly used with Anthropic) + VOYAGE_API_KEY: Required for Voyage embedder + VOYAGE_EMBEDDING_MODEL: Model (default: voyage-3) + + # Ollama (local) + OLLAMA_BASE_URL: Ollama server URL (default: http://localhost:11434) + OLLAMA_LLM_MODEL: Model for LLM (e.g., deepseek-r1:7b) + OLLAMA_EMBEDDING_MODEL: Model for embeddings (e.g., nomic-embed-text) + OLLAMA_EMBEDDING_DIM: Embedding dimension (required for Ollama, e.g., 768) + + # FalkorDB + GRAPHITI_FALKORDB_HOST: FalkorDB host (default: localhost) + GRAPHITI_FALKORDB_PORT: FalkorDB port (default: 6380) + GRAPHITI_FALKORDB_PASSWORD: FalkorDB password (default: empty) + GRAPHITI_DATABASE: Graph database name (default: auto_claude_memory) + GRAPHITI_TELEMETRY_ENABLED: Set to "false" to disable telemetry (default: true) +""" + +import json +import os +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from pathlib import Path +from typing import Optional + +# Default configuration values +DEFAULT_FALKORDB_HOST = "localhost" +DEFAULT_FALKORDB_PORT = 6380 +DEFAULT_DATABASE = "auto_claude_memory" +DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434" + +# Graphiti state marker file (stores connection info and status) +GRAPHITI_STATE_MARKER = ".graphiti_state.json" + +# Episode types for different memory categories +EPISODE_TYPE_SESSION_INSIGHT = "session_insight" +EPISODE_TYPE_CODEBASE_DISCOVERY = "codebase_discovery" +EPISODE_TYPE_PATTERN = "pattern" +EPISODE_TYPE_GOTCHA = "gotcha" +EPISODE_TYPE_TASK_OUTCOME = "task_outcome" +EPISODE_TYPE_QA_RESULT = "qa_result" +EPISODE_TYPE_HISTORICAL_CONTEXT = "historical_context" + + +class LLMProvider(str, Enum): + """Supported LLM providers for Graphiti.""" + + OPENAI = "openai" + ANTHROPIC = "anthropic" + AZURE_OPENAI = "azure_openai" + OLLAMA = "ollama" + + +class EmbedderProvider(str, Enum): + """Supported embedder providers for Graphiti.""" + + OPENAI = "openai" + VOYAGE = "voyage" + AZURE_OPENAI = "azure_openai" + OLLAMA = "ollama" + + +@dataclass +class GraphitiConfig: + """Configuration for Graphiti memory integration with multi-provider support.""" + + # Core settings + enabled: bool = False + llm_provider: str = "openai" + embedder_provider: str = "openai" + + # FalkorDB connection + falkordb_host: str = DEFAULT_FALKORDB_HOST + falkordb_port: int = DEFAULT_FALKORDB_PORT + falkordb_password: str = "" + database: str = DEFAULT_DATABASE + telemetry_enabled: bool = True + + # OpenAI settings + openai_api_key: str = "" + openai_model: str = "gpt-5-mini" + openai_embedding_model: str = "text-embedding-3-small" + + # Anthropic settings (LLM only) + anthropic_api_key: str = "" + anthropic_model: str = "claude-sonnet-4-5-latest" + + # Azure OpenAI settings + azure_openai_api_key: str = "" + azure_openai_base_url: str = "" + azure_openai_llm_deployment: str = "" + azure_openai_embedding_deployment: str = "" + + # Voyage AI settings (embeddings only) + voyage_api_key: str = "" + voyage_embedding_model: str = "voyage-3" + + # Ollama settings (local) + ollama_base_url: str = DEFAULT_OLLAMA_BASE_URL + ollama_llm_model: str = "" + ollama_embedding_model: str = "" + ollama_embedding_dim: int = 0 # Required for Ollama embeddings + + @classmethod + def from_env(cls) -> "GraphitiConfig": + """Create config from environment variables.""" + # Check if Graphiti is explicitly enabled + enabled_str = os.environ.get("GRAPHITI_ENABLED", "").lower() + enabled = enabled_str in ("true", "1", "yes") + + # Provider selection + llm_provider = os.environ.get("GRAPHITI_LLM_PROVIDER", "openai").lower() + embedder_provider = os.environ.get( + "GRAPHITI_EMBEDDER_PROVIDER", "openai" + ).lower() + + # FalkorDB connection settings + falkordb_host = os.environ.get("GRAPHITI_FALKORDB_HOST", DEFAULT_FALKORDB_HOST) + + try: + falkordb_port = int( + os.environ.get("GRAPHITI_FALKORDB_PORT", str(DEFAULT_FALKORDB_PORT)) + ) + except ValueError: + falkordb_port = DEFAULT_FALKORDB_PORT + + falkordb_password = os.environ.get("GRAPHITI_FALKORDB_PASSWORD", "") + database = os.environ.get("GRAPHITI_DATABASE", DEFAULT_DATABASE) + + # Telemetry setting + telemetry_str = os.environ.get("GRAPHITI_TELEMETRY_ENABLED", "true").lower() + telemetry_enabled = telemetry_str not in ("false", "0", "no") + + # OpenAI settings + openai_api_key = os.environ.get("OPENAI_API_KEY", "") + openai_model = os.environ.get("OPENAI_MODEL", "gpt-5-mini") + openai_embedding_model = os.environ.get( + "OPENAI_EMBEDDING_MODEL", "text-embedding-3-small" + ) + + # Anthropic settings + anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY", "") + anthropic_model = os.environ.get( + "GRAPHITI_ANTHROPIC_MODEL", "claude-sonnet-4-5-latest" + ) + + # Azure OpenAI settings + azure_openai_api_key = os.environ.get("AZURE_OPENAI_API_KEY", "") + azure_openai_base_url = os.environ.get("AZURE_OPENAI_BASE_URL", "") + azure_openai_llm_deployment = os.environ.get("AZURE_OPENAI_LLM_DEPLOYMENT", "") + azure_openai_embedding_deployment = os.environ.get( + "AZURE_OPENAI_EMBEDDING_DEPLOYMENT", "" + ) + + # Voyage AI settings + voyage_api_key = os.environ.get("VOYAGE_API_KEY", "") + voyage_embedding_model = os.environ.get("VOYAGE_EMBEDDING_MODEL", "voyage-3") + + # Ollama settings + ollama_base_url = os.environ.get("OLLAMA_BASE_URL", DEFAULT_OLLAMA_BASE_URL) + ollama_llm_model = os.environ.get("OLLAMA_LLM_MODEL", "") + ollama_embedding_model = os.environ.get("OLLAMA_EMBEDDING_MODEL", "") + + # Ollama embedding dimension (required for Ollama) + try: + ollama_embedding_dim = int(os.environ.get("OLLAMA_EMBEDDING_DIM", "0")) + except ValueError: + ollama_embedding_dim = 0 + + return cls( + enabled=enabled, + llm_provider=llm_provider, + embedder_provider=embedder_provider, + falkordb_host=falkordb_host, + falkordb_port=falkordb_port, + falkordb_password=falkordb_password, + database=database, + telemetry_enabled=telemetry_enabled, + openai_api_key=openai_api_key, + openai_model=openai_model, + openai_embedding_model=openai_embedding_model, + anthropic_api_key=anthropic_api_key, + anthropic_model=anthropic_model, + azure_openai_api_key=azure_openai_api_key, + azure_openai_base_url=azure_openai_base_url, + azure_openai_llm_deployment=azure_openai_llm_deployment, + azure_openai_embedding_deployment=azure_openai_embedding_deployment, + voyage_api_key=voyage_api_key, + voyage_embedding_model=voyage_embedding_model, + ollama_base_url=ollama_base_url, + ollama_llm_model=ollama_llm_model, + ollama_embedding_model=ollama_embedding_model, + ollama_embedding_dim=ollama_embedding_dim, + ) + + def is_valid(self) -> bool: + """ + Check if config has minimum required values for operation. + + Returns True if: + - GRAPHITI_ENABLED is true + - LLM provider is configured correctly + - Embedder provider is configured correctly + """ + if not self.enabled: + return False + + # Validate LLM provider + if not self._validate_llm_provider(): + return False + + # Validate embedder provider + if not self._validate_embedder_provider(): + return False + + return True + + def _validate_llm_provider(self) -> bool: + """Validate LLM provider configuration.""" + if self.llm_provider == "openai": + return bool(self.openai_api_key) + elif self.llm_provider == "anthropic": + return bool(self.anthropic_api_key) + elif self.llm_provider == "azure_openai": + return bool( + self.azure_openai_api_key + and self.azure_openai_base_url + and self.azure_openai_llm_deployment + ) + elif self.llm_provider == "ollama": + return bool(self.ollama_llm_model) + return False + + def _validate_embedder_provider(self) -> bool: + """Validate embedder provider configuration.""" + if self.embedder_provider == "openai": + return bool(self.openai_api_key) + elif self.embedder_provider == "voyage": + return bool(self.voyage_api_key) + elif self.embedder_provider == "azure_openai": + return bool( + self.azure_openai_api_key + and self.azure_openai_base_url + and self.azure_openai_embedding_deployment + ) + elif self.embedder_provider == "ollama": + return bool(self.ollama_embedding_model and self.ollama_embedding_dim) + return False + + def get_validation_errors(self) -> list[str]: + """Get list of validation errors for current configuration.""" + errors = [] + + if not self.enabled: + errors.append("GRAPHITI_ENABLED must be set to true") + return errors + + # LLM provider validation + if self.llm_provider == "openai": + if not self.openai_api_key: + errors.append("OpenAI LLM provider requires OPENAI_API_KEY") + elif self.llm_provider == "anthropic": + if not self.anthropic_api_key: + errors.append("Anthropic LLM provider requires ANTHROPIC_API_KEY") + elif self.llm_provider == "azure_openai": + if not self.azure_openai_api_key: + errors.append("Azure OpenAI LLM provider requires AZURE_OPENAI_API_KEY") + if not self.azure_openai_base_url: + errors.append( + "Azure OpenAI LLM provider requires AZURE_OPENAI_BASE_URL" + ) + if not self.azure_openai_llm_deployment: + errors.append( + "Azure OpenAI LLM provider requires AZURE_OPENAI_LLM_DEPLOYMENT" + ) + elif self.llm_provider == "ollama": + if not self.ollama_llm_model: + errors.append("Ollama LLM provider requires OLLAMA_LLM_MODEL") + else: + errors.append(f"Unknown LLM provider: {self.llm_provider}") + + # Embedder provider validation + if self.embedder_provider == "openai": + if not self.openai_api_key: + errors.append("OpenAI embedder provider requires OPENAI_API_KEY") + elif self.embedder_provider == "voyage": + if not self.voyage_api_key: + errors.append("Voyage embedder provider requires VOYAGE_API_KEY") + elif self.embedder_provider == "azure_openai": + if not self.azure_openai_api_key: + errors.append( + "Azure OpenAI embedder provider requires AZURE_OPENAI_API_KEY" + ) + if not self.azure_openai_base_url: + errors.append( + "Azure OpenAI embedder provider requires AZURE_OPENAI_BASE_URL" + ) + if not self.azure_openai_embedding_deployment: + errors.append( + "Azure OpenAI embedder provider requires AZURE_OPENAI_EMBEDDING_DEPLOYMENT" + ) + elif self.embedder_provider == "ollama": + if not self.ollama_embedding_model: + errors.append( + "Ollama embedder provider requires OLLAMA_EMBEDDING_MODEL" + ) + if not self.ollama_embedding_dim: + errors.append("Ollama embedder provider requires OLLAMA_EMBEDDING_DIM") + else: + errors.append(f"Unknown embedder provider: {self.embedder_provider}") + + return errors + + def get_connection_uri(self) -> str: + """Get the FalkorDB connection URI.""" + if self.falkordb_password: + return f"redis://:{self.falkordb_password}@{self.falkordb_host}:{self.falkordb_port}" + return f"redis://{self.falkordb_host}:{self.falkordb_port}" + + def get_provider_summary(self) -> str: + """Get a summary of configured providers.""" + return f"LLM: {self.llm_provider}, Embedder: {self.embedder_provider}" + + +@dataclass +class GraphitiState: + """State of Graphiti integration for an auto-claude spec.""" + + initialized: bool = False + database: str | None = None + indices_built: bool = False + created_at: str | None = None + last_session: int | None = None + episode_count: int = 0 + error_log: list = field(default_factory=list) + # V2 additions + llm_provider: str | None = None + embedder_provider: str | None = None + + def to_dict(self) -> dict: + return { + "initialized": self.initialized, + "database": self.database, + "indices_built": self.indices_built, + "created_at": self.created_at, + "last_session": self.last_session, + "episode_count": self.episode_count, + "error_log": self.error_log[-10:], # Keep last 10 errors + "llm_provider": self.llm_provider, + "embedder_provider": self.embedder_provider, + } + + @classmethod + def from_dict(cls, data: dict) -> "GraphitiState": + return cls( + initialized=data.get("initialized", False), + database=data.get("database"), + indices_built=data.get("indices_built", False), + created_at=data.get("created_at"), + last_session=data.get("last_session"), + episode_count=data.get("episode_count", 0), + error_log=data.get("error_log", []), + llm_provider=data.get("llm_provider"), + embedder_provider=data.get("embedder_provider"), + ) + + def save(self, spec_dir: Path) -> None: + """Save state to the spec directory.""" + marker_file = spec_dir / GRAPHITI_STATE_MARKER + with open(marker_file, "w") as f: + json.dump(self.to_dict(), f, indent=2) + + @classmethod + def load(cls, spec_dir: Path) -> Optional["GraphitiState"]: + """Load state from the spec directory.""" + marker_file = spec_dir / GRAPHITI_STATE_MARKER + if not marker_file.exists(): + return None + + try: + with open(marker_file) as f: + return cls.from_dict(json.load(f)) + except (OSError, json.JSONDecodeError): + return None + + def record_error(self, error_msg: str) -> None: + """Record an error in the state.""" + self.error_log.append( + { + "timestamp": datetime.now().isoformat(), + "error": error_msg[:500], # Limit error message length + } + ) + # Keep only last 10 errors + self.error_log = self.error_log[-10:] + + +def is_graphiti_enabled() -> bool: + """ + Quick check if Graphiti integration is available. + + Returns True if: + - GRAPHITI_ENABLED is set to true/1/yes + - Required provider credentials are configured + """ + config = GraphitiConfig.from_env() + return config.is_valid() + + +def get_graphiti_status() -> dict: + """ + Get the current Graphiti integration status. + + Returns: + Dict with status information: + - enabled: bool + - available: bool (has required dependencies) + - host: str + - port: int + - database: str + - llm_provider: str + - embedder_provider: str + - reason: str (why unavailable if not available) + - errors: list (validation errors if any) + """ + config = GraphitiConfig.from_env() + + status = { + "enabled": config.enabled, + "available": False, + "host": config.falkordb_host, + "port": config.falkordb_port, + "database": config.database, + "llm_provider": config.llm_provider, + "embedder_provider": config.embedder_provider, + "reason": "", + "errors": [], + } + + if not config.enabled: + status["reason"] = "GRAPHITI_ENABLED not set to true" + return status + + # Get validation errors + errors = config.get_validation_errors() + if errors: + status["errors"] = errors + status["reason"] = errors[0] # First error as primary reason + return status + + status["available"] = True + return status + + +def get_available_providers() -> dict: + """ + Get list of available providers based on current environment. + + Returns: + Dict with lists of available LLM and embedder providers + """ + config = GraphitiConfig.from_env() + + available_llm = [] + available_embedder = [] + + # Check OpenAI + if config.openai_api_key: + available_llm.append("openai") + available_embedder.append("openai") + + # Check Anthropic + if config.anthropic_api_key: + available_llm.append("anthropic") + + # Check Azure OpenAI + if config.azure_openai_api_key and config.azure_openai_base_url: + if config.azure_openai_llm_deployment: + available_llm.append("azure_openai") + if config.azure_openai_embedding_deployment: + available_embedder.append("azure_openai") + + # Check Voyage + if config.voyage_api_key: + available_embedder.append("voyage") + + # Check Ollama + if config.ollama_llm_model: + available_llm.append("ollama") + if config.ollama_embedding_model and config.ollama_embedding_dim: + available_embedder.append("ollama") + + return { + "llm_providers": available_llm, + "embedder_providers": available_embedder, + } + + +def validate_graphiti_config() -> tuple[bool, list[str]]: + """ + Validate Graphiti configuration from environment. + + Returns: + Tuple of (is_valid, error_messages) + - is_valid: True if configuration is valid + - error_messages: List of validation error messages (empty if valid) + """ + config = GraphitiConfig.from_env() + + if not config.is_valid(): + errors = config.get_validation_errors() + return False, errors + + return True, [] diff --git a/auto-claude/graphiti_memory.py b/auto-claude/integrations/graphiti/memory.py similarity index 100% rename from auto-claude/graphiti_memory.py rename to auto-claude/integrations/graphiti/memory.py diff --git a/auto-claude/integrations/graphiti/providers.py b/auto-claude/integrations/graphiti/providers.py new file mode 100644 index 00000000..87affdec --- /dev/null +++ b/auto-claude/integrations/graphiti/providers.py @@ -0,0 +1,70 @@ +""" +Graphiti Multi-Provider Entry Point +==================================== + +Main entry point for Graphiti provider functionality. +This module re-exports all functionality from the graphiti_providers package. + +The actual implementation has been refactored into a package structure: +- graphiti_providers/exceptions.py - Provider exceptions +- graphiti_providers/models.py - Embedding dimensions and constants +- graphiti_providers/llm_providers/ - LLM provider implementations +- graphiti_providers/embedder_providers/ - Embedder provider implementations +- graphiti_providers/cross_encoder.py - Cross-encoder/reranker +- graphiti_providers/validators.py - Validation and health checks +- graphiti_providers/utils.py - Utility functions +- graphiti_providers/factory.py - Factory functions + +For backward compatibility, this module re-exports all public APIs. + +Usage: + from graphiti_providers import create_llm_client, create_embedder + from graphiti_config import GraphitiConfig + + config = GraphitiConfig.from_env() + llm_client = create_llm_client(config) + embedder = create_embedder(config) +""" + +# Re-export all public APIs from the package +from graphiti_providers import ( + # Exceptions + ProviderError, + ProviderNotInstalled, + # Factory functions + create_llm_client, + create_embedder, + create_cross_encoder, + # Models + EMBEDDING_DIMENSIONS, + get_expected_embedding_dim, + # Validators + validate_embedding_config, + test_llm_connection, + test_embedder_connection, + test_ollama_connection, + # Utilities + is_graphiti_enabled, + get_graph_hints, +) + +__all__ = [ + # Exceptions + "ProviderError", + "ProviderNotInstalled", + # Factory functions + "create_llm_client", + "create_embedder", + "create_cross_encoder", + # Models + "EMBEDDING_DIMENSIONS", + "get_expected_embedding_dim", + # Validators + "validate_embedding_config", + "test_llm_connection", + "test_embedder_connection", + "test_ollama_connection", + # Utilities + "is_graphiti_enabled", + "get_graph_hints", +] diff --git a/auto-claude/graphiti_providers/__init__.py b/auto-claude/integrations/graphiti/providers_pkg/__init__.py similarity index 100% rename from auto-claude/graphiti_providers/__init__.py rename to auto-claude/integrations/graphiti/providers_pkg/__init__.py diff --git a/auto-claude/graphiti_providers/cross_encoder.py b/auto-claude/integrations/graphiti/providers_pkg/cross_encoder.py similarity index 100% rename from auto-claude/graphiti_providers/cross_encoder.py rename to auto-claude/integrations/graphiti/providers_pkg/cross_encoder.py diff --git a/auto-claude/graphiti_providers/embedder_providers/__init__.py b/auto-claude/integrations/graphiti/providers_pkg/embedder_providers/__init__.py similarity index 100% rename from auto-claude/graphiti_providers/embedder_providers/__init__.py rename to auto-claude/integrations/graphiti/providers_pkg/embedder_providers/__init__.py diff --git a/auto-claude/graphiti_providers/embedder_providers/azure_openai_embedder.py b/auto-claude/integrations/graphiti/providers_pkg/embedder_providers/azure_openai_embedder.py similarity index 100% rename from auto-claude/graphiti_providers/embedder_providers/azure_openai_embedder.py rename to auto-claude/integrations/graphiti/providers_pkg/embedder_providers/azure_openai_embedder.py diff --git a/auto-claude/graphiti_providers/embedder_providers/ollama_embedder.py b/auto-claude/integrations/graphiti/providers_pkg/embedder_providers/ollama_embedder.py similarity index 100% rename from auto-claude/graphiti_providers/embedder_providers/ollama_embedder.py rename to auto-claude/integrations/graphiti/providers_pkg/embedder_providers/ollama_embedder.py diff --git a/auto-claude/graphiti_providers/embedder_providers/openai_embedder.py b/auto-claude/integrations/graphiti/providers_pkg/embedder_providers/openai_embedder.py similarity index 100% rename from auto-claude/graphiti_providers/embedder_providers/openai_embedder.py rename to auto-claude/integrations/graphiti/providers_pkg/embedder_providers/openai_embedder.py diff --git a/auto-claude/graphiti_providers/embedder_providers/voyage_embedder.py b/auto-claude/integrations/graphiti/providers_pkg/embedder_providers/voyage_embedder.py similarity index 100% rename from auto-claude/graphiti_providers/embedder_providers/voyage_embedder.py rename to auto-claude/integrations/graphiti/providers_pkg/embedder_providers/voyage_embedder.py diff --git a/auto-claude/graphiti_providers/exceptions.py b/auto-claude/integrations/graphiti/providers_pkg/exceptions.py similarity index 100% rename from auto-claude/graphiti_providers/exceptions.py rename to auto-claude/integrations/graphiti/providers_pkg/exceptions.py diff --git a/auto-claude/graphiti_providers/factory.py b/auto-claude/integrations/graphiti/providers_pkg/factory.py similarity index 100% rename from auto-claude/graphiti_providers/factory.py rename to auto-claude/integrations/graphiti/providers_pkg/factory.py diff --git a/auto-claude/graphiti_providers/llm_providers/__init__.py b/auto-claude/integrations/graphiti/providers_pkg/llm_providers/__init__.py similarity index 100% rename from auto-claude/graphiti_providers/llm_providers/__init__.py rename to auto-claude/integrations/graphiti/providers_pkg/llm_providers/__init__.py diff --git a/auto-claude/graphiti_providers/llm_providers/anthropic_llm.py b/auto-claude/integrations/graphiti/providers_pkg/llm_providers/anthropic_llm.py similarity index 100% rename from auto-claude/graphiti_providers/llm_providers/anthropic_llm.py rename to auto-claude/integrations/graphiti/providers_pkg/llm_providers/anthropic_llm.py diff --git a/auto-claude/graphiti_providers/llm_providers/azure_openai_llm.py b/auto-claude/integrations/graphiti/providers_pkg/llm_providers/azure_openai_llm.py similarity index 100% rename from auto-claude/graphiti_providers/llm_providers/azure_openai_llm.py rename to auto-claude/integrations/graphiti/providers_pkg/llm_providers/azure_openai_llm.py diff --git a/auto-claude/graphiti_providers/llm_providers/ollama_llm.py b/auto-claude/integrations/graphiti/providers_pkg/llm_providers/ollama_llm.py similarity index 100% rename from auto-claude/graphiti_providers/llm_providers/ollama_llm.py rename to auto-claude/integrations/graphiti/providers_pkg/llm_providers/ollama_llm.py diff --git a/auto-claude/graphiti_providers/llm_providers/openai_llm.py b/auto-claude/integrations/graphiti/providers_pkg/llm_providers/openai_llm.py similarity index 100% rename from auto-claude/graphiti_providers/llm_providers/openai_llm.py rename to auto-claude/integrations/graphiti/providers_pkg/llm_providers/openai_llm.py diff --git a/auto-claude/graphiti_providers/models.py b/auto-claude/integrations/graphiti/providers_pkg/models.py similarity index 100% rename from auto-claude/graphiti_providers/models.py rename to auto-claude/integrations/graphiti/providers_pkg/models.py diff --git a/auto-claude/graphiti_providers/utils.py b/auto-claude/integrations/graphiti/providers_pkg/utils.py similarity index 100% rename from auto-claude/graphiti_providers/utils.py rename to auto-claude/integrations/graphiti/providers_pkg/utils.py diff --git a/auto-claude/graphiti_providers/validators.py b/auto-claude/integrations/graphiti/providers_pkg/validators.py similarity index 100% rename from auto-claude/graphiti_providers/validators.py rename to auto-claude/integrations/graphiti/providers_pkg/validators.py diff --git a/auto-claude/graphiti/__init__.py b/auto-claude/integrations/graphiti/queries_pkg/__init__.py similarity index 100% rename from auto-claude/graphiti/__init__.py rename to auto-claude/integrations/graphiti/queries_pkg/__init__.py diff --git a/auto-claude/graphiti/client.py b/auto-claude/integrations/graphiti/queries_pkg/client.py similarity index 100% rename from auto-claude/graphiti/client.py rename to auto-claude/integrations/graphiti/queries_pkg/client.py diff --git a/auto-claude/graphiti/graphiti.py b/auto-claude/integrations/graphiti/queries_pkg/graphiti.py similarity index 100% rename from auto-claude/graphiti/graphiti.py rename to auto-claude/integrations/graphiti/queries_pkg/graphiti.py diff --git a/auto-claude/graphiti/queries.py b/auto-claude/integrations/graphiti/queries_pkg/queries.py similarity index 90% rename from auto-claude/graphiti/queries.py rename to auto-claude/integrations/graphiti/queries_pkg/queries.py index 52f43364..3ec5db70 100644 --- a/auto-claude/graphiti/queries.py +++ b/auto-claude/integrations/graphiti/queries_pkg/queries.py @@ -291,7 +291,11 @@ class GraphitiQueries: ) saved_count += 1 except Exception as e: - logger.debug(f"Failed to save file insight: {e}") + if "duplicate_facts" in str(e): + logger.debug(f"Graphiti deduplication warning (non-fatal): {e}") + saved_count += 1 + else: + logger.debug(f"Failed to save file insight: {e}") # 2. Save patterns for pattern in insights.get("patterns_discovered", []): @@ -330,7 +334,11 @@ class GraphitiQueries: ) saved_count += 1 except Exception as e: - logger.debug(f"Failed to save pattern: {e}") + if "duplicate_facts" in str(e): + logger.debug(f"Graphiti deduplication warning (non-fatal): {e}") + saved_count += 1 + else: + logger.debug(f"Failed to save pattern: {e}") # 3. Save gotchas for gotcha in insights.get("gotchas_discovered", []): @@ -367,7 +375,11 @@ class GraphitiQueries: ) saved_count += 1 except Exception as e: - logger.debug(f"Failed to save gotcha: {e}") + if "duplicate_facts" in str(e): + logger.debug(f"Graphiti deduplication warning (non-fatal): {e}") + saved_count += 1 + else: + logger.debug(f"Failed to save gotcha: {e}") # 4. Save approach outcome outcome = insights.get("approach_outcome", {}) @@ -400,7 +412,13 @@ class GraphitiQueries: ) saved_count += 1 except Exception as e: - logger.debug(f"Failed to save task outcome: {e}") + # Graphiti deduplication can fail with "invalid duplicate_facts idx" + # This is a known issue in graphiti-core - episode is still partially saved + if "duplicate_facts" in str(e): + logger.debug(f"Graphiti deduplication warning (non-fatal): {e}") + saved_count += 1 # Episode likely saved, just dedup failed + else: + logger.debug(f"Failed to save task outcome: {e}") # 5. Save recommendations recommendations = insights.get("recommendations", []) @@ -427,7 +445,11 @@ class GraphitiQueries: ) saved_count += 1 except Exception as e: - logger.debug(f"Failed to save recommendations: {e}") + if "duplicate_facts" in str(e): + logger.debug(f"Graphiti deduplication warning (non-fatal): {e}") + saved_count += 1 + else: + logger.debug(f"Failed to save recommendations: {e}") logger.info( f"Saved {saved_count}/{total_count} structured insights to Graphiti " diff --git a/auto-claude/graphiti/schema.py b/auto-claude/integrations/graphiti/queries_pkg/schema.py similarity index 100% rename from auto-claude/graphiti/schema.py rename to auto-claude/integrations/graphiti/queries_pkg/schema.py diff --git a/auto-claude/graphiti/search.py b/auto-claude/integrations/graphiti/queries_pkg/search.py similarity index 100% rename from auto-claude/graphiti/search.py rename to auto-claude/integrations/graphiti/queries_pkg/search.py diff --git a/auto-claude/test_graphiti_memory.py b/auto-claude/integrations/graphiti/test_graphiti_memory.py similarity index 100% rename from auto-claude/test_graphiti_memory.py rename to auto-claude/integrations/graphiti/test_graphiti_memory.py diff --git a/auto-claude/integrations/linear/__init__.py b/auto-claude/integrations/linear/__init__.py new file mode 100644 index 00000000..5bf91085 --- /dev/null +++ b/auto-claude/integrations/linear/__init__.py @@ -0,0 +1,42 @@ +""" +Linear Integration +================== + +Integration with Linear issue tracking. +""" + +from .config import LinearConfig +from .integration import LinearManager +from .updater import ( + LinearTaskState, + is_linear_enabled, + get_linear_api_key, + create_linear_task, + update_linear_status, + STATUS_TODO, + STATUS_IN_PROGRESS, + STATUS_IN_REVIEW, + STATUS_DONE, + STATUS_CANCELED, +) + +# Aliases for backward compatibility +LinearIntegration = LinearManager +LinearUpdater = LinearTaskState # Alias - old code may expect this name + +__all__ = [ + "LinearConfig", + "LinearManager", + "LinearIntegration", + "LinearTaskState", + "LinearUpdater", + "is_linear_enabled", + "get_linear_api_key", + "create_linear_task", + "update_linear_status", + "STATUS_TODO", + "STATUS_IN_PROGRESS", + "STATUS_IN_REVIEW", + "STATUS_DONE", + "STATUS_CANCELED", +] diff --git a/auto-claude/integrations/linear/config.py b/auto-claude/integrations/linear/config.py new file mode 100644 index 00000000..25bd149f --- /dev/null +++ b/auto-claude/integrations/linear/config.py @@ -0,0 +1,342 @@ +""" +Linear Integration Configuration +================================ + +Constants, status mappings, and configuration helpers for Linear integration. +Mirrors the approach from Linear-Coding-Agent-Harness. +""" + +import json +import os +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Optional + +# Linear Status Constants (map to Linear workflow states) +STATUS_TODO = "Todo" +STATUS_IN_PROGRESS = "In Progress" +STATUS_DONE = "Done" +STATUS_BLOCKED = "Blocked" # For stuck subtasks +STATUS_CANCELED = "Canceled" + +# Linear Priority Constants (1=Urgent, 4=Low, 0=No priority) +PRIORITY_URGENT = 1 # Core infrastructure, blockers +PRIORITY_HIGH = 2 # Primary features, dependencies +PRIORITY_MEDIUM = 3 # Secondary features +PRIORITY_LOW = 4 # Polish, nice-to-haves +PRIORITY_NONE = 0 # No priority set + +# Subtask status to Linear status mapping +SUBTASK_TO_LINEAR_STATUS = { + "pending": STATUS_TODO, + "in_progress": STATUS_IN_PROGRESS, + "completed": STATUS_DONE, + "blocked": STATUS_BLOCKED, + "failed": STATUS_BLOCKED, # Map failures to Blocked for visibility + "stuck": STATUS_BLOCKED, +} + +# Linear labels for categorization +LABELS = { + "phase": "phase", # Phase label prefix (e.g., "phase-1") + "service": "service", # Service label prefix (e.g., "service-backend") + "stuck": "stuck", # Mark stuck subtasks + "auto_build": "auto-claude", # All auto-claude issues + "needs_review": "needs-review", +} + +# Linear project marker file (stores team/project IDs) +LINEAR_PROJECT_MARKER = ".linear_project.json" + +# Meta issue for session tracking +META_ISSUE_TITLE = "[META] Build Progress Tracker" + + +@dataclass +class LinearConfig: + """Configuration for Linear integration.""" + + api_key: str + team_id: str | None = None + project_id: str | None = None + project_name: str | None = None + meta_issue_id: str | None = None + enabled: bool = True + + @classmethod + def from_env(cls) -> "LinearConfig": + """Create config from environment variables.""" + api_key = os.environ.get("LINEAR_API_KEY", "") + + return cls( + api_key=api_key, + team_id=os.environ.get("LINEAR_TEAM_ID"), + project_id=os.environ.get("LINEAR_PROJECT_ID"), + enabled=bool(api_key), + ) + + def is_valid(self) -> bool: + """Check if config has minimum required values.""" + return bool(self.api_key) + + +@dataclass +class LinearProjectState: + """State of a Linear project for an auto-claude spec.""" + + initialized: bool = False + team_id: str | None = None + project_id: str | None = None + project_name: str | None = None + meta_issue_id: str | None = None + total_issues: int = 0 + created_at: str | None = None + issue_mapping: dict = None # subtask_id -> issue_id mapping + + def __post_init__(self): + if self.issue_mapping is None: + self.issue_mapping = {} + + def to_dict(self) -> dict: + return { + "initialized": self.initialized, + "team_id": self.team_id, + "project_id": self.project_id, + "project_name": self.project_name, + "meta_issue_id": self.meta_issue_id, + "total_issues": self.total_issues, + "created_at": self.created_at, + "issue_mapping": self.issue_mapping, + } + + @classmethod + def from_dict(cls, data: dict) -> "LinearProjectState": + return cls( + initialized=data.get("initialized", False), + team_id=data.get("team_id"), + project_id=data.get("project_id"), + project_name=data.get("project_name"), + meta_issue_id=data.get("meta_issue_id"), + total_issues=data.get("total_issues", 0), + created_at=data.get("created_at"), + issue_mapping=data.get("issue_mapping", {}), + ) + + def save(self, spec_dir: Path) -> None: + """Save state to the spec directory.""" + marker_file = spec_dir / LINEAR_PROJECT_MARKER + with open(marker_file, "w") as f: + json.dump(self.to_dict(), f, indent=2) + + @classmethod + def load(cls, spec_dir: Path) -> Optional["LinearProjectState"]: + """Load state from the spec directory.""" + marker_file = spec_dir / LINEAR_PROJECT_MARKER + if not marker_file.exists(): + return None + + try: + with open(marker_file) as f: + return cls.from_dict(json.load(f)) + except (OSError, json.JSONDecodeError): + return None + + +def get_linear_status(subtask_status: str) -> str: + """ + Map subtask status to Linear status. + + Args: + subtask_status: Status from implementation_plan.json + + Returns: + Corresponding Linear status string + """ + return SUBTASK_TO_LINEAR_STATUS.get(subtask_status, STATUS_TODO) + + +def get_priority_for_phase(phase_num: int, total_phases: int) -> int: + """ + Determine Linear priority based on phase number. + + Early phases are higher priority (they're dependencies). + + Args: + phase_num: Phase number (1-indexed) + total_phases: Total number of phases + + Returns: + Linear priority value (1-4) + """ + if total_phases <= 1: + return PRIORITY_HIGH + + # First quarter of phases = Urgent + # Second quarter = High + # Third quarter = Medium + # Fourth quarter = Low + position = phase_num / total_phases + + if position <= 0.25: + return PRIORITY_URGENT + elif position <= 0.5: + return PRIORITY_HIGH + elif position <= 0.75: + return PRIORITY_MEDIUM + else: + return PRIORITY_LOW + + +def format_subtask_description(subtask: dict, phase: dict = None) -> str: + """ + Format a subtask as a Linear issue description. + + Args: + subtask: Subtask dict from implementation_plan.json + phase: Optional phase dict for context + + Returns: + Markdown-formatted description + """ + lines = [] + + # Description + if subtask.get("description"): + lines.append(f"## Description\n{subtask['description']}\n") + + # Service + if subtask.get("service"): + lines.append(f"**Service:** {subtask['service']}") + elif subtask.get("all_services"): + lines.append("**Scope:** All services (integration)") + + # Phase info + if phase: + lines.append(f"**Phase:** {phase.get('name', phase.get('id', 'Unknown'))}") + + # Files to modify + if subtask.get("files_to_modify"): + lines.append("\n## Files to Modify") + for f in subtask["files_to_modify"]: + lines.append(f"- `{f}`") + + # Files to create + if subtask.get("files_to_create"): + lines.append("\n## Files to Create") + for f in subtask["files_to_create"]: + lines.append(f"- `{f}`") + + # Patterns to follow + if subtask.get("patterns_from"): + lines.append("\n## Reference Patterns") + for f in subtask["patterns_from"]: + lines.append(f"- `{f}`") + + # Verification + if subtask.get("verification"): + v = subtask["verification"] + lines.append("\n## Verification") + lines.append(f"**Type:** {v.get('type', 'none')}") + if v.get("run"): + lines.append(f"**Command:** `{v['run']}`") + if v.get("url"): + lines.append(f"**URL:** {v['url']}") + if v.get("scenario"): + lines.append(f"**Scenario:** {v['scenario']}") + + # Auto-build metadata + lines.append("\n---") + lines.append("*This issue was created by the Auto-Build Framework*") + + return "\n".join(lines) + + +def format_session_comment( + session_num: int, + subtask_id: str, + success: bool, + approach: str = "", + error: str = "", + git_commit: str = "", +) -> str: + """ + Format a session result as a Linear comment. + + Args: + session_num: Session number + subtask_id: Subtask being worked on + success: Whether the session succeeded + approach: What was attempted + error: Error message if failed + git_commit: Git commit hash if any + + Returns: + Markdown-formatted comment + """ + status_emoji = "✅" if success else "❌" + lines = [ + f"## Session #{session_num} {status_emoji}", + f"**Subtask:** `{subtask_id}`", + f"**Status:** {'Completed' if success else 'In Progress'}", + f"**Time:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + ] + + if approach: + lines.append(f"\n**Approach:** {approach}") + + if git_commit: + lines.append(f"\n**Commit:** `{git_commit[:8]}`") + + if error: + lines.append(f"\n**Error:**\n```\n{error[:500]}\n```") + + return "\n".join(lines) + + +def format_stuck_subtask_comment( + subtask_id: str, + attempt_count: int, + attempts: list[dict], + reason: str = "", +) -> str: + """ + Format a detailed comment for stuck subtasks. + + Args: + subtask_id: Stuck subtask ID + attempt_count: Number of attempts + attempts: List of attempt records + reason: Why it's stuck + + Returns: + Markdown-formatted comment for escalation + """ + lines = [ + "## ⚠️ Subtask Marked as STUCK", + f"**Subtask:** `{subtask_id}`", + f"**Attempts:** {attempt_count}", + f"**Time:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + ] + + if reason: + lines.append(f"\n**Reason:** {reason}") + + # Add attempt history + if attempts: + lines.append("\n### Attempt History") + for i, attempt in enumerate(attempts[-5:], 1): # Last 5 attempts + status = "✅" if attempt.get("success") else "❌" + lines.append(f"\n**Attempt {i}:** {status}") + if attempt.get("approach"): + lines.append(f"- Approach: {attempt['approach'][:200]}") + if attempt.get("error"): + lines.append(f"- Error: {attempt['error'][:200]}") + + lines.append("\n### Recommended Actions") + lines.append("1. Review the approach and error patterns above") + lines.append("2. Check for missing dependencies or configuration") + lines.append("3. Consider manual intervention or different approach") + lines.append("4. Update HUMAN_INPUT.md with guidance for the agent") + + return "\n".join(lines) diff --git a/auto-claude/integrations/linear/integration.py b/auto-claude/integrations/linear/integration.py new file mode 100644 index 00000000..a31b98f2 --- /dev/null +++ b/auto-claude/integrations/linear/integration.py @@ -0,0 +1,553 @@ +""" +Linear Integration Manager +========================== + +Manages synchronization between Auto-Build subtasks and Linear issues. +Provides real-time visibility into build progress through Linear. + +The integration is OPTIONAL - if LINEAR_API_KEY is not set, all operations +gracefully no-op and the build continues with local tracking only. + +Key Features: +- Subtask → Issue mapping (sync implementation_plan.json to Linear) +- Session attempt recording (comments on issues) +- Stuck subtask escalation (move to Blocked, add detailed comments) +- Progress tracking via META issue +""" + +import json +import os +from datetime import datetime +from pathlib import Path + +from .config import ( + LABELS, + STATUS_BLOCKED, + LinearConfig, + LinearProjectState, + format_session_comment, + format_stuck_subtask_comment, + format_subtask_description, + get_linear_status, + get_priority_for_phase, +) + + +class LinearManager: + """ + Manages Linear integration for an Auto-Build spec. + + This class provides a high-level interface for: + - Creating/syncing issues from implementation_plan.json + - Recording session attempts and results + - Escalating stuck subtasks + - Tracking overall progress + + All operations are idempotent and gracefully handle Linear being unavailable. + """ + + def __init__(self, spec_dir: Path, project_dir: Path): + """ + Initialize Linear manager. + + Args: + spec_dir: Spec directory (contains implementation_plan.json) + project_dir: Project root directory + """ + self.spec_dir = spec_dir + self.project_dir = project_dir + self.config = LinearConfig.from_env() + self.state: LinearProjectState | None = None + self._mcp_available = False + + # Load existing state if available + self.state = LinearProjectState.load(spec_dir) + + # Check if Linear MCP tools are available + self._check_mcp_availability() + + def _check_mcp_availability(self) -> None: + """Check if Linear MCP tools are available in the environment.""" + # In agent context, MCP tools are available via claude-code + # We'll assume they're available if LINEAR_API_KEY is set + self._mcp_available = self.config.is_valid() + + @property + def is_enabled(self) -> bool: + """Check if Linear integration is enabled and available.""" + return self.config.is_valid() and self._mcp_available + + @property + def is_initialized(self) -> bool: + """Check if Linear project has been initialized for this spec.""" + return self.state is not None and self.state.initialized + + def get_issue_id(self, subtask_id: str) -> str | None: + """ + Get the Linear issue ID for a subtask. + + Args: + subtask_id: Subtask ID from implementation_plan.json + + Returns: + Linear issue ID or None if not mapped + """ + if not self.state: + return None + return self.state.issue_mapping.get(subtask_id) + + def set_issue_id(self, subtask_id: str, issue_id: str) -> None: + """ + Store the mapping between a subtask and its Linear issue. + + Args: + subtask_id: Subtask ID from implementation_plan.json + issue_id: Linear issue ID + """ + if not self.state: + self.state = LinearProjectState() + + self.state.issue_mapping[subtask_id] = issue_id + self.state.save(self.spec_dir) + + def initialize_project(self, team_id: str, project_name: str) -> bool: + """ + Initialize a Linear project for this spec. + + This should be called by the agent during the planner session + to set up the Linear project and create initial issues. + + Args: + team_id: Linear team ID + project_name: Name for the Linear project + + Returns: + True if successful + """ + if not self.is_enabled: + print("Linear integration not enabled (LINEAR_API_KEY not set)") + return False + + # Create initial state + self.state = LinearProjectState( + initialized=True, + team_id=team_id, + project_name=project_name, + created_at=datetime.now().isoformat(), + ) + + self.state.save(self.spec_dir) + return True + + def update_project_id(self, project_id: str) -> None: + """Update the Linear project ID after creation.""" + if self.state: + self.state.project_id = project_id + self.state.save(self.spec_dir) + + def update_meta_issue_id(self, meta_issue_id: str) -> None: + """Update the META issue ID after creation.""" + if self.state: + self.state.meta_issue_id = meta_issue_id + self.state.save(self.spec_dir) + + def load_implementation_plan(self) -> dict | None: + """Load the implementation plan from spec directory.""" + plan_file = self.spec_dir / "implementation_plan.json" + if not plan_file.exists(): + return None + + try: + with open(plan_file) as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + return None + + def get_subtasks_for_sync(self) -> list[dict]: + """ + Get all subtasks that need Linear issues. + + Returns: + List of subtask dicts with phase context + """ + plan = self.load_implementation_plan() + if not plan: + return [] + + subtasks = [] + phases = plan.get("phases", []) + total_phases = len(phases) + + for phase in phases: + phase_num = phase.get("phase", 1) + phase_name = phase.get("name", f"Phase {phase_num}") + + for subtask in phase.get("subtasks", []): + subtasks.append( + { + **subtask, + "phase_num": phase_num, + "phase_name": phase_name, + "total_phases": total_phases, + "phase_depends_on": phase.get("depends_on", []), + } + ) + + return subtasks + + def generate_issue_data(self, subtask: dict) -> dict: + """ + Generate Linear issue data from a subtask. + + Args: + subtask: Subtask dict with phase context + + Returns: + Dict suitable for Linear create_issue + """ + phase = { + "name": subtask.get("phase_name"), + "id": subtask.get("phase_num"), + } + + # Determine priority based on phase position + priority = get_priority_for_phase( + subtask.get("phase_num", 1), subtask.get("total_phases", 1) + ) + + # Build labels list + labels = [LABELS["auto_build"]] + if subtask.get("service"): + labels.append(f"{LABELS['service']}-{subtask['service']}") + if subtask.get("phase_num"): + labels.append(f"{LABELS['phase']}-{subtask['phase_num']}") + + return { + "title": f"[{subtask.get('id', 'subtask')}] {subtask.get('description', 'Implement subtask')[:100]}", + "description": format_subtask_description(subtask, phase), + "priority": priority, + "labels": labels, + "status": get_linear_status(subtask.get("status", "pending")), + } + + def record_session_result( + self, + subtask_id: str, + session_num: int, + success: bool, + approach: str = "", + error: str = "", + git_commit: str = "", + ) -> str: + """ + Record a session result as a Linear comment. + + This is called by post_session_processing in agent.py. + + Args: + subtask_id: Subtask being worked on + session_num: Session number + success: Whether the session succeeded + approach: What was attempted + error: Error message if failed + git_commit: Git commit hash if any + + Returns: + Formatted comment body (for logging even if Linear unavailable) + """ + comment = format_session_comment( + session_num=session_num, + subtask_id=subtask_id, + success=success, + approach=approach, + error=error, + git_commit=git_commit, + ) + + # Note: Actual Linear API call will be done by the agent + # This method prepares the data and returns it + return comment + + def prepare_status_update(self, subtask_id: str, new_status: str) -> dict: + """ + Prepare data for a Linear issue status update. + + Args: + subtask_id: Subtask ID + new_status: New subtask status (pending, in_progress, completed, etc.) + + Returns: + Dict with issue_id and linear_status for the update + """ + issue_id = self.get_issue_id(subtask_id) + linear_status = get_linear_status(new_status) + + return { + "issue_id": issue_id, + "status": linear_status, + "subtask_id": subtask_id, + } + + def prepare_stuck_escalation( + self, + subtask_id: str, + attempt_count: int, + attempts: list[dict], + reason: str = "", + ) -> dict: + """ + Prepare data for escalating a stuck subtask. + + This creates the comment body and status update data. + + Args: + subtask_id: Stuck subtask ID + attempt_count: Number of attempts + attempts: List of attempt records + reason: Why it's stuck + + Returns: + Dict with issue_id, comment, labels for escalation + """ + issue_id = self.get_issue_id(subtask_id) + comment = format_stuck_subtask_comment( + subtask_id=subtask_id, + attempt_count=attempt_count, + attempts=attempts, + reason=reason, + ) + + return { + "issue_id": issue_id, + "subtask_id": subtask_id, + "status": STATUS_BLOCKED, + "comment": comment, + "labels": [LABELS["stuck"], LABELS["needs_review"]], + } + + def get_progress_summary(self) -> dict: + """ + Get a summary of Linear integration progress. + + Returns: + Dict with progress statistics + """ + plan = self.load_implementation_plan() + if not plan: + return { + "enabled": self.is_enabled, + "initialized": False, + "total_subtasks": 0, + "mapped_subtasks": 0, + } + + subtasks = self.get_subtasks_for_sync() + mapped = sum(1 for s in subtasks if self.get_issue_id(s.get("id", ""))) + + return { + "enabled": self.is_enabled, + "initialized": self.is_initialized, + "team_id": self.state.team_id if self.state else None, + "project_id": self.state.project_id if self.state else None, + "project_name": self.state.project_name if self.state else None, + "meta_issue_id": self.state.meta_issue_id if self.state else None, + "total_subtasks": len(subtasks), + "mapped_subtasks": mapped, + } + + def get_linear_context_for_prompt(self) -> str: + """ + Generate Linear context section for agent prompts. + + This is included in the subtask prompt to give the agent + awareness of Linear integration status. + + Returns: + Markdown-formatted context string + """ + if not self.is_enabled: + return "" + + summary = self.get_progress_summary() + + if not summary["initialized"]: + return """ +## Linear Integration + +Linear integration is enabled but not yet initialized. +During the planner session, create a Linear project and sync issues. + +Available Linear MCP tools: +- `mcp__linear-server__list_teams` - List available teams +- `mcp__linear-server__create_project` - Create a new project +- `mcp__linear-server__create_issue` - Create issues for subtasks +- `mcp__linear-server__update_issue` - Update issue status +- `mcp__linear-server__create_comment` - Add session comments +""" + + lines = [ + "## Linear Integration", + "", + f"**Project:** {summary['project_name']}", + f"**Issues:** {summary['mapped_subtasks']}/{summary['total_subtasks']} subtasks mapped", + "", + "When working on a subtask:", + "1. Update issue status to 'In Progress' at start", + "2. Add comments with progress/blockers", + "3. Update status to 'Done' when subtask completes", + "4. If stuck, status will be set to 'Blocked' automatically", + ] + + return "\n".join(lines) + + def save_state(self) -> None: + """Save the current state to disk.""" + if self.state: + self.state.save(self.spec_dir) + + +# Utility functions for integration with other modules + + +def get_linear_manager(spec_dir: Path, project_dir: Path) -> LinearManager: + """ + Get a LinearManager instance for the given spec. + + This is the main entry point for other modules. + + Args: + spec_dir: Spec directory + project_dir: Project root directory + + Returns: + LinearManager instance + """ + return LinearManager(spec_dir, project_dir) + + +def is_linear_enabled() -> bool: + """Quick check if Linear integration is available.""" + return bool(os.environ.get("LINEAR_API_KEY")) + + +def prepare_planner_linear_instructions(spec_dir: Path) -> str: + """ + Generate Linear setup instructions for the planner agent. + + This is included in the planner prompt when Linear is enabled. + + Args: + spec_dir: Spec directory + + Returns: + Markdown instructions for Linear setup + """ + if not is_linear_enabled(): + return "" + + return """ +## Linear Integration Setup + +Linear integration is ENABLED. After creating the implementation plan: + +### Step 1: Find the Team +``` +Use mcp__linear-server__list_teams to find your team ID +``` + +### Step 2: Create the Project +``` +Use mcp__linear-server__create_project with: +- team: Your team ID +- name: The feature/spec name +- description: Brief summary from spec.md +``` +Save the project ID to .linear_project.json + +### Step 3: Create Issues for Each Subtask +For each subtask in implementation_plan.json: +``` +Use mcp__linear-server__create_issue with: +- team: Your team ID +- project: The project ID +- title: "[subtask-id] Description" +- description: Formatted subtask details +- priority: Based on phase (1=urgent for early phases, 4=low for polish) +- labels: ["auto-claude", "phase-N", "service-NAME"] +``` +Save the subtask_id -> issue_id mapping to .linear_project.json + +### Step 4: Create META Issue +``` +Use mcp__linear-server__create_issue with: +- title: "[META] Build Progress Tracker" +- description: "Session summaries and overall progress tracking" +``` +This issue receives session summary comments. + +### Important Notes +- Update .linear_project.json after each Linear operation +- The JSON structure should include: + - initialized: true + - team_id: "..." + - project_id: "..." + - meta_issue_id: "..." + - issue_mapping: { "subtask-1-1": "LIN-123", ... } +""" + + +def prepare_coder_linear_instructions( + spec_dir: Path, + subtask_id: str, +) -> str: + """ + Generate Linear instructions for the coding agent. + + Args: + spec_dir: Spec directory + subtask_id: Current subtask being worked on + + Returns: + Markdown instructions for Linear updates + """ + if not is_linear_enabled(): + return "" + + manager = LinearManager(spec_dir, spec_dir.parent.parent) # Approximate project_dir + + if not manager.is_initialized: + return "" + + issue_id = manager.get_issue_id(subtask_id) + if not issue_id: + return "" + + return f""" +## Linear Updates + +This subtask is linked to Linear issue: `{issue_id}` + +### At Session Start +Update the issue status to "In Progress": +``` +mcp__linear-server__update_issue(id="{issue_id}", state="In Progress") +``` + +### During Work +Add comments for significant progress or blockers: +``` +mcp__linear-server__create_comment(issueId="{issue_id}", body="...") +``` + +### On Completion +Update status to "Done": +``` +mcp__linear-server__update_issue(id="{issue_id}", state="Done") +``` + +### Session Summary +At session end, add a comment to the META issue with: +- What was accomplished +- Any blockers or issues found +- Recommendations for next session +""" diff --git a/auto-claude/integrations/linear/updater.py b/auto-claude/integrations/linear/updater.py new file mode 100644 index 00000000..d60e7f25 --- /dev/null +++ b/auto-claude/integrations/linear/updater.py @@ -0,0 +1,442 @@ +""" +Linear Updater - Python-Orchestrated Linear Updates +==================================================== + +Provides reliable Linear updates via focused mini-agent calls. +Instead of relying on agents to remember Linear updates in long prompts, +the Python orchestrator triggers small, focused agents at key transitions. + +Design Principles: +- ONE task per spec (not one issue per subtask) +- Python orchestrator controls when updates happen +- Small prompts that can't lose context +- Graceful degradation if Linear unavailable + +Status Flow: + Todo -> In Progress -> In Review -> (human) -> Done + | | | + | | +-- QA approved, awaiting human merge + | +-- Planner/Coder working + +-- Task created from spec +""" + +import json +import os +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Optional + +from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient + +# Linear status constants (matching Valma AI team setup) +STATUS_TODO = "Todo" +STATUS_IN_PROGRESS = "In Progress" +STATUS_IN_REVIEW = "In Review" # Custom status for QA phase +STATUS_DONE = "Done" +STATUS_CANCELED = "Canceled" + +# State file name +LINEAR_TASK_FILE = ".linear_task.json" + +# Linear MCP tools needed for updates +LINEAR_TOOLS = [ + "mcp__linear-server__list_teams", + "mcp__linear-server__create_issue", + "mcp__linear-server__update_issue", + "mcp__linear-server__create_comment", + "mcp__linear-server__list_issue_statuses", +] + + +@dataclass +class LinearTaskState: + """State of a Linear task for an auto-claude spec.""" + + task_id: str | None = None + task_title: str | None = None + team_id: str | None = None + status: str = STATUS_TODO + created_at: str | None = None + + def to_dict(self) -> dict: + return { + "task_id": self.task_id, + "task_title": self.task_title, + "team_id": self.team_id, + "status": self.status, + "created_at": self.created_at, + } + + @classmethod + def from_dict(cls, data: dict) -> "LinearTaskState": + return cls( + task_id=data.get("task_id"), + task_title=data.get("task_title"), + team_id=data.get("team_id"), + status=data.get("status", STATUS_TODO), + created_at=data.get("created_at"), + ) + + def save(self, spec_dir: Path) -> None: + """Save state to the spec directory.""" + state_file = spec_dir / LINEAR_TASK_FILE + with open(state_file, "w") as f: + json.dump(self.to_dict(), f, indent=2) + + @classmethod + def load(cls, spec_dir: Path) -> Optional["LinearTaskState"]: + """Load state from the spec directory.""" + state_file = spec_dir / LINEAR_TASK_FILE + if not state_file.exists(): + return None + + try: + with open(state_file) as f: + return cls.from_dict(json.load(f)) + except (OSError, json.JSONDecodeError): + return None + + +def is_linear_enabled() -> bool: + """Check if Linear integration is available.""" + return bool(os.environ.get("LINEAR_API_KEY")) + + +def get_linear_api_key() -> str: + """Get the Linear API key from environment.""" + return os.environ.get("LINEAR_API_KEY", "") + + +def _create_linear_client() -> ClaudeSDKClient: + """ + Create a minimal Claude client with only Linear MCP tools. + Used for focused mini-agent calls. + """ + oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") + if not oauth_token: + raise ValueError("CLAUDE_CODE_OAUTH_TOKEN not set") + + linear_api_key = get_linear_api_key() + if not linear_api_key: + raise ValueError("LINEAR_API_KEY not set") + + return ClaudeSDKClient( + options=ClaudeAgentOptions( + model="claude-haiku-4-5", # Fast & cheap model for simple API calls + system_prompt="You are a Linear API assistant. Execute the requested Linear operation precisely.", + allowed_tools=LINEAR_TOOLS, + mcp_servers={ + "linear": { + "type": "http", + "url": "https://mcp.linear.app/mcp", + "headers": {"Authorization": f"Bearer {linear_api_key}"}, + } + }, + max_turns=10, # Should complete in 1-3 turns + ) + ) + + +async def _run_linear_agent(prompt: str) -> str | None: + """ + Run a focused mini-agent for a Linear operation. + + Args: + prompt: The focused prompt for the Linear operation + + Returns: + The response text, or None if failed + """ + try: + client = _create_linear_client() + + async with client: + await client.query(prompt) + + response_text = "" + async for msg in client.receive_response(): + msg_type = type(msg).__name__ + if msg_type == "AssistantMessage" and hasattr(msg, "content"): + for block in msg.content: + block_type = type(block).__name__ + if block_type == "TextBlock" and hasattr(block, "text"): + response_text += block.text + + return response_text + + except Exception as e: + print(f"Linear update failed: {e}") + return None + + +async def create_linear_task( + spec_dir: Path, + title: str, + description: str | None = None, +) -> LinearTaskState | None: + """ + Create a new Linear task for a spec. + + Called by spec_runner.py after requirements gathering. + + Args: + spec_dir: Spec directory to save state + title: Task title (the task name from user) + description: Optional task description + + Returns: + LinearTaskState if successful, None if failed + """ + if not is_linear_enabled(): + return None + + # Check if task already exists + existing = LinearTaskState.load(spec_dir) + if existing and existing.task_id: + print(f"Linear task already exists: {existing.task_id}") + return existing + + desc_part = f'\n - description: "{description}"' if description else "" + + prompt = f"""Create a Linear task with these details: + +1. First, use mcp__linear-server__list_teams to find the team ID +2. Then, use mcp__linear-server__create_issue with: + - teamId: [the team ID from step 1] + - title: "{title}"{desc_part} + +After creating the issue, tell me: +- The issue ID (like "VAL-123") +- The team ID you used + +Format your final response as: +TASK_ID: [the issue ID] +TEAM_ID: [the team ID] +""" + + response = await _run_linear_agent(prompt) + if not response: + return None + + # Parse response for task_id and team_id + task_id = None + team_id = None + + for line in response.split("\n"): + line = line.strip() + if line.startswith("TASK_ID:"): + task_id = line.replace("TASK_ID:", "").strip() + elif line.startswith("TEAM_ID:"): + team_id = line.replace("TEAM_ID:", "").strip() + + if not task_id: + print(f"Failed to parse task ID from response: {response[:200]}") + return None + + # Create and save state + state = LinearTaskState( + task_id=task_id, + task_title=title, + team_id=team_id, + status=STATUS_TODO, + created_at=datetime.now().isoformat(), + ) + state.save(spec_dir) + + print(f"Created Linear task: {task_id}") + return state + + +async def update_linear_status( + spec_dir: Path, + new_status: str, +) -> bool: + """ + Update the Linear task status. + + Args: + spec_dir: Spec directory with .linear_task.json + new_status: New status (STATUS_TODO, STATUS_IN_PROGRESS, STATUS_IN_REVIEW, STATUS_DONE) + + Returns: + True if successful, False otherwise + """ + if not is_linear_enabled(): + return False + + state = LinearTaskState.load(spec_dir) + if not state or not state.task_id: + print("No Linear task found for this spec") + return False + + # Don't update if already at this status + if state.status == new_status: + return True + + prompt = f"""Update Linear issue status: + +1. First, use mcp__linear-server__list_issue_statuses with teamId: "{state.team_id}" to find the state ID for "{new_status}" +2. Then, use mcp__linear-server__update_issue with: + - issueId: "{state.task_id}" + - stateId: [the state ID for "{new_status}" from step 1] + +Confirm when done. +""" + + response = await _run_linear_agent(prompt) + if response: + state.status = new_status + state.save(spec_dir) + print(f"Updated Linear task {state.task_id} to: {new_status}") + return True + + return False + + +async def add_linear_comment( + spec_dir: Path, + comment: str, +) -> bool: + """ + Add a comment to the Linear task. + + Args: + spec_dir: Spec directory with .linear_task.json + comment: Comment text to add + + Returns: + True if successful, False otherwise + """ + if not is_linear_enabled(): + return False + + state = LinearTaskState.load(spec_dir) + if not state or not state.task_id: + print("No Linear task found for this spec") + return False + + # Escape any quotes in the comment + safe_comment = comment.replace('"', '\\"').replace("\n", "\\n") + + prompt = f"""Add a comment to Linear issue: + +Use mcp__linear-server__create_comment with: +- issueId: "{state.task_id}" +- body: "{safe_comment}" + +Confirm when done. +""" + + response = await _run_linear_agent(prompt) + if response: + print(f"Added comment to Linear task {state.task_id}") + return True + + return False + + +# === Convenience functions for specific transitions === + + +async def linear_task_started(spec_dir: Path) -> bool: + """ + Mark task as started (In Progress). + Called when planner session begins. + """ + success = await update_linear_status(spec_dir, STATUS_IN_PROGRESS) + if success: + await add_linear_comment(spec_dir, "Build started - planning phase initiated") + return success + + +async def linear_subtask_completed( + spec_dir: Path, + subtask_id: str, + completed_count: int, + total_count: int, +) -> bool: + """ + Record subtask completion as a comment. + Called after each successful coder session. + """ + comment = f"Completed {subtask_id} ({completed_count}/{total_count} subtasks done)" + return await add_linear_comment(spec_dir, comment) + + +async def linear_subtask_failed( + spec_dir: Path, + subtask_id: str, + attempt: int, + error_summary: str, +) -> bool: + """ + Record subtask failure as a comment. + Called after failed coder session. + """ + comment = f"Subtask {subtask_id} failed (attempt {attempt}): {error_summary[:200]}" + return await add_linear_comment(spec_dir, comment) + + +async def linear_build_complete(spec_dir: Path) -> bool: + """ + Record build completion, moving to QA. + Called when all subtasks are completed. + """ + comment = "All subtasks completed - moving to QA validation" + return await add_linear_comment(spec_dir, comment) + + +async def linear_qa_started(spec_dir: Path) -> bool: + """ + Mark task as In Review for QA phase. + Called when QA validation loop starts. + """ + success = await update_linear_status(spec_dir, STATUS_IN_REVIEW) + if success: + await add_linear_comment(spec_dir, "QA validation started") + return success + + +async def linear_qa_approved(spec_dir: Path) -> bool: + """ + Record QA approval (stays In Review for human). + Called when QA approves the build. + """ + comment = "QA approved - awaiting human review for merge" + return await add_linear_comment(spec_dir, comment) + + +async def linear_qa_rejected( + spec_dir: Path, + issues_count: int, + iteration: int, +) -> bool: + """ + Record QA rejection. + Called when QA rejects the build. + """ + comment = f"QA iteration {iteration}: Found {issues_count} issues - applying fixes" + return await add_linear_comment(spec_dir, comment) + + +async def linear_qa_max_iterations(spec_dir: Path, iterations: int) -> bool: + """ + Record QA max iterations reached. + Called when QA loop exhausts retries. + """ + comment = f"QA reached max iterations ({iterations}) - needs human intervention" + return await add_linear_comment(spec_dir, comment) + + +async def linear_task_stuck( + spec_dir: Path, + subtask_id: str, + attempt_count: int, +) -> bool: + """ + Record that a subtask is stuck. + Called when subtask exceeds retry limit. + """ + comment = f"Subtask {subtask_id} is STUCK after {attempt_count} attempts - needs human review" + return await add_linear_comment(spec_dir, comment) diff --git a/auto-claude/linear_config.py b/auto-claude/linear_config.py index 25bd149f..1684a1a9 100644 --- a/auto-claude/linear_config.py +++ b/auto-claude/linear_config.py @@ -1,342 +1,2 @@ -""" -Linear Integration Configuration -================================ - -Constants, status mappings, and configuration helpers for Linear integration. -Mirrors the approach from Linear-Coding-Agent-Harness. -""" - -import json -import os -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -from typing import Optional - -# Linear Status Constants (map to Linear workflow states) -STATUS_TODO = "Todo" -STATUS_IN_PROGRESS = "In Progress" -STATUS_DONE = "Done" -STATUS_BLOCKED = "Blocked" # For stuck subtasks -STATUS_CANCELED = "Canceled" - -# Linear Priority Constants (1=Urgent, 4=Low, 0=No priority) -PRIORITY_URGENT = 1 # Core infrastructure, blockers -PRIORITY_HIGH = 2 # Primary features, dependencies -PRIORITY_MEDIUM = 3 # Secondary features -PRIORITY_LOW = 4 # Polish, nice-to-haves -PRIORITY_NONE = 0 # No priority set - -# Subtask status to Linear status mapping -SUBTASK_TO_LINEAR_STATUS = { - "pending": STATUS_TODO, - "in_progress": STATUS_IN_PROGRESS, - "completed": STATUS_DONE, - "blocked": STATUS_BLOCKED, - "failed": STATUS_BLOCKED, # Map failures to Blocked for visibility - "stuck": STATUS_BLOCKED, -} - -# Linear labels for categorization -LABELS = { - "phase": "phase", # Phase label prefix (e.g., "phase-1") - "service": "service", # Service label prefix (e.g., "service-backend") - "stuck": "stuck", # Mark stuck subtasks - "auto_build": "auto-claude", # All auto-claude issues - "needs_review": "needs-review", -} - -# Linear project marker file (stores team/project IDs) -LINEAR_PROJECT_MARKER = ".linear_project.json" - -# Meta issue for session tracking -META_ISSUE_TITLE = "[META] Build Progress Tracker" - - -@dataclass -class LinearConfig: - """Configuration for Linear integration.""" - - api_key: str - team_id: str | None = None - project_id: str | None = None - project_name: str | None = None - meta_issue_id: str | None = None - enabled: bool = True - - @classmethod - def from_env(cls) -> "LinearConfig": - """Create config from environment variables.""" - api_key = os.environ.get("LINEAR_API_KEY", "") - - return cls( - api_key=api_key, - team_id=os.environ.get("LINEAR_TEAM_ID"), - project_id=os.environ.get("LINEAR_PROJECT_ID"), - enabled=bool(api_key), - ) - - def is_valid(self) -> bool: - """Check if config has minimum required values.""" - return bool(self.api_key) - - -@dataclass -class LinearProjectState: - """State of a Linear project for an auto-claude spec.""" - - initialized: bool = False - team_id: str | None = None - project_id: str | None = None - project_name: str | None = None - meta_issue_id: str | None = None - total_issues: int = 0 - created_at: str | None = None - issue_mapping: dict = None # subtask_id -> issue_id mapping - - def __post_init__(self): - if self.issue_mapping is None: - self.issue_mapping = {} - - def to_dict(self) -> dict: - return { - "initialized": self.initialized, - "team_id": self.team_id, - "project_id": self.project_id, - "project_name": self.project_name, - "meta_issue_id": self.meta_issue_id, - "total_issues": self.total_issues, - "created_at": self.created_at, - "issue_mapping": self.issue_mapping, - } - - @classmethod - def from_dict(cls, data: dict) -> "LinearProjectState": - return cls( - initialized=data.get("initialized", False), - team_id=data.get("team_id"), - project_id=data.get("project_id"), - project_name=data.get("project_name"), - meta_issue_id=data.get("meta_issue_id"), - total_issues=data.get("total_issues", 0), - created_at=data.get("created_at"), - issue_mapping=data.get("issue_mapping", {}), - ) - - def save(self, spec_dir: Path) -> None: - """Save state to the spec directory.""" - marker_file = spec_dir / LINEAR_PROJECT_MARKER - with open(marker_file, "w") as f: - json.dump(self.to_dict(), f, indent=2) - - @classmethod - def load(cls, spec_dir: Path) -> Optional["LinearProjectState"]: - """Load state from the spec directory.""" - marker_file = spec_dir / LINEAR_PROJECT_MARKER - if not marker_file.exists(): - return None - - try: - with open(marker_file) as f: - return cls.from_dict(json.load(f)) - except (OSError, json.JSONDecodeError): - return None - - -def get_linear_status(subtask_status: str) -> str: - """ - Map subtask status to Linear status. - - Args: - subtask_status: Status from implementation_plan.json - - Returns: - Corresponding Linear status string - """ - return SUBTASK_TO_LINEAR_STATUS.get(subtask_status, STATUS_TODO) - - -def get_priority_for_phase(phase_num: int, total_phases: int) -> int: - """ - Determine Linear priority based on phase number. - - Early phases are higher priority (they're dependencies). - - Args: - phase_num: Phase number (1-indexed) - total_phases: Total number of phases - - Returns: - Linear priority value (1-4) - """ - if total_phases <= 1: - return PRIORITY_HIGH - - # First quarter of phases = Urgent - # Second quarter = High - # Third quarter = Medium - # Fourth quarter = Low - position = phase_num / total_phases - - if position <= 0.25: - return PRIORITY_URGENT - elif position <= 0.5: - return PRIORITY_HIGH - elif position <= 0.75: - return PRIORITY_MEDIUM - else: - return PRIORITY_LOW - - -def format_subtask_description(subtask: dict, phase: dict = None) -> str: - """ - Format a subtask as a Linear issue description. - - Args: - subtask: Subtask dict from implementation_plan.json - phase: Optional phase dict for context - - Returns: - Markdown-formatted description - """ - lines = [] - - # Description - if subtask.get("description"): - lines.append(f"## Description\n{subtask['description']}\n") - - # Service - if subtask.get("service"): - lines.append(f"**Service:** {subtask['service']}") - elif subtask.get("all_services"): - lines.append("**Scope:** All services (integration)") - - # Phase info - if phase: - lines.append(f"**Phase:** {phase.get('name', phase.get('id', 'Unknown'))}") - - # Files to modify - if subtask.get("files_to_modify"): - lines.append("\n## Files to Modify") - for f in subtask["files_to_modify"]: - lines.append(f"- `{f}`") - - # Files to create - if subtask.get("files_to_create"): - lines.append("\n## Files to Create") - for f in subtask["files_to_create"]: - lines.append(f"- `{f}`") - - # Patterns to follow - if subtask.get("patterns_from"): - lines.append("\n## Reference Patterns") - for f in subtask["patterns_from"]: - lines.append(f"- `{f}`") - - # Verification - if subtask.get("verification"): - v = subtask["verification"] - lines.append("\n## Verification") - lines.append(f"**Type:** {v.get('type', 'none')}") - if v.get("run"): - lines.append(f"**Command:** `{v['run']}`") - if v.get("url"): - lines.append(f"**URL:** {v['url']}") - if v.get("scenario"): - lines.append(f"**Scenario:** {v['scenario']}") - - # Auto-build metadata - lines.append("\n---") - lines.append("*This issue was created by the Auto-Build Framework*") - - return "\n".join(lines) - - -def format_session_comment( - session_num: int, - subtask_id: str, - success: bool, - approach: str = "", - error: str = "", - git_commit: str = "", -) -> str: - """ - Format a session result as a Linear comment. - - Args: - session_num: Session number - subtask_id: Subtask being worked on - success: Whether the session succeeded - approach: What was attempted - error: Error message if failed - git_commit: Git commit hash if any - - Returns: - Markdown-formatted comment - """ - status_emoji = "✅" if success else "❌" - lines = [ - f"## Session #{session_num} {status_emoji}", - f"**Subtask:** `{subtask_id}`", - f"**Status:** {'Completed' if success else 'In Progress'}", - f"**Time:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", - ] - - if approach: - lines.append(f"\n**Approach:** {approach}") - - if git_commit: - lines.append(f"\n**Commit:** `{git_commit[:8]}`") - - if error: - lines.append(f"\n**Error:**\n```\n{error[:500]}\n```") - - return "\n".join(lines) - - -def format_stuck_subtask_comment( - subtask_id: str, - attempt_count: int, - attempts: list[dict], - reason: str = "", -) -> str: - """ - Format a detailed comment for stuck subtasks. - - Args: - subtask_id: Stuck subtask ID - attempt_count: Number of attempts - attempts: List of attempt records - reason: Why it's stuck - - Returns: - Markdown-formatted comment for escalation - """ - lines = [ - "## ⚠️ Subtask Marked as STUCK", - f"**Subtask:** `{subtask_id}`", - f"**Attempts:** {attempt_count}", - f"**Time:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", - ] - - if reason: - lines.append(f"\n**Reason:** {reason}") - - # Add attempt history - if attempts: - lines.append("\n### Attempt History") - for i, attempt in enumerate(attempts[-5:], 1): # Last 5 attempts - status = "✅" if attempt.get("success") else "❌" - lines.append(f"\n**Attempt {i}:** {status}") - if attempt.get("approach"): - lines.append(f"- Approach: {attempt['approach'][:200]}") - if attempt.get("error"): - lines.append(f"- Error: {attempt['error'][:200]}") - - lines.append("\n### Recommended Actions") - lines.append("1. Review the approach and error patterns above") - lines.append("2. Check for missing dependencies or configuration") - lines.append("3. Consider manual intervention or different approach") - lines.append("4. Update HUMAN_INPUT.md with guidance for the agent") - - return "\n".join(lines) +"""Backward compatibility shim - import from integrations.linear.config instead.""" +from integrations.linear.config import * diff --git a/auto-claude/linear_integration.py b/auto-claude/linear_integration.py index 2bd18182..180f0b61 100644 --- a/auto-claude/linear_integration.py +++ b/auto-claude/linear_integration.py @@ -1,553 +1,2 @@ -""" -Linear Integration Manager -========================== - -Manages synchronization between Auto-Build subtasks and Linear issues. -Provides real-time visibility into build progress through Linear. - -The integration is OPTIONAL - if LINEAR_API_KEY is not set, all operations -gracefully no-op and the build continues with local tracking only. - -Key Features: -- Subtask → Issue mapping (sync implementation_plan.json to Linear) -- Session attempt recording (comments on issues) -- Stuck subtask escalation (move to Blocked, add detailed comments) -- Progress tracking via META issue -""" - -import json -import os -from datetime import datetime -from pathlib import Path - -from linear_config import ( - LABELS, - STATUS_BLOCKED, - LinearConfig, - LinearProjectState, - format_session_comment, - format_stuck_subtask_comment, - format_subtask_description, - get_linear_status, - get_priority_for_phase, -) - - -class LinearManager: - """ - Manages Linear integration for an Auto-Build spec. - - This class provides a high-level interface for: - - Creating/syncing issues from implementation_plan.json - - Recording session attempts and results - - Escalating stuck subtasks - - Tracking overall progress - - All operations are idempotent and gracefully handle Linear being unavailable. - """ - - def __init__(self, spec_dir: Path, project_dir: Path): - """ - Initialize Linear manager. - - Args: - spec_dir: Spec directory (contains implementation_plan.json) - project_dir: Project root directory - """ - self.spec_dir = spec_dir - self.project_dir = project_dir - self.config = LinearConfig.from_env() - self.state: LinearProjectState | None = None - self._mcp_available = False - - # Load existing state if available - self.state = LinearProjectState.load(spec_dir) - - # Check if Linear MCP tools are available - self._check_mcp_availability() - - def _check_mcp_availability(self) -> None: - """Check if Linear MCP tools are available in the environment.""" - # In agent context, MCP tools are available via claude-code - # We'll assume they're available if LINEAR_API_KEY is set - self._mcp_available = self.config.is_valid() - - @property - def is_enabled(self) -> bool: - """Check if Linear integration is enabled and available.""" - return self.config.is_valid() and self._mcp_available - - @property - def is_initialized(self) -> bool: - """Check if Linear project has been initialized for this spec.""" - return self.state is not None and self.state.initialized - - def get_issue_id(self, subtask_id: str) -> str | None: - """ - Get the Linear issue ID for a subtask. - - Args: - subtask_id: Subtask ID from implementation_plan.json - - Returns: - Linear issue ID or None if not mapped - """ - if not self.state: - return None - return self.state.issue_mapping.get(subtask_id) - - def set_issue_id(self, subtask_id: str, issue_id: str) -> None: - """ - Store the mapping between a subtask and its Linear issue. - - Args: - subtask_id: Subtask ID from implementation_plan.json - issue_id: Linear issue ID - """ - if not self.state: - self.state = LinearProjectState() - - self.state.issue_mapping[subtask_id] = issue_id - self.state.save(self.spec_dir) - - def initialize_project(self, team_id: str, project_name: str) -> bool: - """ - Initialize a Linear project for this spec. - - This should be called by the agent during the planner session - to set up the Linear project and create initial issues. - - Args: - team_id: Linear team ID - project_name: Name for the Linear project - - Returns: - True if successful - """ - if not self.is_enabled: - print("Linear integration not enabled (LINEAR_API_KEY not set)") - return False - - # Create initial state - self.state = LinearProjectState( - initialized=True, - team_id=team_id, - project_name=project_name, - created_at=datetime.now().isoformat(), - ) - - self.state.save(self.spec_dir) - return True - - def update_project_id(self, project_id: str) -> None: - """Update the Linear project ID after creation.""" - if self.state: - self.state.project_id = project_id - self.state.save(self.spec_dir) - - def update_meta_issue_id(self, meta_issue_id: str) -> None: - """Update the META issue ID after creation.""" - if self.state: - self.state.meta_issue_id = meta_issue_id - self.state.save(self.spec_dir) - - def load_implementation_plan(self) -> dict | None: - """Load the implementation plan from spec directory.""" - plan_file = self.spec_dir / "implementation_plan.json" - if not plan_file.exists(): - return None - - try: - with open(plan_file) as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - return None - - def get_subtasks_for_sync(self) -> list[dict]: - """ - Get all subtasks that need Linear issues. - - Returns: - List of subtask dicts with phase context - """ - plan = self.load_implementation_plan() - if not plan: - return [] - - subtasks = [] - phases = plan.get("phases", []) - total_phases = len(phases) - - for phase in phases: - phase_num = phase.get("phase", 1) - phase_name = phase.get("name", f"Phase {phase_num}") - - for subtask in phase.get("subtasks", []): - subtasks.append( - { - **subtask, - "phase_num": phase_num, - "phase_name": phase_name, - "total_phases": total_phases, - "phase_depends_on": phase.get("depends_on", []), - } - ) - - return subtasks - - def generate_issue_data(self, subtask: dict) -> dict: - """ - Generate Linear issue data from a subtask. - - Args: - subtask: Subtask dict with phase context - - Returns: - Dict suitable for Linear create_issue - """ - phase = { - "name": subtask.get("phase_name"), - "id": subtask.get("phase_num"), - } - - # Determine priority based on phase position - priority = get_priority_for_phase( - subtask.get("phase_num", 1), subtask.get("total_phases", 1) - ) - - # Build labels list - labels = [LABELS["auto_build"]] - if subtask.get("service"): - labels.append(f"{LABELS['service']}-{subtask['service']}") - if subtask.get("phase_num"): - labels.append(f"{LABELS['phase']}-{subtask['phase_num']}") - - return { - "title": f"[{subtask.get('id', 'subtask')}] {subtask.get('description', 'Implement subtask')[:100]}", - "description": format_subtask_description(subtask, phase), - "priority": priority, - "labels": labels, - "status": get_linear_status(subtask.get("status", "pending")), - } - - def record_session_result( - self, - subtask_id: str, - session_num: int, - success: bool, - approach: str = "", - error: str = "", - git_commit: str = "", - ) -> str: - """ - Record a session result as a Linear comment. - - This is called by post_session_processing in agent.py. - - Args: - subtask_id: Subtask being worked on - session_num: Session number - success: Whether the session succeeded - approach: What was attempted - error: Error message if failed - git_commit: Git commit hash if any - - Returns: - Formatted comment body (for logging even if Linear unavailable) - """ - comment = format_session_comment( - session_num=session_num, - subtask_id=subtask_id, - success=success, - approach=approach, - error=error, - git_commit=git_commit, - ) - - # Note: Actual Linear API call will be done by the agent - # This method prepares the data and returns it - return comment - - def prepare_status_update(self, subtask_id: str, new_status: str) -> dict: - """ - Prepare data for a Linear issue status update. - - Args: - subtask_id: Subtask ID - new_status: New subtask status (pending, in_progress, completed, etc.) - - Returns: - Dict with issue_id and linear_status for the update - """ - issue_id = self.get_issue_id(subtask_id) - linear_status = get_linear_status(new_status) - - return { - "issue_id": issue_id, - "status": linear_status, - "subtask_id": subtask_id, - } - - def prepare_stuck_escalation( - self, - subtask_id: str, - attempt_count: int, - attempts: list[dict], - reason: str = "", - ) -> dict: - """ - Prepare data for escalating a stuck subtask. - - This creates the comment body and status update data. - - Args: - subtask_id: Stuck subtask ID - attempt_count: Number of attempts - attempts: List of attempt records - reason: Why it's stuck - - Returns: - Dict with issue_id, comment, labels for escalation - """ - issue_id = self.get_issue_id(subtask_id) - comment = format_stuck_subtask_comment( - subtask_id=subtask_id, - attempt_count=attempt_count, - attempts=attempts, - reason=reason, - ) - - return { - "issue_id": issue_id, - "subtask_id": subtask_id, - "status": STATUS_BLOCKED, - "comment": comment, - "labels": [LABELS["stuck"], LABELS["needs_review"]], - } - - def get_progress_summary(self) -> dict: - """ - Get a summary of Linear integration progress. - - Returns: - Dict with progress statistics - """ - plan = self.load_implementation_plan() - if not plan: - return { - "enabled": self.is_enabled, - "initialized": False, - "total_subtasks": 0, - "mapped_subtasks": 0, - } - - subtasks = self.get_subtasks_for_sync() - mapped = sum(1 for s in subtasks if self.get_issue_id(s.get("id", ""))) - - return { - "enabled": self.is_enabled, - "initialized": self.is_initialized, - "team_id": self.state.team_id if self.state else None, - "project_id": self.state.project_id if self.state else None, - "project_name": self.state.project_name if self.state else None, - "meta_issue_id": self.state.meta_issue_id if self.state else None, - "total_subtasks": len(subtasks), - "mapped_subtasks": mapped, - } - - def get_linear_context_for_prompt(self) -> str: - """ - Generate Linear context section for agent prompts. - - This is included in the subtask prompt to give the agent - awareness of Linear integration status. - - Returns: - Markdown-formatted context string - """ - if not self.is_enabled: - return "" - - summary = self.get_progress_summary() - - if not summary["initialized"]: - return """ -## Linear Integration - -Linear integration is enabled but not yet initialized. -During the planner session, create a Linear project and sync issues. - -Available Linear MCP tools: -- `mcp__linear-server__list_teams` - List available teams -- `mcp__linear-server__create_project` - Create a new project -- `mcp__linear-server__create_issue` - Create issues for subtasks -- `mcp__linear-server__update_issue` - Update issue status -- `mcp__linear-server__create_comment` - Add session comments -""" - - lines = [ - "## Linear Integration", - "", - f"**Project:** {summary['project_name']}", - f"**Issues:** {summary['mapped_subtasks']}/{summary['total_subtasks']} subtasks mapped", - "", - "When working on a subtask:", - "1. Update issue status to 'In Progress' at start", - "2. Add comments with progress/blockers", - "3. Update status to 'Done' when subtask completes", - "4. If stuck, status will be set to 'Blocked' automatically", - ] - - return "\n".join(lines) - - def save_state(self) -> None: - """Save the current state to disk.""" - if self.state: - self.state.save(self.spec_dir) - - -# Utility functions for integration with other modules - - -def get_linear_manager(spec_dir: Path, project_dir: Path) -> LinearManager: - """ - Get a LinearManager instance for the given spec. - - This is the main entry point for other modules. - - Args: - spec_dir: Spec directory - project_dir: Project root directory - - Returns: - LinearManager instance - """ - return LinearManager(spec_dir, project_dir) - - -def is_linear_enabled() -> bool: - """Quick check if Linear integration is available.""" - return bool(os.environ.get("LINEAR_API_KEY")) - - -def prepare_planner_linear_instructions(spec_dir: Path) -> str: - """ - Generate Linear setup instructions for the planner agent. - - This is included in the planner prompt when Linear is enabled. - - Args: - spec_dir: Spec directory - - Returns: - Markdown instructions for Linear setup - """ - if not is_linear_enabled(): - return "" - - return """ -## Linear Integration Setup - -Linear integration is ENABLED. After creating the implementation plan: - -### Step 1: Find the Team -``` -Use mcp__linear-server__list_teams to find your team ID -``` - -### Step 2: Create the Project -``` -Use mcp__linear-server__create_project with: -- team: Your team ID -- name: The feature/spec name -- description: Brief summary from spec.md -``` -Save the project ID to .linear_project.json - -### Step 3: Create Issues for Each Subtask -For each subtask in implementation_plan.json: -``` -Use mcp__linear-server__create_issue with: -- team: Your team ID -- project: The project ID -- title: "[subtask-id] Description" -- description: Formatted subtask details -- priority: Based on phase (1=urgent for early phases, 4=low for polish) -- labels: ["auto-claude", "phase-N", "service-NAME"] -``` -Save the subtask_id -> issue_id mapping to .linear_project.json - -### Step 4: Create META Issue -``` -Use mcp__linear-server__create_issue with: -- title: "[META] Build Progress Tracker" -- description: "Session summaries and overall progress tracking" -``` -This issue receives session summary comments. - -### Important Notes -- Update .linear_project.json after each Linear operation -- The JSON structure should include: - - initialized: true - - team_id: "..." - - project_id: "..." - - meta_issue_id: "..." - - issue_mapping: { "subtask-1-1": "LIN-123", ... } -""" - - -def prepare_coder_linear_instructions( - spec_dir: Path, - subtask_id: str, -) -> str: - """ - Generate Linear instructions for the coding agent. - - Args: - spec_dir: Spec directory - subtask_id: Current subtask being worked on - - Returns: - Markdown instructions for Linear updates - """ - if not is_linear_enabled(): - return "" - - manager = LinearManager(spec_dir, spec_dir.parent.parent) # Approximate project_dir - - if not manager.is_initialized: - return "" - - issue_id = manager.get_issue_id(subtask_id) - if not issue_id: - return "" - - return f""" -## Linear Updates - -This subtask is linked to Linear issue: `{issue_id}` - -### At Session Start -Update the issue status to "In Progress": -``` -mcp__linear-server__update_issue(id="{issue_id}", state="In Progress") -``` - -### During Work -Add comments for significant progress or blockers: -``` -mcp__linear-server__create_comment(issueId="{issue_id}", body="...") -``` - -### On Completion -Update status to "Done": -``` -mcp__linear-server__update_issue(id="{issue_id}", state="Done") -``` - -### Session Summary -At session end, add a comment to the META issue with: -- What was accomplished -- Any blockers or issues found -- Recommendations for next session -""" +"""Backward compatibility shim - import from integrations.linear.integration instead.""" +from integrations.linear.integration import * diff --git a/auto-claude/linear_updater.py b/auto-claude/linear_updater.py index d60e7f25..870fdb7a 100644 --- a/auto-claude/linear_updater.py +++ b/auto-claude/linear_updater.py @@ -1,442 +1,2 @@ -""" -Linear Updater - Python-Orchestrated Linear Updates -==================================================== - -Provides reliable Linear updates via focused mini-agent calls. -Instead of relying on agents to remember Linear updates in long prompts, -the Python orchestrator triggers small, focused agents at key transitions. - -Design Principles: -- ONE task per spec (not one issue per subtask) -- Python orchestrator controls when updates happen -- Small prompts that can't lose context -- Graceful degradation if Linear unavailable - -Status Flow: - Todo -> In Progress -> In Review -> (human) -> Done - | | | - | | +-- QA approved, awaiting human merge - | +-- Planner/Coder working - +-- Task created from spec -""" - -import json -import os -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path -from typing import Optional - -from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient - -# Linear status constants (matching Valma AI team setup) -STATUS_TODO = "Todo" -STATUS_IN_PROGRESS = "In Progress" -STATUS_IN_REVIEW = "In Review" # Custom status for QA phase -STATUS_DONE = "Done" -STATUS_CANCELED = "Canceled" - -# State file name -LINEAR_TASK_FILE = ".linear_task.json" - -# Linear MCP tools needed for updates -LINEAR_TOOLS = [ - "mcp__linear-server__list_teams", - "mcp__linear-server__create_issue", - "mcp__linear-server__update_issue", - "mcp__linear-server__create_comment", - "mcp__linear-server__list_issue_statuses", -] - - -@dataclass -class LinearTaskState: - """State of a Linear task for an auto-claude spec.""" - - task_id: str | None = None - task_title: str | None = None - team_id: str | None = None - status: str = STATUS_TODO - created_at: str | None = None - - def to_dict(self) -> dict: - return { - "task_id": self.task_id, - "task_title": self.task_title, - "team_id": self.team_id, - "status": self.status, - "created_at": self.created_at, - } - - @classmethod - def from_dict(cls, data: dict) -> "LinearTaskState": - return cls( - task_id=data.get("task_id"), - task_title=data.get("task_title"), - team_id=data.get("team_id"), - status=data.get("status", STATUS_TODO), - created_at=data.get("created_at"), - ) - - def save(self, spec_dir: Path) -> None: - """Save state to the spec directory.""" - state_file = spec_dir / LINEAR_TASK_FILE - with open(state_file, "w") as f: - json.dump(self.to_dict(), f, indent=2) - - @classmethod - def load(cls, spec_dir: Path) -> Optional["LinearTaskState"]: - """Load state from the spec directory.""" - state_file = spec_dir / LINEAR_TASK_FILE - if not state_file.exists(): - return None - - try: - with open(state_file) as f: - return cls.from_dict(json.load(f)) - except (OSError, json.JSONDecodeError): - return None - - -def is_linear_enabled() -> bool: - """Check if Linear integration is available.""" - return bool(os.environ.get("LINEAR_API_KEY")) - - -def get_linear_api_key() -> str: - """Get the Linear API key from environment.""" - return os.environ.get("LINEAR_API_KEY", "") - - -def _create_linear_client() -> ClaudeSDKClient: - """ - Create a minimal Claude client with only Linear MCP tools. - Used for focused mini-agent calls. - """ - oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") - if not oauth_token: - raise ValueError("CLAUDE_CODE_OAUTH_TOKEN not set") - - linear_api_key = get_linear_api_key() - if not linear_api_key: - raise ValueError("LINEAR_API_KEY not set") - - return ClaudeSDKClient( - options=ClaudeAgentOptions( - model="claude-haiku-4-5", # Fast & cheap model for simple API calls - system_prompt="You are a Linear API assistant. Execute the requested Linear operation precisely.", - allowed_tools=LINEAR_TOOLS, - mcp_servers={ - "linear": { - "type": "http", - "url": "https://mcp.linear.app/mcp", - "headers": {"Authorization": f"Bearer {linear_api_key}"}, - } - }, - max_turns=10, # Should complete in 1-3 turns - ) - ) - - -async def _run_linear_agent(prompt: str) -> str | None: - """ - Run a focused mini-agent for a Linear operation. - - Args: - prompt: The focused prompt for the Linear operation - - Returns: - The response text, or None if failed - """ - try: - client = _create_linear_client() - - async with client: - await client.query(prompt) - - response_text = "" - async for msg in client.receive_response(): - msg_type = type(msg).__name__ - if msg_type == "AssistantMessage" and hasattr(msg, "content"): - for block in msg.content: - block_type = type(block).__name__ - if block_type == "TextBlock" and hasattr(block, "text"): - response_text += block.text - - return response_text - - except Exception as e: - print(f"Linear update failed: {e}") - return None - - -async def create_linear_task( - spec_dir: Path, - title: str, - description: str | None = None, -) -> LinearTaskState | None: - """ - Create a new Linear task for a spec. - - Called by spec_runner.py after requirements gathering. - - Args: - spec_dir: Spec directory to save state - title: Task title (the task name from user) - description: Optional task description - - Returns: - LinearTaskState if successful, None if failed - """ - if not is_linear_enabled(): - return None - - # Check if task already exists - existing = LinearTaskState.load(spec_dir) - if existing and existing.task_id: - print(f"Linear task already exists: {existing.task_id}") - return existing - - desc_part = f'\n - description: "{description}"' if description else "" - - prompt = f"""Create a Linear task with these details: - -1. First, use mcp__linear-server__list_teams to find the team ID -2. Then, use mcp__linear-server__create_issue with: - - teamId: [the team ID from step 1] - - title: "{title}"{desc_part} - -After creating the issue, tell me: -- The issue ID (like "VAL-123") -- The team ID you used - -Format your final response as: -TASK_ID: [the issue ID] -TEAM_ID: [the team ID] -""" - - response = await _run_linear_agent(prompt) - if not response: - return None - - # Parse response for task_id and team_id - task_id = None - team_id = None - - for line in response.split("\n"): - line = line.strip() - if line.startswith("TASK_ID:"): - task_id = line.replace("TASK_ID:", "").strip() - elif line.startswith("TEAM_ID:"): - team_id = line.replace("TEAM_ID:", "").strip() - - if not task_id: - print(f"Failed to parse task ID from response: {response[:200]}") - return None - - # Create and save state - state = LinearTaskState( - task_id=task_id, - task_title=title, - team_id=team_id, - status=STATUS_TODO, - created_at=datetime.now().isoformat(), - ) - state.save(spec_dir) - - print(f"Created Linear task: {task_id}") - return state - - -async def update_linear_status( - spec_dir: Path, - new_status: str, -) -> bool: - """ - Update the Linear task status. - - Args: - spec_dir: Spec directory with .linear_task.json - new_status: New status (STATUS_TODO, STATUS_IN_PROGRESS, STATUS_IN_REVIEW, STATUS_DONE) - - Returns: - True if successful, False otherwise - """ - if not is_linear_enabled(): - return False - - state = LinearTaskState.load(spec_dir) - if not state or not state.task_id: - print("No Linear task found for this spec") - return False - - # Don't update if already at this status - if state.status == new_status: - return True - - prompt = f"""Update Linear issue status: - -1. First, use mcp__linear-server__list_issue_statuses with teamId: "{state.team_id}" to find the state ID for "{new_status}" -2. Then, use mcp__linear-server__update_issue with: - - issueId: "{state.task_id}" - - stateId: [the state ID for "{new_status}" from step 1] - -Confirm when done. -""" - - response = await _run_linear_agent(prompt) - if response: - state.status = new_status - state.save(spec_dir) - print(f"Updated Linear task {state.task_id} to: {new_status}") - return True - - return False - - -async def add_linear_comment( - spec_dir: Path, - comment: str, -) -> bool: - """ - Add a comment to the Linear task. - - Args: - spec_dir: Spec directory with .linear_task.json - comment: Comment text to add - - Returns: - True if successful, False otherwise - """ - if not is_linear_enabled(): - return False - - state = LinearTaskState.load(spec_dir) - if not state or not state.task_id: - print("No Linear task found for this spec") - return False - - # Escape any quotes in the comment - safe_comment = comment.replace('"', '\\"').replace("\n", "\\n") - - prompt = f"""Add a comment to Linear issue: - -Use mcp__linear-server__create_comment with: -- issueId: "{state.task_id}" -- body: "{safe_comment}" - -Confirm when done. -""" - - response = await _run_linear_agent(prompt) - if response: - print(f"Added comment to Linear task {state.task_id}") - return True - - return False - - -# === Convenience functions for specific transitions === - - -async def linear_task_started(spec_dir: Path) -> bool: - """ - Mark task as started (In Progress). - Called when planner session begins. - """ - success = await update_linear_status(spec_dir, STATUS_IN_PROGRESS) - if success: - await add_linear_comment(spec_dir, "Build started - planning phase initiated") - return success - - -async def linear_subtask_completed( - spec_dir: Path, - subtask_id: str, - completed_count: int, - total_count: int, -) -> bool: - """ - Record subtask completion as a comment. - Called after each successful coder session. - """ - comment = f"Completed {subtask_id} ({completed_count}/{total_count} subtasks done)" - return await add_linear_comment(spec_dir, comment) - - -async def linear_subtask_failed( - spec_dir: Path, - subtask_id: str, - attempt: int, - error_summary: str, -) -> bool: - """ - Record subtask failure as a comment. - Called after failed coder session. - """ - comment = f"Subtask {subtask_id} failed (attempt {attempt}): {error_summary[:200]}" - return await add_linear_comment(spec_dir, comment) - - -async def linear_build_complete(spec_dir: Path) -> bool: - """ - Record build completion, moving to QA. - Called when all subtasks are completed. - """ - comment = "All subtasks completed - moving to QA validation" - return await add_linear_comment(spec_dir, comment) - - -async def linear_qa_started(spec_dir: Path) -> bool: - """ - Mark task as In Review for QA phase. - Called when QA validation loop starts. - """ - success = await update_linear_status(spec_dir, STATUS_IN_REVIEW) - if success: - await add_linear_comment(spec_dir, "QA validation started") - return success - - -async def linear_qa_approved(spec_dir: Path) -> bool: - """ - Record QA approval (stays In Review for human). - Called when QA approves the build. - """ - comment = "QA approved - awaiting human review for merge" - return await add_linear_comment(spec_dir, comment) - - -async def linear_qa_rejected( - spec_dir: Path, - issues_count: int, - iteration: int, -) -> bool: - """ - Record QA rejection. - Called when QA rejects the build. - """ - comment = f"QA iteration {iteration}: Found {issues_count} issues - applying fixes" - return await add_linear_comment(spec_dir, comment) - - -async def linear_qa_max_iterations(spec_dir: Path, iterations: int) -> bool: - """ - Record QA max iterations reached. - Called when QA loop exhausts retries. - """ - comment = f"QA reached max iterations ({iterations}) - needs human intervention" - return await add_linear_comment(spec_dir, comment) - - -async def linear_task_stuck( - spec_dir: Path, - subtask_id: str, - attempt_count: int, -) -> bool: - """ - Record that a subtask is stuck. - Called when subtask exceeds retry limit. - """ - comment = f"Subtask {subtask_id} is STUCK after {attempt_count} attempts - needs human review" - return await add_linear_comment(spec_dir, comment) +"""Backward compatibility shim - import from integrations.linear.updater instead.""" +from integrations.linear.updater import * diff --git a/auto-claude/memory.py b/auto-claude/memory/main.py similarity index 100% rename from auto-claude/memory.py rename to auto-claude/memory/main.py diff --git a/auto-claude/merge/file_evolution/storage.py b/auto-claude/merge/file_evolution/storage.py index e5700408..a3538cd2 100644 --- a/auto-claude/merge/file_evolution/storage.py +++ b/auto-claude/merge/file_evolution/storage.py @@ -175,7 +175,10 @@ class EvolutionStorage: path = Path(file_path) if path.is_absolute(): try: - return str(path.relative_to(self.project_dir)) + # Resolve both paths to handle symlinks (e.g., /var -> /private/var on macOS) + resolved_path = path.resolve() + return str(resolved_path.relative_to(self.project_dir)) except ValueError: + # Path is not under project_dir, return as-is return str(path) return str(path) diff --git a/auto-claude/planner.py b/auto-claude/planner_lib/main.py similarity index 100% rename from auto-claude/planner.py rename to auto-claude/planner_lib/main.py diff --git a/auto-claude/prediction.py b/auto-claude/prediction/main.py similarity index 100% rename from auto-claude/prediction.py rename to auto-claude/prediction/main.py diff --git a/auto-claude/prediction_old.py b/auto-claude/prediction_old.py deleted file mode 100644 index fad813be..00000000 --- a/auto-claude/prediction_old.py +++ /dev/null @@ -1,710 +0,0 @@ -#!/usr/bin/env python3 -""" -Predictive Bug Prevention -========================== - -Generates pre-implementation checklists to prevent common bugs BEFORE they happen. -Uses historical data from memory system and pattern analysis to predict likely issues. - -The key insight: Most bugs are predictable based on: -1. Type of work (API, frontend, database, etc.) -2. Past failures in similar subtasks -3. Known gotchas in this codebase -4. Missing integration points - -Usage: - from prediction import BugPredictor - - predictor = BugPredictor(spec_dir) - checklist = predictor.generate_checklist(subtask) - markdown = predictor.format_checklist_markdown(checklist) -""" - -import json -import re -from dataclasses import dataclass, field -from pathlib import Path - - -@dataclass -class PredictedIssue: - """A potential issue that might occur during implementation.""" - - category: str # "integration", "pattern", "edge_case", "security", "performance" - description: str - likelihood: str # "high", "medium", "low" - prevention: str # How to avoid it - - def to_dict(self) -> dict: - return { - "category": self.category, - "description": self.description, - "likelihood": self.likelihood, - "prevention": self.prevention, - } - - -@dataclass -class PreImplementationChecklist: - """Complete checklist for a subtask before implementation.""" - - subtask_id: str - subtask_description: str - predicted_issues: list[PredictedIssue] = field(default_factory=list) - patterns_to_follow: list[str] = field(default_factory=list) - files_to_reference: list[str] = field(default_factory=list) - common_mistakes: list[str] = field(default_factory=list) - verification_reminders: list[str] = field(default_factory=list) - - -class BugPredictor: - """Predicts likely bugs and generates pre-implementation checklists.""" - - def __init__(self, spec_dir: Path): - """ - Initialize the bug predictor. - - Args: - spec_dir: Path to the spec directory (e.g., auto-claude/specs/001-feature/) - """ - self.spec_dir = Path(spec_dir) - self.memory_dir = self.spec_dir / "memory" - self.gotchas_file = self.memory_dir / "gotchas.md" - self.patterns_file = self.memory_dir / "patterns.md" - self.history_file = self.memory_dir / "attempt_history.json" - - # Common issue patterns by work type - self.COMMON_ISSUES = self._get_common_issues() - - def _get_common_issues(self) -> dict[str, list[PredictedIssue]]: - """Get common issue patterns by work type.""" - return { - "api_endpoint": [ - PredictedIssue( - "integration", - "CORS configuration missing or incorrect", - "high", - "Check existing CORS setup in similar endpoints and ensure new routes are included", - ), - PredictedIssue( - "security", - "Authentication middleware not applied", - "high", - "Verify auth decorator is applied if endpoint requires authentication", - ), - PredictedIssue( - "pattern", - "Response format doesn't match API conventions", - "medium", - 'Check existing endpoints for response structure (e.g., {"data": ..., "error": ...})', - ), - PredictedIssue( - "edge_case", - "Missing input validation", - "high", - "Add validation for all user inputs to prevent invalid data and SQL injection", - ), - PredictedIssue( - "edge_case", - "Error handling not comprehensive", - "medium", - "Handle edge cases: missing fields, invalid types, database errors, etc.", - ), - ], - "database_model": [ - PredictedIssue( - "integration", - "Database migration not created or run", - "high", - "Create migration after model changes and run db upgrade before testing", - ), - PredictedIssue( - "pattern", - "Field naming doesn't match conventions", - "medium", - "Check existing models for naming style (snake_case, timestamps, etc.)", - ), - PredictedIssue( - "edge_case", - "Missing indexes on frequently queried fields", - "low", - "Add indexes for foreign keys and fields used in WHERE clauses", - ), - PredictedIssue( - "pattern", - "Relationship configuration incorrect", - "medium", - "Check existing relationships for backref and cascade patterns", - ), - ], - "frontend_component": [ - PredictedIssue( - "integration", - "API client not used correctly", - "high", - "Use existing ApiClient or hook pattern, don't call fetch() directly", - ), - PredictedIssue( - "pattern", - "State management doesn't follow conventions", - "medium", - "Follow existing hook patterns (useState, useEffect, custom hooks)", - ), - PredictedIssue( - "edge_case", - "Loading and error states not handled", - "high", - "Show loading indicator during async operations and display errors to users", - ), - PredictedIssue( - "pattern", - "Styling doesn't match design system", - "low", - "Use existing CSS classes or styled components from the design system", - ), - PredictedIssue( - "edge_case", - "Form validation missing", - "medium", - "Add client-side validation before submission and show helpful error messages", - ), - ], - "celery_task": [ - PredictedIssue( - "integration", - "Task not registered with Celery app", - "high", - "Import task in celery app initialization or __init__.py", - ), - PredictedIssue( - "pattern", - "Arguments not JSON-serializable", - "high", - "Use only JSON-serializable arguments (no objects, use IDs instead)", - ), - PredictedIssue( - "edge_case", - "Retry logic not implemented", - "medium", - "Add retry decorator for network/external service failures", - ), - PredictedIssue( - "integration", - "Task not called from correct location", - "medium", - "Call with .delay() or .apply_async() after database commit", - ), - ], - "authentication": [ - PredictedIssue( - "security", - "Password not hashed", - "high", - "Use bcrypt or similar for password hashing, never store plaintext", - ), - PredictedIssue( - "security", - "Token not validated properly", - "high", - "Verify token signature and expiration on every request", - ), - PredictedIssue( - "security", - "Session not invalidated on logout", - "medium", - "Clear session/token on logout and after password changes", - ), - ], - "database_query": [ - PredictedIssue( - "performance", - "N+1 query problem", - "medium", - "Use eager loading (joinedload/selectinload) for relationships", - ), - PredictedIssue( - "security", - "SQL injection vulnerability", - "high", - "Use parameterized queries, never concatenate user input into SQL", - ), - PredictedIssue( - "edge_case", - "Large result sets not paginated", - "medium", - "Add pagination for queries that could return many results", - ), - ], - "file_upload": [ - PredictedIssue( - "security", - "File type not validated", - "high", - "Validate file extension and MIME type, don't trust user input", - ), - PredictedIssue( - "security", - "File size not limited", - "high", - "Set maximum file size to prevent DoS attacks", - ), - PredictedIssue( - "edge_case", - "Uploaded files not cleaned up on error", - "low", - "Use try/finally or context managers to ensure cleanup", - ), - ], - } - - def load_known_gotchas(self) -> list[str]: - """Load gotchas from previous sessions.""" - if not self.gotchas_file.exists(): - return [] - - gotchas = [] - content = self.gotchas_file.read_text() - - # Parse markdown list items - for line in content.split("\n"): - line = line.strip() - if line.startswith("-") or line.startswith("*"): - gotcha = line.lstrip("-*").strip() - if gotcha: - gotchas.append(gotcha) - - return gotchas - - def load_known_patterns(self) -> list[str]: - """Load successful patterns from previous sessions.""" - if not self.patterns_file.exists(): - return [] - - patterns = [] - content = self.patterns_file.read_text() - - # Parse markdown sections - current_pattern = None - for line in content.split("\n"): - line = line.strip() - if line.startswith("##"): - # Pattern heading - current_pattern = line.lstrip("#").strip() - elif line and current_pattern: - # Pattern detail - if line.startswith("-") or line.startswith("*"): - detail = line.lstrip("-*").strip() - patterns.append(f"{current_pattern}: {detail}") - - return patterns - - def load_attempt_history(self) -> list[dict]: - """Load historical subtask attempts.""" - if not self.history_file.exists(): - return [] - - try: - with open(self.history_file) as f: - history = json.load(f) - return history.get("attempts", []) - except (OSError, json.JSONDecodeError): - return [] - - def _detect_work_type(self, subtask: dict) -> list[str]: - """ - Detect what type of work this subtask involves. - - Returns a list of work types (e.g., ["api_endpoint", "database_model"]) - """ - work_types = [] - - description = subtask.get("description", "").lower() - files = subtask.get("files_to_modify", []) + subtask.get("files_to_create", []) - service = subtask.get("service", "").lower() - - # API endpoint detection - if any( - kw in description - for kw in ["endpoint", "api", "route", "request", "response"] - ): - work_types.append("api_endpoint") - if any("routes" in f or "api" in f for f in files): - work_types.append("api_endpoint") - - # Database model detection - if any( - kw in description for kw in ["model", "database", "migration", "schema"] - ): - work_types.append("database_model") - if any("models" in f or "migration" in f for f in files): - work_types.append("database_model") - - # Frontend component detection - if service in ["frontend", "web", "ui"]: - work_types.append("frontend_component") - if any(f.endswith((".tsx", ".jsx", ".vue", ".svelte")) for f in files): - work_types.append("frontend_component") - - # Celery task detection - if "celery" in description or "task" in description or "worker" in service: - work_types.append("celery_task") - if any("task" in f for f in files): - work_types.append("celery_task") - - # Authentication detection - if any( - kw in description - for kw in ["auth", "login", "password", "token", "session"] - ): - work_types.append("authentication") - - # Database query detection - if any(kw in description for kw in ["query", "search", "filter", "fetch"]): - work_types.append("database_query") - - # File upload detection - if any(kw in description for kw in ["upload", "file", "image", "attachment"]): - work_types.append("file_upload") - - return work_types - - def analyze_subtask_risks(self, subtask: dict) -> list[PredictedIssue]: - """ - Predict likely issues for a subtask based on work type and history. - - Args: - subtask: Subtask dictionary with keys like description, files_to_modify, etc. - - Returns: - List of predicted issues - """ - issues = [] - - # Get work types - work_types = self._detect_work_type(subtask) - - # Add common issues for detected work types - for work_type in work_types: - if work_type in self.COMMON_ISSUES: - issues.extend(self.COMMON_ISSUES[work_type]) - - # Add issues from similar past failures - similar_failures = self.get_similar_past_failures(subtask) - for failure in similar_failures: - failure_reason = failure.get("failure_reason", "") - if failure_reason: - issues.append( - PredictedIssue( - "pattern", - f"Similar subtask failed: {failure_reason}", - "high", - "Review the failed attempt in memory/attempt_history.json", - ) - ) - - # Deduplicate by description - seen = set() - unique_issues = [] - for issue in issues: - if issue.description not in seen: - seen.add(issue.description) - unique_issues.append(issue) - - # Sort by likelihood (high first) - likelihood_order = {"high": 0, "medium": 1, "low": 2} - unique_issues.sort(key=lambda i: likelihood_order.get(i.likelihood, 3)) - - # Return top 7 most relevant - return unique_issues[:7] - - def get_similar_past_failures(self, subtask: dict) -> list[dict]: - """ - Find subtasks similar to this one that failed before. - - Args: - subtask: Current subtask to analyze - - Returns: - List of similar failed attempts from history - """ - history = self.load_attempt_history() - if not history: - return [] - - subtask_desc = subtask.get("description", "").lower() - subtask_files = set( - subtask.get("files_to_modify", []) + subtask.get("files_to_create", []) - ) - - similar = [] - for attempt in history: - # Only look at failures - if attempt.get("status") != "failed": - continue - - # Check similarity - attempt_desc = attempt.get("subtask_description", "").lower() - attempt_files = set(attempt.get("files_modified", [])) - - # Calculate similarity score - score = 0 - - # Description keyword overlap - subtask_keywords = set(re.findall(r"\w+", subtask_desc)) - attempt_keywords = set(re.findall(r"\w+", attempt_desc)) - common_keywords = subtask_keywords & attempt_keywords - if common_keywords: - score += len(common_keywords) - - # File overlap - common_files = subtask_files & attempt_files - if common_files: - score += len(common_files) * 3 # Files are stronger signal - - if score > 2: # Threshold for similarity - similar.append( - { - "subtask_id": attempt.get("subtask_id"), - "description": attempt.get("subtask_description"), - "failure_reason": attempt.get("error_message", "Unknown error"), - "similarity_score": score, - } - ) - - # Sort by similarity - similar.sort(key=lambda x: x["similarity_score"], reverse=True) - return similar[:3] # Top 3 similar failures - - def generate_checklist(self, subtask: dict) -> PreImplementationChecklist: - """ - Generate a complete pre-implementation checklist for a subtask. - - Args: - subtask: Subtask dictionary from implementation_plan.json - - Returns: - PreImplementationChecklist ready for formatting - """ - checklist = PreImplementationChecklist( - subtask_id=subtask.get("id", "unknown"), - subtask_description=subtask.get("description", ""), - ) - - # Predict issues - checklist.predicted_issues = self.analyze_subtask_risks(subtask) - - # Load patterns to follow - known_patterns = self.load_known_patterns() - # Filter to most relevant patterns based on subtask - work_types = self._detect_work_type(subtask) - relevant_patterns = [] - for pattern in known_patterns: - pattern_lower = pattern.lower() - # Check if pattern mentions any work type - if any(wt.replace("_", " ") in pattern_lower for wt in work_types): - relevant_patterns.append(pattern) - # Or if it mentions any file being modified - elif any( - f.split("/")[-1] in pattern_lower - for f in subtask.get("files_to_modify", []) - ): - relevant_patterns.append(pattern) - - checklist.patterns_to_follow = relevant_patterns[:5] # Top 5 - - # Files to reference (from subtask's patterns_from) - checklist.files_to_reference = subtask.get("patterns_from", []) - - # Common mistakes (gotchas from memory) - gotchas = self.load_known_gotchas() - # Filter to relevant gotchas - relevant_gotchas = [] - for gotcha in gotchas: - gotcha_lower = gotcha.lower() - # Check relevance to current subtask - if any( - kw in gotcha_lower - for kw in subtask.get("description", "").lower().split() - ): - relevant_gotchas.append(gotcha) - elif any(wt.replace("_", " ") in gotcha_lower for wt in work_types): - relevant_gotchas.append(gotcha) - - checklist.common_mistakes = relevant_gotchas[:5] # Top 5 - - # Verification reminders - verification = subtask.get("verification", {}) - if verification: - ver_type = verification.get("type") - if ver_type == "api": - checklist.verification_reminders.append( - f"Test API endpoint: {verification.get('method', 'GET')} {verification.get('url', '')}" - ) - elif ver_type == "browser": - checklist.verification_reminders.append( - f"Test in browser: {verification.get('scenario', 'Check functionality')}" - ) - elif ver_type == "command": - checklist.verification_reminders.append( - f"Run command: {verification.get('run', '')}" - ) - - return checklist - - def format_checklist_markdown(self, checklist: PreImplementationChecklist) -> str: - """ - Format checklist as markdown for agent consumption. - - Args: - checklist: PreImplementationChecklist to format - - Returns: - Markdown-formatted checklist string - """ - lines = [] - - lines.append( - f"## Pre-Implementation Checklist: {checklist.subtask_description}" - ) - lines.append("") - - # Predicted issues - if checklist.predicted_issues: - lines.append("### Predicted Issues (based on similar work)") - lines.append("") - lines.append("| Issue | Likelihood | Prevention |") - lines.append("|-------|------------|------------|") - - for issue in checklist.predicted_issues: - # Escape pipe characters in content - desc = issue.description.replace("|", "\\|") - prev = issue.prevention.replace("|", "\\|") - lines.append(f"| {desc} | {issue.likelihood.capitalize()} | {prev} |") - - lines.append("") - - # Patterns to follow - if checklist.patterns_to_follow: - lines.append("### Patterns to Follow") - lines.append("") - lines.append("From previous sessions and codebase analysis:") - for pattern in checklist.patterns_to_follow: - lines.append(f"- {pattern}") - lines.append("") - - # Known gotchas - if checklist.common_mistakes: - lines.append("### Known Gotchas in This Codebase") - lines.append("") - lines.append("From memory/gotchas.md:") - for gotcha in checklist.common_mistakes: - lines.append(f"- [ ] {gotcha}") - lines.append("") - - # Files to reference - if checklist.files_to_reference: - lines.append("### Files to Reference") - lines.append("") - for file_path in checklist.files_to_reference: - # Extract filename and suggest what to look for - filename = file_path.split("/")[-1] - lines.append( - f"- `{file_path}` - Check for similar patterns and code style" - ) - lines.append("") - - # Verification reminders - if checklist.verification_reminders: - lines.append("### Verification Reminders") - lines.append("") - for reminder in checklist.verification_reminders: - lines.append(f"- [ ] {reminder}") - lines.append("") - - # Pre-implementation checklist - lines.append("### Before You Start Implementing") - lines.append("") - lines.append("- [ ] I have read and understood all predicted issues above") - lines.append( - "- [ ] I have reviewed the reference files to understand existing patterns" - ) - lines.append("- [ ] I know how to prevent the high-likelihood issues") - lines.append("- [ ] I understand the verification requirements") - lines.append("") - - return "\n".join(lines) - - -def generate_subtask_checklist(spec_dir: Path, subtask: dict) -> str: - """ - Convenience function to generate and format a checklist for a subtask. - - Args: - spec_dir: Path to spec directory - subtask: Subtask dictionary - - Returns: - Markdown-formatted checklist - """ - predictor = BugPredictor(spec_dir) - checklist = predictor.generate_checklist(subtask) - return predictor.format_checklist_markdown(checklist) - - -# CLI for testing -if __name__ == "__main__": - import sys - - if len(sys.argv) < 2: - print("Usage: python prediction.py [--demo]") - print(" python prediction.py auto-claude/specs/001-feature/") - sys.exit(1) - - spec_dir = Path(sys.argv[1]) - - if "--demo" in sys.argv: - # Demo with sample subtask - demo_subtask = { - "id": "avatar-endpoint", - "description": "POST /api/users/avatar endpoint for uploading user avatars", - "service": "backend", - "files_to_modify": ["app/routes/users.py"], - "files_to_create": [], - "patterns_from": ["app/routes/profile.py"], - "verification": { - "type": "api", - "method": "POST", - "url": "/api/users/avatar", - "expect_status": 200, - }, - } - - checklist_md = generate_subtask_checklist(spec_dir, demo_subtask) - print(checklist_md) - else: - # Load from implementation plan - plan_file = spec_dir / "implementation_plan.json" - if not plan_file.exists(): - print(f"Error: No implementation_plan.json found in {spec_dir}") - sys.exit(1) - - with open(plan_file) as f: - plan = json.load(f) - - # Find first pending subtask - subtask = None - for phase in plan.get("phases", []): - for c in phase.get("subtasks", []): - if c.get("status") == "pending": - subtask = c - break - if subtask: - break - - if not subtask: - print("No pending subtasks found") - sys.exit(0) - - # Generate checklist - checklist_md = generate_subtask_checklist(spec_dir, subtask) - print(checklist_md) diff --git a/auto-claude/progress.py b/auto-claude/progress.py index 1e416045..329c2847 100644 --- a/auto-claude/progress.py +++ b/auto-claude/progress.py @@ -1,467 +1,2 @@ -""" -Progress Tracking Utilities -=========================== - -Functions for tracking and displaying progress of the autonomous coding agent. -Uses subtask-based implementation plans (implementation_plan.json). - -Enhanced with colored output, icons, and better visual formatting. -""" - -import json -from pathlib import Path - -from ui import ( - Icons, - bold, - box, - highlight, - icon, - muted, - print_phase_status, - print_status, - progress_bar, - success, - warning, -) - - -def count_subtasks(spec_dir: Path) -> tuple[int, int]: - """ - Count completed and total subtasks in implementation_plan.json. - - Args: - spec_dir: Directory containing implementation_plan.json - - Returns: - (completed_count, total_count) - """ - plan_file = spec_dir / "implementation_plan.json" - - if not plan_file.exists(): - return 0, 0 - - try: - with open(plan_file) as f: - plan = json.load(f) - - total = 0 - completed = 0 - - for phase in plan.get("phases", []): - for subtask in phase.get("subtasks", []): - total += 1 - if subtask.get("status") == "completed": - completed += 1 - - return completed, total - except (OSError, json.JSONDecodeError): - return 0, 0 - - -def count_subtasks_detailed(spec_dir: Path) -> dict: - """ - Count subtasks by status. - - Returns: - Dict with completed, in_progress, pending, failed counts - """ - plan_file = spec_dir / "implementation_plan.json" - - result = { - "completed": 0, - "in_progress": 0, - "pending": 0, - "failed": 0, - "total": 0, - } - - if not plan_file.exists(): - return result - - try: - with open(plan_file) as f: - plan = json.load(f) - - for phase in plan.get("phases", []): - for subtask in phase.get("subtasks", []): - result["total"] += 1 - status = subtask.get("status", "pending") - if status in result: - result[status] += 1 - else: - result["pending"] += 1 - - return result - except (OSError, json.JSONDecodeError): - return result - - -def is_build_complete(spec_dir: Path) -> bool: - """ - Check if all subtasks are completed. - - Args: - spec_dir: Directory containing implementation_plan.json - - Returns: - True if all subtasks complete, False otherwise - """ - completed, total = count_subtasks(spec_dir) - return total > 0 and completed == total - - -def get_progress_percentage(spec_dir: Path) -> float: - """ - Get the progress as a percentage. - - Args: - spec_dir: Directory containing implementation_plan.json - - Returns: - Percentage of subtasks completed (0-100) - """ - completed, total = count_subtasks(spec_dir) - if total == 0: - return 0.0 - return (completed / total) * 100 - - -def print_session_header( - session_num: int, - is_planner: bool, - subtask_id: str = None, - subtask_desc: str = None, - phase_name: str = None, - attempt: int = 1, -) -> None: - """Print a formatted header for the session.""" - session_type = "PLANNER AGENT" if is_planner else "CODING AGENT" - session_icon = Icons.GEAR if is_planner else Icons.LIGHTNING - - content = [ - bold(f"{icon(session_icon)} SESSION {session_num}: {session_type}"), - ] - - if subtask_id: - content.append("") - subtask_line = f"{icon(Icons.SUBTASK)} Subtask: {highlight(subtask_id)}" - if subtask_desc: - # Truncate long descriptions - desc = subtask_desc[:50] + "..." if len(subtask_desc) > 50 else subtask_desc - subtask_line += f" - {desc}" - content.append(subtask_line) - - if phase_name: - content.append(f"{icon(Icons.PHASE)} Phase: {phase_name}") - - if attempt > 1: - content.append(warning(f"{icon(Icons.WARNING)} Attempt: {attempt}")) - - print() - print(box(content, width=70, style="heavy")) - print() - - -def print_progress_summary(spec_dir: Path, show_next: bool = True) -> None: - """Print a summary of current progress with enhanced formatting.""" - completed, total = count_subtasks(spec_dir) - - if total > 0: - print() - # Progress bar - print(f"Progress: {progress_bar(completed, total, width=40)}") - - # Status message - if completed == total: - print_status("BUILD COMPLETE - All subtasks completed!", "success") - else: - remaining = total - completed - print_status(f"{remaining} subtasks remaining", "info") - - # Phase summary - try: - with open(spec_dir / "implementation_plan.json") as f: - plan = json.load(f) - - print("\nPhases:") - for phase in plan.get("phases", []): - phase_subtasks = phase.get("subtasks", []) - phase_completed = sum( - 1 for s in phase_subtasks if s.get("status") == "completed" - ) - phase_total = len(phase_subtasks) - phase_name = phase.get("name", phase.get("id", "Unknown")) - - if phase_completed == phase_total: - status = "complete" - elif phase_completed > 0 or any( - s.get("status") == "in_progress" for s in phase_subtasks - ): - status = "in_progress" - else: - # Check if blocked by dependencies - deps = phase.get("depends_on", []) - all_deps_complete = True - for dep_id in deps: - for p in plan.get("phases", []): - if p.get("id") == dep_id or p.get("phase") == dep_id: - p_subtasks = p.get("subtasks", []) - if not all( - s.get("status") == "completed" for s in p_subtasks - ): - all_deps_complete = False - break - status = "pending" if all_deps_complete else "blocked" - - print_phase_status(phase_name, phase_completed, phase_total, status) - - # Show next subtask if requested - if show_next and completed < total: - next_subtask = get_next_subtask(spec_dir) - if next_subtask: - print() - next_id = next_subtask.get("id", "unknown") - next_desc = next_subtask.get("description", "") - if len(next_desc) > 60: - next_desc = next_desc[:57] + "..." - print( - f" {icon(Icons.ARROW_RIGHT)} Next: {highlight(next_id)} - {next_desc}" - ) - - except (OSError, json.JSONDecodeError): - pass - else: - print() - print_status("No implementation subtasks yet - planner needs to run", "pending") - - -def print_build_complete_banner(spec_dir: Path) -> None: - """Print a completion banner.""" - content = [ - success(f"{icon(Icons.SUCCESS)} BUILD COMPLETE!"), - "", - "All subtasks have been implemented successfully.", - "", - muted("Next steps:"), - f" 1. Review the {highlight('auto-claude/*')} branch", - " 2. Run manual tests", - " 3. Create a PR and merge to main", - ] - - print() - print(box(content, width=70, style="heavy")) - print() - - -def print_paused_banner( - spec_dir: Path, - spec_name: str, - has_worktree: bool = False, -) -> None: - """Print a paused banner with resume instructions.""" - completed, total = count_subtasks(spec_dir) - - content = [ - warning(f"{icon(Icons.PAUSE)} BUILD PAUSED"), - "", - f"Progress saved: {completed}/{total} subtasks complete", - ] - - if has_worktree: - content.append("") - content.append(muted("Your build is in a separate workspace and is safe.")) - - print() - print(box(content, width=70, style="heavy")) - - -def get_plan_summary(spec_dir: Path) -> dict: - """ - Get a detailed summary of implementation plan status. - - Args: - spec_dir: Directory containing implementation_plan.json - - Returns: - Dictionary with plan statistics - """ - plan_file = spec_dir / "implementation_plan.json" - - if not plan_file.exists(): - return { - "workflow_type": None, - "total_phases": 0, - "total_subtasks": 0, - "completed_subtasks": 0, - "pending_subtasks": 0, - "in_progress_subtasks": 0, - "failed_subtasks": 0, - "phases": [], - } - - try: - with open(plan_file) as f: - plan = json.load(f) - - summary = { - "workflow_type": plan.get("workflow_type"), - "total_phases": len(plan.get("phases", [])), - "total_subtasks": 0, - "completed_subtasks": 0, - "pending_subtasks": 0, - "in_progress_subtasks": 0, - "failed_subtasks": 0, - "phases": [], - } - - for phase in plan.get("phases", []): - phase_info = { - "id": phase.get("id"), - "phase": phase.get("phase"), - "name": phase.get("name"), - "depends_on": phase.get("depends_on", []), - "subtasks": [], - "completed": 0, - "total": 0, - } - - for subtask in phase.get("subtasks", []): - status = subtask.get("status", "pending") - summary["total_subtasks"] += 1 - phase_info["total"] += 1 - - if status == "completed": - summary["completed_subtasks"] += 1 - phase_info["completed"] += 1 - elif status == "in_progress": - summary["in_progress_subtasks"] += 1 - elif status == "failed": - summary["failed_subtasks"] += 1 - else: - summary["pending_subtasks"] += 1 - - phase_info["subtasks"].append( - { - "id": subtask.get("id"), - "description": subtask.get("description"), - "status": status, - "service": subtask.get("service"), - } - ) - - summary["phases"].append(phase_info) - - return summary - - except (OSError, json.JSONDecodeError): - return { - "workflow_type": None, - "total_phases": 0, - "total_subtasks": 0, - "completed_subtasks": 0, - "pending_subtasks": 0, - "in_progress_subtasks": 0, - "failed_subtasks": 0, - "phases": [], - } - - -def get_current_phase(spec_dir: Path) -> dict | None: - """Get the current phase being worked on.""" - plan_file = spec_dir / "implementation_plan.json" - - if not plan_file.exists(): - return None - - try: - with open(plan_file) as f: - plan = json.load(f) - - for phase in plan.get("phases", []): - subtasks = phase.get("subtasks", []) - # Phase is current if it has incomplete subtasks and dependencies are met - has_incomplete = any(s.get("status") != "completed" for s in subtasks) - if has_incomplete: - return { - "id": phase.get("id"), - "phase": phase.get("phase"), - "name": phase.get("name"), - "completed": sum( - 1 for s in subtasks if s.get("status") == "completed" - ), - "total": len(subtasks), - } - - return None - - except (OSError, json.JSONDecodeError): - return None - - -def get_next_subtask(spec_dir: Path) -> dict | None: - """ - Find the next subtask to work on, respecting phase dependencies. - - Args: - spec_dir: Directory containing implementation_plan.json - - Returns: - The next subtask dict to work on, or None if all complete - """ - plan_file = spec_dir / "implementation_plan.json" - - if not plan_file.exists(): - return None - - try: - with open(plan_file) as f: - plan = json.load(f) - - phases = plan.get("phases", []) - - # Build a map of phase completion - phase_complete = {} - for phase in phases: - phase_id = phase.get("id") or phase.get("phase") - subtasks = phase.get("subtasks", []) - phase_complete[phase_id] = all( - s.get("status") == "completed" for s in subtasks - ) - - # Find next available subtask - for phase in phases: - phase_id = phase.get("id") or phase.get("phase") - depends_on = phase.get("depends_on", []) - - # Check if dependencies are satisfied - deps_satisfied = all(phase_complete.get(dep, False) for dep in depends_on) - if not deps_satisfied: - continue - - # Find first pending subtask in this phase - for subtask in phase.get("subtasks", []): - if subtask.get("status") == "pending": - return { - "phase_id": phase_id, - "phase_name": phase.get("name"), - "phase_num": phase.get("phase"), - **subtask, - } - - return None - - except (OSError, json.JSONDecodeError): - return None - - -def format_duration(seconds: float) -> str: - """Format a duration in human-readable form.""" - if seconds < 60: - return f"{seconds:.0f}s" - elif seconds < 3600: - minutes = seconds / 60 - return f"{minutes:.1f}m" - else: - hours = seconds / 3600 - return f"{hours:.1f}h" +"""Backward compatibility shim - import from core.progress instead.""" +from core.progress import * diff --git a/auto-claude/prompt_generator.py b/auto-claude/prompt_generator.py index 15d2bc9b..da412aa6 100644 --- a/auto-claude/prompt_generator.py +++ b/auto-claude/prompt_generator.py @@ -1,378 +1,2 @@ -""" -Prompt Generator -================ - -Generates minimal, focused prompts for each subtask. -Instead of a 900-line mega-prompt, each subtask gets a tailored ~100-line prompt -with only the context it needs. - -This approach: -- Reduces token usage by ~80% -- Keeps the agent focused on ONE task -- Moves bookkeeping to Python orchestration -""" - -import json -from pathlib import Path - - -def get_relative_spec_path(spec_dir: Path, project_dir: Path) -> str: - """ - Get the spec directory path relative to the project/working directory. - - This ensures the AI gets a usable path regardless of absolute locations. - - Args: - spec_dir: Absolute path to spec directory - project_dir: Absolute path to project/working directory - - Returns: - Relative path string (e.g., "./auto-claude/specs/003-new-spec") - """ - try: - # Try to make path relative to project_dir - relative = spec_dir.relative_to(project_dir) - return f"./{relative}" - except ValueError: - # If spec_dir is not under project_dir, return the name only - # This shouldn't happen if workspace.py correctly copies spec files - return f"./auto-claude/specs/{spec_dir.name}" - - -def generate_environment_context(project_dir: Path, spec_dir: Path) -> str: - """ - Generate environment context header for prompts. - - This explicitly tells the AI where it is working, preventing path confusion. - - Args: - project_dir: The working directory for the AI - spec_dir: The spec directory (may be absolute or relative) - - Returns: - Markdown string with environment context - """ - relative_spec = get_relative_spec_path(spec_dir, project_dir) - - return f"""## YOUR ENVIRONMENT - -**Working Directory:** `{project_dir}` -**Spec Location:** `{relative_spec}/` - -Your filesystem is restricted to your working directory. All file paths should be -relative to this location. Do NOT use absolute paths. - -**Important Files:** -- Spec: `{relative_spec}/spec.md` -- Plan: `{relative_spec}/implementation_plan.json` -- Progress: `{relative_spec}/build-progress.txt` -- Context: `{relative_spec}/context.json` - ---- - -""" - - -def generate_subtask_prompt( - spec_dir: Path, - project_dir: Path, - subtask: dict, - phase: dict, - attempt_count: int = 0, - recovery_hints: list[str] | None = None, -) -> str: - """ - Generate a minimal, focused prompt for implementing a single subtask. - - Args: - spec_dir: Directory containing spec files - project_dir: Root project directory (working directory) - subtask: The subtask to implement - phase: The phase containing this subtask - attempt_count: Number of previous attempts (for retry context) - recovery_hints: Hints from previous failed attempts - - Returns: - A focused prompt string (~100 lines instead of 900) - """ - subtask_id = subtask.get("id", "unknown") - description = subtask.get("description", "No description") - service = subtask.get("service", "all") - files_to_modify = subtask.get("files_to_modify", []) - files_to_create = subtask.get("files_to_create", []) - patterns_from = subtask.get("patterns_from", []) - verification = subtask.get("verification", {}) - - # Get relative spec path - relative_spec = get_relative_spec_path(spec_dir, project_dir) - - # Build the prompt - sections = [] - - # Environment context first - sections.append(generate_environment_context(project_dir, spec_dir)) - - # Header - sections.append(f"""# Subtask Implementation Task - -**Subtask ID:** `{subtask_id}` -**Phase:** {phase.get("name", phase.get("id", "Unknown"))} -**Service:** {service} - -## Description - -{description} -""") - - # Recovery context if this is a retry - if attempt_count > 0: - sections.append(f""" -## ⚠️ RETRY ATTEMPT ({attempt_count + 1}) - -This subtask has been attempted {attempt_count} time(s) before without success. -You MUST use a DIFFERENT approach than previous attempts. -""") - if recovery_hints: - sections.append("**Previous attempt insights:**") - for hint in recovery_hints: - sections.append(f"- {hint}") - sections.append("") - - # Files section - sections.append("## Files\n") - - if files_to_modify: - sections.append("**Files to Modify:**") - for f in files_to_modify: - sections.append(f"- `{f}`") - sections.append("") - - if files_to_create: - sections.append("**Files to Create:**") - for f in files_to_create: - sections.append(f"- `{f}`") - sections.append("") - - if patterns_from: - sections.append("**Pattern Files (study these first):**") - for f in patterns_from: - sections.append(f"- `{f}`") - sections.append("") - - # Verification - sections.append("## Verification\n") - v_type = verification.get("type", "manual") - - if v_type == "command": - sections.append(f"""Run this command to verify: -```bash -{verification.get("command", 'echo "No command specified"')} -``` -Expected: {verification.get("expected", "Success")} -""") - elif v_type == "api": - method = verification.get("method", "GET") - url = verification.get("url", "http://localhost") - body = verification.get("body", {}) - expected_status = verification.get("expected_status", 200) - sections.append(f"""Test the API endpoint: -```bash -curl -X {method} {url} -H "Content-Type: application/json" {f"-d '{json.dumps(body)}'" if body else ""} -``` -Expected status: {expected_status} -""") - elif v_type == "browser": - url = verification.get("url", "http://localhost:3000") - checks = verification.get("checks", []) - sections.append(f"""Open in browser: {url} - -Verify:""") - for check in checks: - sections.append(f"- [ ] {check}") - sections.append("") - elif v_type == "e2e": - steps = verification.get("steps", []) - sections.append("End-to-end verification steps:") - for i, step in enumerate(steps, 1): - sections.append(f"{i}. {step}") - sections.append("") - else: - instructions = verification.get("instructions", "Manual verification required") - sections.append(f"**Manual Verification:**\n{instructions}\n") - - # Instructions - sections.append(f"""## Instructions - -1. **Read the pattern files** to understand code style and conventions -2. **Read the files to modify** (if any) to understand current implementation -3. **Implement the subtask** following the patterns exactly -4. **Run verification** and fix any issues -5. **Commit your changes:** - ```bash - git add . - git commit -m "auto-claude: {subtask_id} - {description[:50]}" - ``` -6. **Update the plan** - set this subtask's status to "completed" in implementation_plan.json - -## Quality Checklist - -Before marking complete, verify: -- [ ] Follows patterns from reference files -- [ ] No console.log/print debugging statements -- [ ] Error handling in place -- [ ] Verification passes -- [ ] Clean commit with descriptive message - -## Important - -- Focus ONLY on this subtask - don't modify unrelated code -- If verification fails, FIX IT before committing -- If you encounter a blocker, document it in build-progress.txt -""") - - # Note: Linear updates are now handled by Python orchestrator via linear_updater.py - # Agents no longer need to call Linear MCP tools directly - - return "\n".join(sections) - - -def generate_planner_prompt(spec_dir: Path, project_dir: Path | None = None) -> str: - """ - Generate the planner prompt (used only once at start). - This is a simplified version that focuses on plan creation. - - Args: - spec_dir: Directory containing spec.md - project_dir: Working directory (for relative paths) - - Returns: - Planner prompt string - """ - # Load the full planner prompt from file - prompts_dir = Path(__file__).parent / "prompts" - planner_file = prompts_dir / "planner.md" - - if planner_file.exists(): - prompt = planner_file.read_text() - else: - prompt = ( - "Read spec.md and create implementation_plan.json with phases and subtasks." - ) - - # Use project_dir for relative paths, or infer from spec_dir - if project_dir is None: - # Infer: spec_dir is typically project/auto-claude/specs/XXX - project_dir = spec_dir.parent.parent.parent - - # Get relative path for spec directory - relative_spec = get_relative_spec_path(spec_dir, project_dir) - - # Build header with environment context - header = generate_environment_context(project_dir, spec_dir) - - # Add spec-specific instructions - header += f"""## SPEC LOCATION - -Your spec file is located at: `{relative_spec}/spec.md` - -Store all build artifacts in this spec directory: -- `{relative_spec}/implementation_plan.json` - Subtask-based implementation plan -- `{relative_spec}/build-progress.txt` - Progress notes -- `{relative_spec}/init.sh` - Environment setup script - -The project root is your current working directory. Implement code in the project root, -not in the spec directory. - ---- - -""" - # Note: Linear task creation and updates are now handled by Python orchestrator - # via linear_updater.py - agents no longer need Linear instructions in prompts - - return header + prompt - - -def load_subtask_context( - spec_dir: Path, - project_dir: Path, - subtask: dict, - max_file_lines: int = 200, -) -> dict: - """ - Load minimal context needed for a subtask. - - Args: - spec_dir: Spec directory - project_dir: Project root - subtask: The subtask being implemented - max_file_lines: Maximum lines to include per file - - Returns: - Dict with file contents and relevant context - """ - context = { - "patterns": {}, - "files_to_modify": {}, - "spec_excerpt": None, - } - - # Load pattern files (truncated) - for pattern_path in subtask.get("patterns_from", []): - full_path = project_dir / pattern_path - if full_path.exists(): - try: - lines = full_path.read_text().split("\n") - if len(lines) > max_file_lines: - content = "\n".join(lines[:max_file_lines]) - content += ( - f"\n\n... (truncated, {len(lines) - max_file_lines} more lines)" - ) - else: - content = "\n".join(lines) - context["patterns"][pattern_path] = content - except Exception: - context["patterns"][pattern_path] = "(Could not read file)" - - # Load files to modify (truncated) - for file_path in subtask.get("files_to_modify", []): - full_path = project_dir / file_path - if full_path.exists(): - try: - lines = full_path.read_text().split("\n") - if len(lines) > max_file_lines: - content = "\n".join(lines[:max_file_lines]) - content += ( - f"\n\n... (truncated, {len(lines) - max_file_lines} more lines)" - ) - else: - content = "\n".join(lines) - context["files_to_modify"][file_path] = content - except Exception: - context["files_to_modify"][file_path] = "(Could not read file)" - - return context - - -def format_context_for_prompt(context: dict) -> str: - """ - Format loaded context into a prompt section. - - Args: - context: Dict from load_subtask_context - - Returns: - Formatted string to append to prompt - """ - sections = [] - - if context.get("patterns"): - sections.append("## Reference Files (Patterns to Follow)\n") - for path, content in context["patterns"].items(): - sections.append(f"### `{path}`\n```\n{content}\n```\n") - - if context.get("files_to_modify"): - sections.append("## Current File Contents (To Modify)\n") - for path, content in context["files_to_modify"].items(): - sections.append(f"### `{path}`\n```\n{content}\n```\n") - - return "\n".join(sections) +"""Backward compatibility shim - import from prompts_pkg.prompt_generator instead.""" +from prompts_pkg.prompt_generator import * diff --git a/auto-claude/prompts.py b/auto-claude/prompts.py index fcda78fa..e1e21b2c 100644 --- a/auto-claude/prompts.py +++ b/auto-claude/prompts.py @@ -1,263 +1,2 @@ -""" -Prompt Loading Utilities -======================== - -Functions for loading agent prompts from markdown files. -""" - -import json -from pathlib import Path - -# Directory containing prompt files -PROMPTS_DIR = Path(__file__).parent / "prompts" - - -def get_planner_prompt(spec_dir: Path) -> str: - """ - Load the planner agent prompt with spec path injected. - The planner creates subtask-based implementation plans. - - Args: - spec_dir: Directory containing the spec.md file - - Returns: - The planner prompt content with spec path - """ - prompt_file = PROMPTS_DIR / "planner.md" - - if not prompt_file.exists(): - raise FileNotFoundError( - f"Planner prompt not found at {prompt_file}\n" - "Make sure the auto-claude/prompts/planner.md file exists." - ) - - prompt = prompt_file.read_text() - - # Inject spec directory information at the beginning - spec_context = f"""## SPEC LOCATION - -Your spec file is located at: `{spec_dir}/spec.md` - -Store all build artifacts in this spec directory: -- `{spec_dir}/implementation_plan.json` - Subtask-based implementation plan -- `{spec_dir}/build-progress.txt` - Progress notes -- `{spec_dir}/init.sh` - Environment setup script - -The project root is the parent of auto-claude/. Implement code in the project root, not in the spec directory. - ---- - -""" - return spec_context + prompt - - -def get_coding_prompt(spec_dir: Path) -> str: - """ - Load the coding agent prompt with spec path injected. - - Args: - spec_dir: Directory containing the spec.md and implementation_plan.json - - Returns: - The coding agent prompt content with spec path - """ - prompt_file = PROMPTS_DIR / "coder.md" - - if not prompt_file.exists(): - raise FileNotFoundError( - f"Coding prompt not found at {prompt_file}\n" - "Make sure the auto-claude/prompts/coder.md file exists." - ) - - prompt = prompt_file.read_text() - - spec_context = f"""## SPEC LOCATION - -Your spec and progress files are located at: -- Spec: `{spec_dir}/spec.md` -- Implementation plan: `{spec_dir}/implementation_plan.json` -- Progress notes: `{spec_dir}/build-progress.txt` -- Recovery context: `{spec_dir}/memory/attempt_history.json` - -The project root is the parent of auto-claude/. All code goes in the project root, not in the spec directory. - ---- - -""" - - # Check for recovery context (stuck subtasks, retry hints) - recovery_context = _get_recovery_context(spec_dir) - if recovery_context: - spec_context += recovery_context - - # Check for human input file - human_input_file = spec_dir / "HUMAN_INPUT.md" - if human_input_file.exists(): - human_input = human_input_file.read_text().strip() - if human_input: - spec_context += f"""## HUMAN INPUT (READ THIS FIRST!) - -The human has left you instructions. READ AND FOLLOW THESE CAREFULLY: - -{human_input} - -After addressing this input, you may delete or clear the HUMAN_INPUT.md file. - ---- - -""" - - return spec_context + prompt - - -def _get_recovery_context(spec_dir: Path) -> str: - """ - Get recovery context if there are failed attempts or stuck subtasks. - - Args: - spec_dir: Spec directory containing memory/ - - Returns: - Recovery context string or empty string - """ - import json - - attempt_history_file = spec_dir / "memory" / "attempt_history.json" - - if not attempt_history_file.exists(): - return "" - - try: - with open(attempt_history_file) as f: - history = json.load(f) - - # Check for stuck subtasks - stuck_subtasks = history.get("stuck_subtasks", []) - if stuck_subtasks: - context = """## ⚠️ RECOVERY ALERT - STUCK SUBTASKS DETECTED - -Some subtasks have been attempted multiple times without success. These subtasks need: -- A COMPLETELY DIFFERENT approach -- Possibly simpler implementation -- Or escalation to human if infeasible - -Stuck subtasks: -""" - for stuck in stuck_subtasks: - context += f"- {stuck['subtask_id']}: {stuck['reason']} ({stuck['attempt_count']} attempts)\n" - - context += "\nBefore working on any subtask, check memory/attempt_history.json for previous attempts!\n\n---\n\n" - return context - - # Check for subtasks with multiple attempts - subtasks_with_retries = [] - for subtask_id, subtask_data in history.get("subtasks", {}).items(): - attempts = subtask_data.get("attempts", []) - if len(attempts) > 1 and subtask_data.get("status") != "completed": - subtasks_with_retries.append((subtask_id, len(attempts))) - - if subtasks_with_retries: - context = """## ⚠️ RECOVERY CONTEXT - RETRY AWARENESS - -Some subtasks have been attempted before. When working on these: -1. READ memory/attempt_history.json for the specific subtask -2. See what approaches were tried -3. Use a DIFFERENT approach - -Subtasks with previous attempts: -""" - for subtask_id, attempt_count in subtasks_with_retries: - context += f"- {subtask_id}: {attempt_count} attempts\n" - - context += "\n---\n\n" - return context - - return "" - - except (OSError, json.JSONDecodeError): - return "" - - -def get_followup_planner_prompt(spec_dir: Path) -> str: - """ - Load the follow-up planner agent prompt with spec path and key files injected. - The follow-up planner adds new subtasks to an existing completed implementation plan. - - Args: - spec_dir: Directory containing the completed spec and implementation_plan.json - - Returns: - The follow-up planner prompt content with paths injected - """ - prompt_file = PROMPTS_DIR / "followup_planner.md" - - if not prompt_file.exists(): - raise FileNotFoundError( - f"Follow-up planner prompt not found at {prompt_file}\n" - "Make sure the auto-claude/prompts/followup_planner.md file exists." - ) - - prompt = prompt_file.read_text() - - # Inject spec directory information at the beginning - spec_context = f"""## SPEC LOCATION (FOLLOW-UP MODE) - -You are adding follow-up work to a **completed** spec. - -**Key files in this spec directory:** -- Spec: `{spec_dir}/spec.md` -- Follow-up request: `{spec_dir}/FOLLOWUP_REQUEST.md` (READ THIS FIRST!) -- Implementation plan: `{spec_dir}/implementation_plan.json` (APPEND to this, don't replace) -- Progress notes: `{spec_dir}/build-progress.txt` -- Context: `{spec_dir}/context.json` -- Memory: `{spec_dir}/memory/` - -**Important paths:** -- Spec directory: `{spec_dir}` -- Project root: Parent of auto-claude/ (where code should be implemented) - -**Your task:** -1. Read `{spec_dir}/FOLLOWUP_REQUEST.md` to understand what to add -2. Read `{spec_dir}/implementation_plan.json` to see existing phases/subtasks -3. ADD new phase(s) with pending subtasks to the existing plan -4. PRESERVE all existing subtasks and their statuses - ---- - -""" - return spec_context + prompt - - -def is_first_run(spec_dir: Path) -> bool: - """ - Check if this is the first run (no valid implementation plan with subtasks exists yet). - - The spec runner may create a skeleton implementation_plan.json with empty phases. - This function checks for actual phases with subtasks, not just file existence. - - Args: - spec_dir: Directory containing spec files - - Returns: - True if implementation_plan.json doesn't exist or has no subtasks - """ - plan_file = spec_dir / "implementation_plan.json" - - if not plan_file.exists(): - return True - - try: - with open(plan_file) as f: - plan = json.load(f) - - # Check if there are any phases with subtasks - phases = plan.get("phases", []) - if not phases: - return True - - # Check if any phase has subtasks - total_subtasks = sum(len(phase.get("subtasks", [])) for phase in phases) - return total_subtasks == 0 - except (OSError, json.JSONDecodeError): - # If we can't read the file, treat as first run - return True +"""Backward compatibility shim - import from prompts_pkg.prompts instead.""" +from prompts_pkg.prompts import * diff --git a/auto-claude/prompts_pkg/__init__.py b/auto-claude/prompts_pkg/__init__.py new file mode 100644 index 00000000..0047579d --- /dev/null +++ b/auto-claude/prompts_pkg/__init__.py @@ -0,0 +1,39 @@ +""" +Prompts Module +============== + +Prompt generation and templates for AI interactions. +""" + +# Import all functions from prompt_generator +from .prompt_generator import ( + get_relative_spec_path, + generate_environment_context, + generate_subtask_prompt, + generate_planner_prompt, + load_subtask_context, + format_context_for_prompt, +) + +# Import all functions from prompts +from .prompts import ( + get_planner_prompt, + get_coding_prompt, + get_followup_planner_prompt, + is_first_run, +) + +__all__ = [ + # prompt_generator functions + "get_relative_spec_path", + "generate_environment_context", + "generate_subtask_prompt", + "generate_planner_prompt", + "load_subtask_context", + "format_context_for_prompt", + # prompts functions + "get_planner_prompt", + "get_coding_prompt", + "get_followup_planner_prompt", + "is_first_run", +] diff --git a/auto-claude/prompts_pkg/prompt_generator.py b/auto-claude/prompts_pkg/prompt_generator.py new file mode 100644 index 00000000..15d2bc9b --- /dev/null +++ b/auto-claude/prompts_pkg/prompt_generator.py @@ -0,0 +1,378 @@ +""" +Prompt Generator +================ + +Generates minimal, focused prompts for each subtask. +Instead of a 900-line mega-prompt, each subtask gets a tailored ~100-line prompt +with only the context it needs. + +This approach: +- Reduces token usage by ~80% +- Keeps the agent focused on ONE task +- Moves bookkeeping to Python orchestration +""" + +import json +from pathlib import Path + + +def get_relative_spec_path(spec_dir: Path, project_dir: Path) -> str: + """ + Get the spec directory path relative to the project/working directory. + + This ensures the AI gets a usable path regardless of absolute locations. + + Args: + spec_dir: Absolute path to spec directory + project_dir: Absolute path to project/working directory + + Returns: + Relative path string (e.g., "./auto-claude/specs/003-new-spec") + """ + try: + # Try to make path relative to project_dir + relative = spec_dir.relative_to(project_dir) + return f"./{relative}" + except ValueError: + # If spec_dir is not under project_dir, return the name only + # This shouldn't happen if workspace.py correctly copies spec files + return f"./auto-claude/specs/{spec_dir.name}" + + +def generate_environment_context(project_dir: Path, spec_dir: Path) -> str: + """ + Generate environment context header for prompts. + + This explicitly tells the AI where it is working, preventing path confusion. + + Args: + project_dir: The working directory for the AI + spec_dir: The spec directory (may be absolute or relative) + + Returns: + Markdown string with environment context + """ + relative_spec = get_relative_spec_path(spec_dir, project_dir) + + return f"""## YOUR ENVIRONMENT + +**Working Directory:** `{project_dir}` +**Spec Location:** `{relative_spec}/` + +Your filesystem is restricted to your working directory. All file paths should be +relative to this location. Do NOT use absolute paths. + +**Important Files:** +- Spec: `{relative_spec}/spec.md` +- Plan: `{relative_spec}/implementation_plan.json` +- Progress: `{relative_spec}/build-progress.txt` +- Context: `{relative_spec}/context.json` + +--- + +""" + + +def generate_subtask_prompt( + spec_dir: Path, + project_dir: Path, + subtask: dict, + phase: dict, + attempt_count: int = 0, + recovery_hints: list[str] | None = None, +) -> str: + """ + Generate a minimal, focused prompt for implementing a single subtask. + + Args: + spec_dir: Directory containing spec files + project_dir: Root project directory (working directory) + subtask: The subtask to implement + phase: The phase containing this subtask + attempt_count: Number of previous attempts (for retry context) + recovery_hints: Hints from previous failed attempts + + Returns: + A focused prompt string (~100 lines instead of 900) + """ + subtask_id = subtask.get("id", "unknown") + description = subtask.get("description", "No description") + service = subtask.get("service", "all") + files_to_modify = subtask.get("files_to_modify", []) + files_to_create = subtask.get("files_to_create", []) + patterns_from = subtask.get("patterns_from", []) + verification = subtask.get("verification", {}) + + # Get relative spec path + relative_spec = get_relative_spec_path(spec_dir, project_dir) + + # Build the prompt + sections = [] + + # Environment context first + sections.append(generate_environment_context(project_dir, spec_dir)) + + # Header + sections.append(f"""# Subtask Implementation Task + +**Subtask ID:** `{subtask_id}` +**Phase:** {phase.get("name", phase.get("id", "Unknown"))} +**Service:** {service} + +## Description + +{description} +""") + + # Recovery context if this is a retry + if attempt_count > 0: + sections.append(f""" +## ⚠️ RETRY ATTEMPT ({attempt_count + 1}) + +This subtask has been attempted {attempt_count} time(s) before without success. +You MUST use a DIFFERENT approach than previous attempts. +""") + if recovery_hints: + sections.append("**Previous attempt insights:**") + for hint in recovery_hints: + sections.append(f"- {hint}") + sections.append("") + + # Files section + sections.append("## Files\n") + + if files_to_modify: + sections.append("**Files to Modify:**") + for f in files_to_modify: + sections.append(f"- `{f}`") + sections.append("") + + if files_to_create: + sections.append("**Files to Create:**") + for f in files_to_create: + sections.append(f"- `{f}`") + sections.append("") + + if patterns_from: + sections.append("**Pattern Files (study these first):**") + for f in patterns_from: + sections.append(f"- `{f}`") + sections.append("") + + # Verification + sections.append("## Verification\n") + v_type = verification.get("type", "manual") + + if v_type == "command": + sections.append(f"""Run this command to verify: +```bash +{verification.get("command", 'echo "No command specified"')} +``` +Expected: {verification.get("expected", "Success")} +""") + elif v_type == "api": + method = verification.get("method", "GET") + url = verification.get("url", "http://localhost") + body = verification.get("body", {}) + expected_status = verification.get("expected_status", 200) + sections.append(f"""Test the API endpoint: +```bash +curl -X {method} {url} -H "Content-Type: application/json" {f"-d '{json.dumps(body)}'" if body else ""} +``` +Expected status: {expected_status} +""") + elif v_type == "browser": + url = verification.get("url", "http://localhost:3000") + checks = verification.get("checks", []) + sections.append(f"""Open in browser: {url} + +Verify:""") + for check in checks: + sections.append(f"- [ ] {check}") + sections.append("") + elif v_type == "e2e": + steps = verification.get("steps", []) + sections.append("End-to-end verification steps:") + for i, step in enumerate(steps, 1): + sections.append(f"{i}. {step}") + sections.append("") + else: + instructions = verification.get("instructions", "Manual verification required") + sections.append(f"**Manual Verification:**\n{instructions}\n") + + # Instructions + sections.append(f"""## Instructions + +1. **Read the pattern files** to understand code style and conventions +2. **Read the files to modify** (if any) to understand current implementation +3. **Implement the subtask** following the patterns exactly +4. **Run verification** and fix any issues +5. **Commit your changes:** + ```bash + git add . + git commit -m "auto-claude: {subtask_id} - {description[:50]}" + ``` +6. **Update the plan** - set this subtask's status to "completed" in implementation_plan.json + +## Quality Checklist + +Before marking complete, verify: +- [ ] Follows patterns from reference files +- [ ] No console.log/print debugging statements +- [ ] Error handling in place +- [ ] Verification passes +- [ ] Clean commit with descriptive message + +## Important + +- Focus ONLY on this subtask - don't modify unrelated code +- If verification fails, FIX IT before committing +- If you encounter a blocker, document it in build-progress.txt +""") + + # Note: Linear updates are now handled by Python orchestrator via linear_updater.py + # Agents no longer need to call Linear MCP tools directly + + return "\n".join(sections) + + +def generate_planner_prompt(spec_dir: Path, project_dir: Path | None = None) -> str: + """ + Generate the planner prompt (used only once at start). + This is a simplified version that focuses on plan creation. + + Args: + spec_dir: Directory containing spec.md + project_dir: Working directory (for relative paths) + + Returns: + Planner prompt string + """ + # Load the full planner prompt from file + prompts_dir = Path(__file__).parent / "prompts" + planner_file = prompts_dir / "planner.md" + + if planner_file.exists(): + prompt = planner_file.read_text() + else: + prompt = ( + "Read spec.md and create implementation_plan.json with phases and subtasks." + ) + + # Use project_dir for relative paths, or infer from spec_dir + if project_dir is None: + # Infer: spec_dir is typically project/auto-claude/specs/XXX + project_dir = spec_dir.parent.parent.parent + + # Get relative path for spec directory + relative_spec = get_relative_spec_path(spec_dir, project_dir) + + # Build header with environment context + header = generate_environment_context(project_dir, spec_dir) + + # Add spec-specific instructions + header += f"""## SPEC LOCATION + +Your spec file is located at: `{relative_spec}/spec.md` + +Store all build artifacts in this spec directory: +- `{relative_spec}/implementation_plan.json` - Subtask-based implementation plan +- `{relative_spec}/build-progress.txt` - Progress notes +- `{relative_spec}/init.sh` - Environment setup script + +The project root is your current working directory. Implement code in the project root, +not in the spec directory. + +--- + +""" + # Note: Linear task creation and updates are now handled by Python orchestrator + # via linear_updater.py - agents no longer need Linear instructions in prompts + + return header + prompt + + +def load_subtask_context( + spec_dir: Path, + project_dir: Path, + subtask: dict, + max_file_lines: int = 200, +) -> dict: + """ + Load minimal context needed for a subtask. + + Args: + spec_dir: Spec directory + project_dir: Project root + subtask: The subtask being implemented + max_file_lines: Maximum lines to include per file + + Returns: + Dict with file contents and relevant context + """ + context = { + "patterns": {}, + "files_to_modify": {}, + "spec_excerpt": None, + } + + # Load pattern files (truncated) + for pattern_path in subtask.get("patterns_from", []): + full_path = project_dir / pattern_path + if full_path.exists(): + try: + lines = full_path.read_text().split("\n") + if len(lines) > max_file_lines: + content = "\n".join(lines[:max_file_lines]) + content += ( + f"\n\n... (truncated, {len(lines) - max_file_lines} more lines)" + ) + else: + content = "\n".join(lines) + context["patterns"][pattern_path] = content + except Exception: + context["patterns"][pattern_path] = "(Could not read file)" + + # Load files to modify (truncated) + for file_path in subtask.get("files_to_modify", []): + full_path = project_dir / file_path + if full_path.exists(): + try: + lines = full_path.read_text().split("\n") + if len(lines) > max_file_lines: + content = "\n".join(lines[:max_file_lines]) + content += ( + f"\n\n... (truncated, {len(lines) - max_file_lines} more lines)" + ) + else: + content = "\n".join(lines) + context["files_to_modify"][file_path] = content + except Exception: + context["files_to_modify"][file_path] = "(Could not read file)" + + return context + + +def format_context_for_prompt(context: dict) -> str: + """ + Format loaded context into a prompt section. + + Args: + context: Dict from load_subtask_context + + Returns: + Formatted string to append to prompt + """ + sections = [] + + if context.get("patterns"): + sections.append("## Reference Files (Patterns to Follow)\n") + for path, content in context["patterns"].items(): + sections.append(f"### `{path}`\n```\n{content}\n```\n") + + if context.get("files_to_modify"): + sections.append("## Current File Contents (To Modify)\n") + for path, content in context["files_to_modify"].items(): + sections.append(f"### `{path}`\n```\n{content}\n```\n") + + return "\n".join(sections) diff --git a/auto-claude/prompts_pkg/prompts.py b/auto-claude/prompts_pkg/prompts.py new file mode 100644 index 00000000..fcda78fa --- /dev/null +++ b/auto-claude/prompts_pkg/prompts.py @@ -0,0 +1,263 @@ +""" +Prompt Loading Utilities +======================== + +Functions for loading agent prompts from markdown files. +""" + +import json +from pathlib import Path + +# Directory containing prompt files +PROMPTS_DIR = Path(__file__).parent / "prompts" + + +def get_planner_prompt(spec_dir: Path) -> str: + """ + Load the planner agent prompt with spec path injected. + The planner creates subtask-based implementation plans. + + Args: + spec_dir: Directory containing the spec.md file + + Returns: + The planner prompt content with spec path + """ + prompt_file = PROMPTS_DIR / "planner.md" + + if not prompt_file.exists(): + raise FileNotFoundError( + f"Planner prompt not found at {prompt_file}\n" + "Make sure the auto-claude/prompts/planner.md file exists." + ) + + prompt = prompt_file.read_text() + + # Inject spec directory information at the beginning + spec_context = f"""## SPEC LOCATION + +Your spec file is located at: `{spec_dir}/spec.md` + +Store all build artifacts in this spec directory: +- `{spec_dir}/implementation_plan.json` - Subtask-based implementation plan +- `{spec_dir}/build-progress.txt` - Progress notes +- `{spec_dir}/init.sh` - Environment setup script + +The project root is the parent of auto-claude/. Implement code in the project root, not in the spec directory. + +--- + +""" + return spec_context + prompt + + +def get_coding_prompt(spec_dir: Path) -> str: + """ + Load the coding agent prompt with spec path injected. + + Args: + spec_dir: Directory containing the spec.md and implementation_plan.json + + Returns: + The coding agent prompt content with spec path + """ + prompt_file = PROMPTS_DIR / "coder.md" + + if not prompt_file.exists(): + raise FileNotFoundError( + f"Coding prompt not found at {prompt_file}\n" + "Make sure the auto-claude/prompts/coder.md file exists." + ) + + prompt = prompt_file.read_text() + + spec_context = f"""## SPEC LOCATION + +Your spec and progress files are located at: +- Spec: `{spec_dir}/spec.md` +- Implementation plan: `{spec_dir}/implementation_plan.json` +- Progress notes: `{spec_dir}/build-progress.txt` +- Recovery context: `{spec_dir}/memory/attempt_history.json` + +The project root is the parent of auto-claude/. All code goes in the project root, not in the spec directory. + +--- + +""" + + # Check for recovery context (stuck subtasks, retry hints) + recovery_context = _get_recovery_context(spec_dir) + if recovery_context: + spec_context += recovery_context + + # Check for human input file + human_input_file = spec_dir / "HUMAN_INPUT.md" + if human_input_file.exists(): + human_input = human_input_file.read_text().strip() + if human_input: + spec_context += f"""## HUMAN INPUT (READ THIS FIRST!) + +The human has left you instructions. READ AND FOLLOW THESE CAREFULLY: + +{human_input} + +After addressing this input, you may delete or clear the HUMAN_INPUT.md file. + +--- + +""" + + return spec_context + prompt + + +def _get_recovery_context(spec_dir: Path) -> str: + """ + Get recovery context if there are failed attempts or stuck subtasks. + + Args: + spec_dir: Spec directory containing memory/ + + Returns: + Recovery context string or empty string + """ + import json + + attempt_history_file = spec_dir / "memory" / "attempt_history.json" + + if not attempt_history_file.exists(): + return "" + + try: + with open(attempt_history_file) as f: + history = json.load(f) + + # Check for stuck subtasks + stuck_subtasks = history.get("stuck_subtasks", []) + if stuck_subtasks: + context = """## ⚠️ RECOVERY ALERT - STUCK SUBTASKS DETECTED + +Some subtasks have been attempted multiple times without success. These subtasks need: +- A COMPLETELY DIFFERENT approach +- Possibly simpler implementation +- Or escalation to human if infeasible + +Stuck subtasks: +""" + for stuck in stuck_subtasks: + context += f"- {stuck['subtask_id']}: {stuck['reason']} ({stuck['attempt_count']} attempts)\n" + + context += "\nBefore working on any subtask, check memory/attempt_history.json for previous attempts!\n\n---\n\n" + return context + + # Check for subtasks with multiple attempts + subtasks_with_retries = [] + for subtask_id, subtask_data in history.get("subtasks", {}).items(): + attempts = subtask_data.get("attempts", []) + if len(attempts) > 1 and subtask_data.get("status") != "completed": + subtasks_with_retries.append((subtask_id, len(attempts))) + + if subtasks_with_retries: + context = """## ⚠️ RECOVERY CONTEXT - RETRY AWARENESS + +Some subtasks have been attempted before. When working on these: +1. READ memory/attempt_history.json for the specific subtask +2. See what approaches were tried +3. Use a DIFFERENT approach + +Subtasks with previous attempts: +""" + for subtask_id, attempt_count in subtasks_with_retries: + context += f"- {subtask_id}: {attempt_count} attempts\n" + + context += "\n---\n\n" + return context + + return "" + + except (OSError, json.JSONDecodeError): + return "" + + +def get_followup_planner_prompt(spec_dir: Path) -> str: + """ + Load the follow-up planner agent prompt with spec path and key files injected. + The follow-up planner adds new subtasks to an existing completed implementation plan. + + Args: + spec_dir: Directory containing the completed spec and implementation_plan.json + + Returns: + The follow-up planner prompt content with paths injected + """ + prompt_file = PROMPTS_DIR / "followup_planner.md" + + if not prompt_file.exists(): + raise FileNotFoundError( + f"Follow-up planner prompt not found at {prompt_file}\n" + "Make sure the auto-claude/prompts/followup_planner.md file exists." + ) + + prompt = prompt_file.read_text() + + # Inject spec directory information at the beginning + spec_context = f"""## SPEC LOCATION (FOLLOW-UP MODE) + +You are adding follow-up work to a **completed** spec. + +**Key files in this spec directory:** +- Spec: `{spec_dir}/spec.md` +- Follow-up request: `{spec_dir}/FOLLOWUP_REQUEST.md` (READ THIS FIRST!) +- Implementation plan: `{spec_dir}/implementation_plan.json` (APPEND to this, don't replace) +- Progress notes: `{spec_dir}/build-progress.txt` +- Context: `{spec_dir}/context.json` +- Memory: `{spec_dir}/memory/` + +**Important paths:** +- Spec directory: `{spec_dir}` +- Project root: Parent of auto-claude/ (where code should be implemented) + +**Your task:** +1. Read `{spec_dir}/FOLLOWUP_REQUEST.md` to understand what to add +2. Read `{spec_dir}/implementation_plan.json` to see existing phases/subtasks +3. ADD new phase(s) with pending subtasks to the existing plan +4. PRESERVE all existing subtasks and their statuses + +--- + +""" + return spec_context + prompt + + +def is_first_run(spec_dir: Path) -> bool: + """ + Check if this is the first run (no valid implementation plan with subtasks exists yet). + + The spec runner may create a skeleton implementation_plan.json with empty phases. + This function checks for actual phases with subtasks, not just file existence. + + Args: + spec_dir: Directory containing spec files + + Returns: + True if implementation_plan.json doesn't exist or has no subtasks + """ + plan_file = spec_dir / "implementation_plan.json" + + if not plan_file.exists(): + return True + + try: + with open(plan_file) as f: + plan = json.load(f) + + # Check if there are any phases with subtasks + phases = plan.get("phases", []) + if not phases: + return True + + # Check if any phase has subtasks + total_subtasks = sum(len(phase.get("subtasks", [])) for phase in phases) + return total_subtasks == 0 + except (OSError, json.JSONDecodeError): + # If we can't read the file, treat as first run + return True diff --git a/auto-claude/qa/qa_loop.py b/auto-claude/qa/qa_loop.py new file mode 100644 index 00000000..213cd4c3 --- /dev/null +++ b/auto-claude/qa/qa_loop.py @@ -0,0 +1,97 @@ +""" +QA Validation Loop (Facade) +============================ + +This module provides backward compatibility by re-exporting the QA +validation system that has been refactored into the qa/ package. + +For new code, prefer importing directly from the qa package: + from qa import run_qa_validation_loop, should_run_qa, is_qa_approved + +Module structure: + - qa/loop.py: Main QA orchestration loop + - qa/reviewer.py: QA reviewer agent session + - qa/fixer.py: QA fixer agent session + - qa/report.py: Issue tracking, reporting, escalation + - qa/criteria.py: Acceptance criteria and status management + +Enhanced features: +- Iteration tracking with detailed history +- Recurring issue detection (3+ occurrences → human escalation) +- No-test project handling +- Integration with validation strategy and risk classification +""" + +# Re-export everything from the qa package for backward compatibility +from qa import ( + ISSUE_SIMILARITY_THRESHOLD, + # Configuration + MAX_QA_ITERATIONS, + RECURRING_ISSUE_THRESHOLD, + _issue_similarity, + _normalize_issue_key, + check_test_discovery, + create_manual_test_plan, + escalate_to_human, + # Report & tracking + get_iteration_history, + get_qa_iteration_count, + get_qa_signoff_status, + get_recurring_issue_summary, + has_recurring_issues, + is_fixes_applied, + is_no_test_project, + is_qa_approved, + is_qa_rejected, + # Criteria & status + load_implementation_plan, + load_qa_fixer_prompt, + # Agent sessions + load_qa_reviewer_prompt, + print_qa_status, + record_iteration, + run_qa_agent_session, + run_qa_fixer_session, + # Main loop + run_qa_validation_loop, + save_implementation_plan, + should_run_fixes, + should_run_qa, +) + +# Maintain original __all__ for explicit exports +__all__ = [ + # Configuration + "MAX_QA_ITERATIONS", + "RECURRING_ISSUE_THRESHOLD", + "ISSUE_SIMILARITY_THRESHOLD", + # Main loop + "run_qa_validation_loop", + # Criteria & status + "load_implementation_plan", + "save_implementation_plan", + "get_qa_signoff_status", + "is_qa_approved", + "is_qa_rejected", + "is_fixes_applied", + "get_qa_iteration_count", + "should_run_qa", + "should_run_fixes", + "print_qa_status", + # Report & tracking + "get_iteration_history", + "record_iteration", + "has_recurring_issues", + "get_recurring_issue_summary", + "escalate_to_human", + "create_manual_test_plan", + "check_test_discovery", + "is_no_test_project", + "_normalize_issue_key", + "_issue_similarity", + # Agent sessions + "load_qa_reviewer_prompt", + "run_qa_agent_session", + "load_qa_fixer_prompt", + "run_qa_fixer_session", +] diff --git a/auto-claude/qa_loop.py b/auto-claude/qa_loop.py index 213cd4c3..8d5c789a 100644 --- a/auto-claude/qa_loop.py +++ b/auto-claude/qa_loop.py @@ -1,65 +1,40 @@ -""" -QA Validation Loop (Facade) -============================ - -This module provides backward compatibility by re-exporting the QA -validation system that has been refactored into the qa/ package. - -For new code, prefer importing directly from the qa package: - from qa import run_qa_validation_loop, should_run_qa, is_qa_approved - -Module structure: - - qa/loop.py: Main QA orchestration loop - - qa/reviewer.py: QA reviewer agent session - - qa/fixer.py: QA fixer agent session - - qa/report.py: Issue tracking, reporting, escalation - - qa/criteria.py: Acceptance criteria and status management - -Enhanced features: -- Iteration tracking with detailed history -- Recurring issue detection (3+ occurrences → human escalation) -- No-test project handling -- Integration with validation strategy and risk classification -""" - -# Re-export everything from the qa package for backward compatibility +"""Backward compatibility shim - import from qa package instead.""" from qa import ( - ISSUE_SIMILARITY_THRESHOLD, # Configuration MAX_QA_ITERATIONS, RECURRING_ISSUE_THRESHOLD, - _issue_similarity, - _normalize_issue_key, - check_test_discovery, - create_manual_test_plan, - escalate_to_human, - # Report & tracking - get_iteration_history, - get_qa_iteration_count, - get_qa_signoff_status, - get_recurring_issue_summary, - has_recurring_issues, - is_fixes_applied, - is_no_test_project, - is_qa_approved, - is_qa_rejected, - # Criteria & status - load_implementation_plan, - load_qa_fixer_prompt, - # Agent sessions - load_qa_reviewer_prompt, - print_qa_status, - record_iteration, - run_qa_agent_session, - run_qa_fixer_session, + ISSUE_SIMILARITY_THRESHOLD, # Main loop run_qa_validation_loop, + # Criteria & status + load_implementation_plan, save_implementation_plan, - should_run_fixes, + get_qa_signoff_status, + is_qa_approved, + is_qa_rejected, + is_fixes_applied, + get_qa_iteration_count, should_run_qa, + should_run_fixes, + print_qa_status, + # Report & tracking + get_iteration_history, + record_iteration, + has_recurring_issues, + get_recurring_issue_summary, + escalate_to_human, + create_manual_test_plan, + check_test_discovery, + is_no_test_project, + _normalize_issue_key, + _issue_similarity, + # Agent sessions + load_qa_reviewer_prompt, + run_qa_agent_session, + load_qa_fixer_prompt, + run_qa_fixer_session, ) -# Maintain original __all__ for explicit exports __all__ = [ # Configuration "MAX_QA_ITERATIONS", diff --git a/auto-claude/recovery.py b/auto-claude/recovery.py index af6fb66c..d5fee827 100644 --- a/auto-claude/recovery.py +++ b/auto-claude/recovery.py @@ -1,606 +1,16 @@ -""" -Smart Rollback and Recovery System -=================================== - -Automatic recovery from build failures, stuck loops, and broken builds. -Enables true "walk away" automation by detecting and recovering from common failure modes. - -Key Features: -- Automatic rollback to last working state -- Circular fix detection (prevents infinite loops) -- Attempt history tracking across sessions -- Smart retry with different approaches -- Escalation to human when stuck -""" - -import json -import subprocess -from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from pathlib import Path - - -class FailureType(Enum): - """Types of failures that can occur during autonomous builds.""" - - BROKEN_BUILD = "broken_build" # Code doesn't compile/run - VERIFICATION_FAILED = "verification_failed" # Subtask verification failed - CIRCULAR_FIX = "circular_fix" # Same fix attempted multiple times - CONTEXT_EXHAUSTED = "context_exhausted" # Ran out of context mid-subtask - UNKNOWN = "unknown" - - -@dataclass -class RecoveryAction: - """Action to take in response to a failure.""" - - action: str # "rollback", "retry", "skip", "escalate" - target: str # commit hash, subtask id, or message - reason: str - - -class RecoveryManager: - """ - Manages recovery from build failures. - - Responsibilities: - - Track attempt history across sessions - - Classify failures and determine recovery actions - - Rollback to working states - - Detect circular fixes (same approach repeatedly) - - Escalate stuck subtasks for human intervention - """ - - def __init__(self, spec_dir: Path, project_dir: Path): - """ - Initialize recovery manager. - - Args: - spec_dir: Spec directory containing memory/ - project_dir: Root project directory for git operations - """ - self.spec_dir = spec_dir - self.project_dir = project_dir - self.memory_dir = spec_dir / "memory" - self.attempt_history_file = self.memory_dir / "attempt_history.json" - self.build_commits_file = self.memory_dir / "build_commits.json" - - # Ensure memory directory exists - self.memory_dir.mkdir(parents=True, exist_ok=True) - - # Initialize files if they don't exist - if not self.attempt_history_file.exists(): - self._init_attempt_history() - - if not self.build_commits_file.exists(): - self._init_build_commits() - - def _init_attempt_history(self) -> None: - """Initialize the attempt history file.""" - initial_data = { - "subtasks": {}, - "stuck_subtasks": [], - "metadata": { - "created_at": datetime.now().isoformat(), - "last_updated": datetime.now().isoformat(), - }, - } - with open(self.attempt_history_file, "w") as f: - json.dump(initial_data, f, indent=2) - - def _init_build_commits(self) -> None: - """Initialize the build commits tracking file.""" - initial_data = { - "commits": [], - "last_good_commit": None, - "metadata": { - "created_at": datetime.now().isoformat(), - "last_updated": datetime.now().isoformat(), - }, - } - with open(self.build_commits_file, "w") as f: - json.dump(initial_data, f, indent=2) - - def _load_attempt_history(self) -> dict: - """Load attempt history from JSON file.""" - try: - with open(self.attempt_history_file) as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - self._init_attempt_history() - with open(self.attempt_history_file) as f: - return json.load(f) - - def _save_attempt_history(self, data: dict) -> None: - """Save attempt history to JSON file.""" - data["metadata"]["last_updated"] = datetime.now().isoformat() - with open(self.attempt_history_file, "w") as f: - json.dump(data, f, indent=2) - - def _load_build_commits(self) -> dict: - """Load build commits from JSON file.""" - try: - with open(self.build_commits_file) as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - self._init_build_commits() - with open(self.build_commits_file) as f: - return json.load(f) - - def _save_build_commits(self, data: dict) -> None: - """Save build commits to JSON file.""" - data["metadata"]["last_updated"] = datetime.now().isoformat() - with open(self.build_commits_file, "w") as f: - json.dump(data, f, indent=2) - - def classify_failure(self, error: str, subtask_id: str) -> FailureType: - """ - Classify what type of failure occurred. - - Args: - error: Error message or description - subtask_id: ID of the subtask that failed - - Returns: - FailureType enum value - """ - error_lower = error.lower() - - # Check for broken build indicators - build_errors = [ - "syntax error", - "compilation error", - "module not found", - "import error", - "cannot find module", - "unexpected token", - "indentation error", - "parse error", - ] - if any(be in error_lower for be in build_errors): - return FailureType.BROKEN_BUILD - - # Check for verification failures - verification_errors = [ - "verification failed", - "expected", - "assertion", - "test failed", - "status code", - ] - if any(ve in error_lower for ve in verification_errors): - return FailureType.VERIFICATION_FAILED - - # Check for context exhaustion - context_errors = ["context", "token limit", "maximum length"] - if any(ce in error_lower for ce in context_errors): - return FailureType.CONTEXT_EXHAUSTED - - # Check for circular fixes (will be determined by attempt history) - if self.is_circular_fix(subtask_id, error): - return FailureType.CIRCULAR_FIX - - return FailureType.UNKNOWN - - def get_attempt_count(self, subtask_id: str) -> int: - """ - Get how many times this subtask has been attempted. - - Args: - subtask_id: ID of the subtask - - Returns: - Number of attempts - """ - history = self._load_attempt_history() - subtask_data = history["subtasks"].get(subtask_id, {}) - return len(subtask_data.get("attempts", [])) - - def record_attempt( - self, - subtask_id: str, - session: int, - success: bool, - approach: str, - error: str | None = None, - ) -> None: - """ - Record an attempt at a subtask. - - Args: - subtask_id: ID of the subtask - session: Session number - success: Whether the attempt succeeded - approach: Description of the approach taken - error: Error message if failed - """ - history = self._load_attempt_history() - - # Initialize subtask entry if it doesn't exist - if subtask_id not in history["subtasks"]: - history["subtasks"][subtask_id] = {"attempts": [], "status": "pending"} - - # Add the attempt - attempt = { - "session": session, - "timestamp": datetime.now().isoformat(), - "approach": approach, - "success": success, - "error": error, - } - history["subtasks"][subtask_id]["attempts"].append(attempt) - - # Update status - if success: - history["subtasks"][subtask_id]["status"] = "completed" - else: - history["subtasks"][subtask_id]["status"] = "failed" - - self._save_attempt_history(history) - - def is_circular_fix(self, subtask_id: str, current_approach: str) -> bool: - """ - Detect if we're trying the same approach repeatedly. - - Args: - subtask_id: ID of the subtask - current_approach: Description of current approach - - Returns: - True if this appears to be a circular fix attempt - """ - history = self._load_attempt_history() - subtask_data = history["subtasks"].get(subtask_id, {}) - attempts = subtask_data.get("attempts", []) - - if len(attempts) < 2: - return False - - # Check if last 3 attempts used similar approaches - # Simple similarity check: look for repeated keywords - recent_attempts = attempts[-3:] if len(attempts) >= 3 else attempts - - # Extract key terms from current approach (ignore common words) - stop_words = { - "with", - "using", - "the", - "a", - "an", - "and", - "or", - "but", - "in", - "on", - "at", - "to", - "for", - "trying", - } - current_keywords = set( - word for word in current_approach.lower().split() if word not in stop_words - ) - - similar_count = 0 - for attempt in recent_attempts: - attempt_keywords = set( - word - for word in attempt["approach"].lower().split() - if word not in stop_words - ) - - # Calculate Jaccard similarity (intersection over union) - overlap = len(current_keywords & attempt_keywords) - total = len(current_keywords | attempt_keywords) - - if total > 0: - similarity = overlap / total - # If >30% of meaningful words overlap, consider it similar - # This catches key technical terms appearing repeatedly - # (e.g., "async await" across multiple attempts) - if similarity > 0.3: - similar_count += 1 - - # If 2+ recent attempts were similar to current approach, it's circular - return similar_count >= 2 - - def determine_recovery_action( - self, failure_type: FailureType, subtask_id: str - ) -> RecoveryAction: - """ - Decide what to do based on failure type and history. - - Args: - failure_type: Type of failure that occurred - subtask_id: ID of the subtask that failed - - Returns: - RecoveryAction describing what to do - """ - attempt_count = self.get_attempt_count(subtask_id) - - if failure_type == FailureType.BROKEN_BUILD: - # Broken build: rollback to last good state - last_good = self.get_last_good_commit() - if last_good: - return RecoveryAction( - action="rollback", - target=last_good, - reason=f"Build broken in subtask {subtask_id}, rolling back to working state", - ) - else: - return RecoveryAction( - action="escalate", - target=subtask_id, - reason="Build broken and no good commit found to rollback to", - ) - - elif failure_type == FailureType.VERIFICATION_FAILED: - # Verification failed: retry with different approach if < 3 attempts - if attempt_count < 3: - return RecoveryAction( - action="retry", - target=subtask_id, - reason=f"Verification failed, retry with different approach (attempt {attempt_count + 1}/3)", - ) - else: - return RecoveryAction( - action="skip", - target=subtask_id, - reason=f"Verification failed after {attempt_count} attempts, marking as stuck", - ) - - elif failure_type == FailureType.CIRCULAR_FIX: - # Circular fix detected: skip and escalate - return RecoveryAction( - action="skip", - target=subtask_id, - reason="Circular fix detected - same approach tried multiple times", - ) - - elif failure_type == FailureType.CONTEXT_EXHAUSTED: - # Context exhausted: commit current progress and continue - return RecoveryAction( - action="continue", - target=subtask_id, - reason="Context exhausted, will commit progress and continue in next session", - ) - - else: # UNKNOWN - # Unknown error: retry once, then escalate - if attempt_count < 2: - return RecoveryAction( - action="retry", - target=subtask_id, - reason=f"Unknown error, retrying (attempt {attempt_count + 1}/2)", - ) - else: - return RecoveryAction( - action="escalate", - target=subtask_id, - reason=f"Unknown error persists after {attempt_count} attempts", - ) - - def get_last_good_commit(self) -> str | None: - """ - Find the most recent commit where build was working. - - Returns: - Commit hash or None - """ - commits = self._load_build_commits() - return commits.get("last_good_commit") - - def record_good_commit(self, commit_hash: str, subtask_id: str) -> None: - """ - Record a commit where the build was working. - - Args: - commit_hash: Git commit hash - subtask_id: Subtask that was successfully completed - """ - commits = self._load_build_commits() - - commit_record = { - "hash": commit_hash, - "subtask_id": subtask_id, - "timestamp": datetime.now().isoformat(), - } - - commits["commits"].append(commit_record) - commits["last_good_commit"] = commit_hash - - self._save_build_commits(commits) - - def rollback_to_commit(self, commit_hash: str) -> bool: - """ - Rollback to a specific commit. - - Args: - commit_hash: Git commit hash to rollback to - - Returns: - True if successful, False otherwise - """ - try: - # Use git reset --hard to rollback - result = subprocess.run( - ["git", "reset", "--hard", commit_hash], - cwd=self.project_dir, - capture_output=True, - text=True, - check=True, - ) - return True - except subprocess.CalledProcessError as e: - print(f"Error rolling back to {commit_hash}: {e.stderr}") - return False - - def mark_subtask_stuck(self, subtask_id: str, reason: str) -> None: - """ - Mark a subtask as needing human intervention. - - Args: - subtask_id: ID of the subtask - reason: Why it's stuck - """ - history = self._load_attempt_history() - - stuck_entry = { - "subtask_id": subtask_id, - "reason": reason, - "escalated_at": datetime.now().isoformat(), - "attempt_count": self.get_attempt_count(subtask_id), - } - - # Check if already in stuck list - existing = [ - s for s in history["stuck_subtasks"] if s["subtask_id"] == subtask_id - ] - if not existing: - history["stuck_subtasks"].append(stuck_entry) - - # Update subtask status - if subtask_id in history["subtasks"]: - history["subtasks"][subtask_id]["status"] = "stuck" - - self._save_attempt_history(history) - - def get_stuck_subtasks(self) -> list[dict]: - """ - Get all subtasks marked as stuck. - - Returns: - List of stuck subtask entries - """ - history = self._load_attempt_history() - return history.get("stuck_subtasks", []) - - def get_subtask_history(self, subtask_id: str) -> dict: - """ - Get the attempt history for a specific subtask. - - Args: - subtask_id: ID of the subtask - - Returns: - Subtask history dict with attempts - """ - history = self._load_attempt_history() - return history["subtasks"].get( - subtask_id, {"attempts": [], "status": "pending"} - ) - - def get_recovery_hints(self, subtask_id: str) -> list[str]: - """ - Get hints for recovery based on previous attempts. - - Args: - subtask_id: ID of the subtask - - Returns: - List of hint strings - """ - subtask_history = self.get_subtask_history(subtask_id) - attempts = subtask_history.get("attempts", []) - - if not attempts: - return ["This is the first attempt at this subtask"] - - hints = [f"Previous attempts: {len(attempts)}"] - - # Add info about what was tried - for i, attempt in enumerate(attempts[-3:], 1): - hints.append( - f"Attempt {i}: {attempt['approach']} - " - f"{'SUCCESS' if attempt['success'] else 'FAILED'}" - ) - if attempt.get("error"): - hints.append(f" Error: {attempt['error'][:100]}") - - # Add guidance - if len(attempts) >= 2: - hints.append( - "\n⚠️ IMPORTANT: Try a DIFFERENT approach than previous attempts" - ) - hints.append( - "Consider: different library, different pattern, or simpler implementation" - ) - - return hints - - def clear_stuck_subtasks(self) -> None: - """Clear all stuck subtasks (for manual resolution).""" - history = self._load_attempt_history() - history["stuck_subtasks"] = [] - self._save_attempt_history(history) - - def reset_subtask(self, subtask_id: str) -> None: - """ - Reset a subtask's attempt history. - - Args: - subtask_id: ID of the subtask to reset - """ - history = self._load_attempt_history() - - # Clear attempt history - if subtask_id in history["subtasks"]: - history["subtasks"][subtask_id] = {"attempts": [], "status": "pending"} - - # Remove from stuck subtasks - history["stuck_subtasks"] = [ - s for s in history["stuck_subtasks"] if s["subtask_id"] != subtask_id - ] - - self._save_attempt_history(history) - - -# Utility functions for integration with agent.py - - -def check_and_recover( - spec_dir: Path, project_dir: Path, subtask_id: str, error: str | None = None -) -> RecoveryAction | None: - """ - Check if recovery is needed and return appropriate action. - - Args: - spec_dir: Spec directory - project_dir: Project directory - subtask_id: Current subtask ID - error: Error message if any - - Returns: - RecoveryAction if recovery needed, None otherwise - """ - if not error: - return None - - manager = RecoveryManager(spec_dir, project_dir) - failure_type = manager.classify_failure(error, subtask_id) - - return manager.determine_recovery_action(failure_type, subtask_id) - - -def get_recovery_context(spec_dir: Path, project_dir: Path, subtask_id: str) -> dict: - """ - Get recovery context for a subtask (for prompt generation). - - Args: - spec_dir: Spec directory - project_dir: Project directory - subtask_id: Subtask ID - - Returns: - Dict with recovery hints and history - """ - manager = RecoveryManager(spec_dir, project_dir) - - return { - "attempt_count": manager.get_attempt_count(subtask_id), - "hints": manager.get_recovery_hints(subtask_id), - "subtask_history": manager.get_subtask_history(subtask_id), - "stuck_subtasks": manager.get_stuck_subtasks(), - } +"""Backward compatibility shim - import from services.recovery instead.""" +from services.recovery import ( + RecoveryManager, + FailureType, + RecoveryAction, + check_and_recover, + get_recovery_context, +) + +__all__ = [ + "RecoveryManager", + "FailureType", + "RecoveryAction", + "check_and_recover", + "get_recovery_context", +] diff --git a/auto-claude/review.py b/auto-claude/review/main.py similarity index 100% rename from auto-claude/review.py rename to auto-claude/review/main.py diff --git a/auto-claude/risk_classifier.py b/auto-claude/risk_classifier.py index 37488c08..742a6631 100644 --- a/auto-claude/risk_classifier.py +++ b/auto-claude/risk_classifier.py @@ -1,589 +1,30 @@ -#!/usr/bin/env python3 -""" -Risk Classifier Module -====================== - -Reads the AI-generated complexity_assessment.json and provides programmatic -access to risk classification and validation recommendations. - -This module serves as the bridge between the AI complexity assessor prompt -and the rest of the validation system. - -Usage: - from risk_classifier import RiskClassifier - - classifier = RiskClassifier() - assessment = classifier.load_assessment(spec_dir) - - if classifier.should_skip_validation(spec_dir): - print("Validation can be skipped for this task") - - test_types = classifier.get_required_test_types(spec_dir) -""" - -import json -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -# ============================================================================= -# DATA CLASSES -# ============================================================================= - - -@dataclass -class ScopeAnalysis: - """Analysis of task scope.""" - - estimated_files: int = 0 - estimated_services: int = 0 - is_cross_cutting: bool = False - notes: str = "" - - -@dataclass -class IntegrationAnalysis: - """Analysis of external integrations.""" - - external_services: list[str] = field(default_factory=list) - new_dependencies: list[str] = field(default_factory=list) - research_needed: bool = False - notes: str = "" - - -@dataclass -class InfrastructureAnalysis: - """Analysis of infrastructure requirements.""" - - docker_changes: bool = False - database_changes: bool = False - config_changes: bool = False - notes: str = "" - - -@dataclass -class KnowledgeAnalysis: - """Analysis of knowledge requirements.""" - - patterns_exist: bool = True - research_required: bool = False - unfamiliar_tech: list[str] = field(default_factory=list) - notes: str = "" - - -@dataclass -class RiskAnalysis: - """Analysis of task risk.""" - - level: str = "low" # low, medium, high - concerns: list[str] = field(default_factory=list) - notes: str = "" - - -@dataclass -class ComplexityAnalysis: - """Full complexity analysis from the AI assessor.""" - - scope: ScopeAnalysis = field(default_factory=ScopeAnalysis) - integrations: IntegrationAnalysis = field(default_factory=IntegrationAnalysis) - infrastructure: InfrastructureAnalysis = field( - default_factory=InfrastructureAnalysis - ) - knowledge: KnowledgeAnalysis = field(default_factory=KnowledgeAnalysis) - risk: RiskAnalysis = field(default_factory=RiskAnalysis) - - -@dataclass -class ValidationRecommendations: - """Validation recommendations from the AI assessor.""" - - risk_level: str = "medium" # trivial, low, medium, high, critical - skip_validation: bool = False - minimal_mode: bool = False - test_types_required: list[str] = field(default_factory=lambda: ["unit"]) - security_scan_required: bool = False - staging_deployment_required: bool = False - reasoning: str = "" - - -@dataclass -class AssessmentFlags: - """Flags indicating special requirements.""" - - needs_research: bool = False - needs_self_critique: bool = False - needs_infrastructure_setup: bool = False - - -@dataclass -class RiskAssessment: - """Complete risk assessment from complexity_assessment.json.""" - - complexity: str # simple, standard, complex - workflow_type: str # feature, refactor, investigation, migration, simple - confidence: float - reasoning: str - analysis: ComplexityAnalysis - recommended_phases: list[str] - flags: AssessmentFlags - validation: ValidationRecommendations - created_at: str | None = None - - @property - def risk_level(self) -> str: - """Get the risk level from validation recommendations.""" - return self.validation.risk_level - - -# ============================================================================= -# RISK CLASSIFIER -# ============================================================================= - - -class RiskClassifier: - """ - Reads AI-generated complexity_assessment.json and provides risk classification. - - The complexity_assessment.json is generated by the AI complexity assessor - agent using the complexity_assessor.md prompt. This module parses that output - and provides programmatic access to the risk classification. - """ - - def __init__(self) -> None: - """Initialize the risk classifier.""" - self._cache: dict[str, RiskAssessment] = {} - - def load_assessment(self, spec_dir: Path) -> RiskAssessment | None: - """ - Load complexity_assessment.json from spec directory. - - Args: - spec_dir: Path to the spec directory containing complexity_assessment.json - - Returns: - RiskAssessment object if file exists and is valid, None otherwise - """ - spec_dir = Path(spec_dir) - cache_key = str(spec_dir.resolve()) - - # Return cached result if available - if cache_key in self._cache: - return self._cache[cache_key] - - assessment_file = spec_dir / "complexity_assessment.json" - if not assessment_file.exists(): - return None - - try: - with open(assessment_file, encoding="utf-8") as f: - data = json.load(f) - - assessment = self._parse_assessment(data) - self._cache[cache_key] = assessment - return assessment - - except (json.JSONDecodeError, KeyError, TypeError) as e: - # Log error but don't crash - return None to allow fallback behavior - print(f"Warning: Failed to parse complexity_assessment.json: {e}") - return None - - def _parse_assessment(self, data: dict[str, Any]) -> RiskAssessment: - """Parse raw JSON data into a RiskAssessment object.""" - # Parse analysis sections - analysis_data = data.get("analysis", {}) - analysis = ComplexityAnalysis( - scope=self._parse_scope(analysis_data.get("scope", {})), - integrations=self._parse_integrations( - analysis_data.get("integrations", {}) - ), - infrastructure=self._parse_infrastructure( - analysis_data.get("infrastructure", {}) - ), - knowledge=self._parse_knowledge(analysis_data.get("knowledge", {})), - risk=self._parse_risk(analysis_data.get("risk", {})), - ) - - # Parse flags - flags_data = data.get("flags", {}) - flags = AssessmentFlags( - needs_research=flags_data.get("needs_research", False), - needs_self_critique=flags_data.get("needs_self_critique", False), - needs_infrastructure_setup=flags_data.get( - "needs_infrastructure_setup", False - ), - ) - - # Parse validation recommendations - validation_data = data.get("validation_recommendations", {}) - validation = self._parse_validation_recommendations(validation_data, analysis) - - return RiskAssessment( - complexity=data.get("complexity", "standard"), - workflow_type=data.get("workflow_type", "feature"), - confidence=float(data.get("confidence", 0.5)), - reasoning=data.get("reasoning", ""), - analysis=analysis, - recommended_phases=data.get("recommended_phases", []), - flags=flags, - validation=validation, - created_at=data.get("created_at"), - ) - - def _parse_scope(self, data: dict[str, Any]) -> ScopeAnalysis: - """Parse scope analysis section.""" - return ScopeAnalysis( - estimated_files=int(data.get("estimated_files", 0)), - estimated_services=int(data.get("estimated_services", 0)), - is_cross_cutting=bool(data.get("is_cross_cutting", False)), - notes=str(data.get("notes", "")), - ) - - def _parse_integrations(self, data: dict[str, Any]) -> IntegrationAnalysis: - """Parse integrations analysis section.""" - return IntegrationAnalysis( - external_services=list(data.get("external_services", [])), - new_dependencies=list(data.get("new_dependencies", [])), - research_needed=bool(data.get("research_needed", False)), - notes=str(data.get("notes", "")), - ) - - def _parse_infrastructure(self, data: dict[str, Any]) -> InfrastructureAnalysis: - """Parse infrastructure analysis section.""" - return InfrastructureAnalysis( - docker_changes=bool(data.get("docker_changes", False)), - database_changes=bool(data.get("database_changes", False)), - config_changes=bool(data.get("config_changes", False)), - notes=str(data.get("notes", "")), - ) - - def _parse_knowledge(self, data: dict[str, Any]) -> KnowledgeAnalysis: - """Parse knowledge analysis section.""" - return KnowledgeAnalysis( - patterns_exist=bool(data.get("patterns_exist", True)), - research_required=bool(data.get("research_required", False)), - unfamiliar_tech=list(data.get("unfamiliar_tech", [])), - notes=str(data.get("notes", "")), - ) - - def _parse_risk(self, data: dict[str, Any]) -> RiskAnalysis: - """Parse risk analysis section.""" - return RiskAnalysis( - level=str(data.get("level", "low")), - concerns=list(data.get("concerns", [])), - notes=str(data.get("notes", "")), - ) - - def _parse_validation_recommendations( - self, data: dict[str, Any], analysis: ComplexityAnalysis - ) -> ValidationRecommendations: - """ - Parse validation recommendations section. - - If validation_recommendations is not present in the JSON (older assessments), - infer appropriate values from the analysis. - """ - if data: - # New format with explicit validation recommendations - return ValidationRecommendations( - risk_level=str(data.get("risk_level", "medium")), - skip_validation=bool(data.get("skip_validation", False)), - minimal_mode=bool(data.get("minimal_mode", False)), - test_types_required=list(data.get("test_types_required", ["unit"])), - security_scan_required=bool(data.get("security_scan_required", False)), - staging_deployment_required=bool( - data.get("staging_deployment_required", False) - ), - reasoning=str(data.get("reasoning", "")), - ) - else: - # Infer from analysis (backward compatibility) - return self._infer_validation_recommendations(analysis) - - def _infer_validation_recommendations( - self, analysis: ComplexityAnalysis - ) -> ValidationRecommendations: - """ - Infer validation recommendations from analysis when not explicitly provided. - - This provides backward compatibility with older complexity assessments - that don't have the validation_recommendations section. - """ - risk_level = analysis.risk.level - - # Map old risk levels to new ones - risk_mapping = { - "low": "low", - "medium": "medium", - "high": "high", - } - normalized_risk = risk_mapping.get(risk_level, "medium") - - # Infer test types based on risk - test_types_map = { - "low": ["unit"], - "medium": ["unit", "integration"], - "high": ["unit", "integration", "e2e"], - } - test_types = test_types_map.get(normalized_risk, ["unit", "integration"]) - - # Security scan for high risk or security-related concerns - security_keywords = [ - "security", - "auth", - "password", - "credential", - "token", - "api key", - ] - has_security_concerns = any( - kw in str(analysis.risk.concerns).lower() for kw in security_keywords - ) - security_scan_required = normalized_risk == "high" or has_security_concerns - - # Staging for database or infrastructure changes - staging_required = ( - analysis.infrastructure.database_changes - and normalized_risk in ["medium", "high"] - ) - - # Minimal mode for simple changes - minimal_mode = ( - analysis.scope.estimated_files <= 2 - and analysis.scope.estimated_services <= 1 - and not analysis.integrations.external_services - ) - - return ValidationRecommendations( - risk_level=normalized_risk, - skip_validation=False, # Never skip by inference - minimal_mode=minimal_mode, - test_types_required=test_types, - security_scan_required=security_scan_required, - staging_deployment_required=staging_required, - reasoning="Inferred from complexity analysis (no explicit recommendations found)", - ) - - def should_skip_validation(self, spec_dir: Path) -> bool: - """ - Quick check if validation can be skipped entirely. - - Args: - spec_dir: Path to the spec directory - - Returns: - True if validation can be skipped (trivial changes), False otherwise - """ - assessment = self.load_assessment(spec_dir) - if not assessment: - return False # When in doubt, don't skip - - return assessment.validation.skip_validation - - def should_use_minimal_mode(self, spec_dir: Path) -> bool: - """ - Check if minimal validation mode should be used. - - Args: - spec_dir: Path to the spec directory - - Returns: - True if minimal mode is recommended, False otherwise - """ - assessment = self.load_assessment(spec_dir) - if not assessment: - return False - - return assessment.validation.minimal_mode - - def get_required_test_types(self, spec_dir: Path) -> list[str]: - """ - Get list of required test types based on risk. - - Args: - spec_dir: Path to the spec directory - - Returns: - List of test types (e.g., ["unit", "integration", "e2e"]) - """ - assessment = self.load_assessment(spec_dir) - if not assessment: - return ["unit"] # Default to unit tests - - return assessment.validation.test_types_required - - def requires_security_scan(self, spec_dir: Path) -> bool: - """ - Check if security scanning is required. - - Args: - spec_dir: Path to the spec directory - - Returns: - True if security scan is required, False otherwise - """ - assessment = self.load_assessment(spec_dir) - if not assessment: - return False - - return assessment.validation.security_scan_required - - def requires_staging_deployment(self, spec_dir: Path) -> bool: - """ - Check if staging deployment is required. - - Args: - spec_dir: Path to the spec directory - - Returns: - True if staging deployment is required, False otherwise - """ - assessment = self.load_assessment(spec_dir) - if not assessment: - return False - - return assessment.validation.staging_deployment_required - - def get_risk_level(self, spec_dir: Path) -> str: - """ - Get the risk level for the task. - - Args: - spec_dir: Path to the spec directory - - Returns: - Risk level string (trivial, low, medium, high, critical) - """ - assessment = self.load_assessment(spec_dir) - if not assessment: - return "medium" # Default to medium when unknown - - return assessment.validation.risk_level - - def get_complexity(self, spec_dir: Path) -> str: - """ - Get the complexity level for the task. - - Args: - spec_dir: Path to the spec directory - - Returns: - Complexity level string (simple, standard, complex) - """ - assessment = self.load_assessment(spec_dir) - if not assessment: - return "standard" # Default to standard when unknown - - return assessment.complexity - - def get_validation_summary(self, spec_dir: Path) -> dict[str, Any]: - """ - Get a summary of validation requirements. - - Args: - spec_dir: Path to the spec directory - - Returns: - Dictionary with validation summary - """ - assessment = self.load_assessment(spec_dir) - if not assessment: - return { - "risk_level": "unknown", - "complexity": "unknown", - "skip_validation": False, - "minimal_mode": False, - "test_types": ["unit"], - "security_scan": False, - "staging_deployment": False, - "confidence": 0.0, - } - - return { - "risk_level": assessment.validation.risk_level, - "complexity": assessment.complexity, - "skip_validation": assessment.validation.skip_validation, - "minimal_mode": assessment.validation.minimal_mode, - "test_types": assessment.validation.test_types_required, - "security_scan": assessment.validation.security_scan_required, - "staging_deployment": assessment.validation.staging_deployment_required, - "confidence": assessment.confidence, - "reasoning": assessment.validation.reasoning, - } - - def clear_cache(self) -> None: - """Clear the internal cache of loaded assessments.""" - self._cache.clear() - - -# ============================================================================= -# CONVENIENCE FUNCTIONS -# ============================================================================= - - -def load_risk_assessment(spec_dir: Path) -> RiskAssessment | None: - """ - Convenience function to load a risk assessment. - - Args: - spec_dir: Path to the spec directory - - Returns: - RiskAssessment object or None - """ - classifier = RiskClassifier() - return classifier.load_assessment(spec_dir) - - -def get_validation_requirements(spec_dir: Path) -> dict[str, Any]: - """ - Convenience function to get validation requirements. - - Args: - spec_dir: Path to the spec directory - - Returns: - Dictionary with validation requirements - """ - classifier = RiskClassifier() - return classifier.get_validation_summary(spec_dir) - - -# ============================================================================= -# CLI -# ============================================================================= - - -def main() -> None: - """CLI entry point for testing.""" - import argparse - - parser = argparse.ArgumentParser(description="Load and display risk assessment") - parser.add_argument( - "spec_dir", - type=Path, - help="Path to spec directory with complexity_assessment.json", - ) - parser.add_argument("--json", action="store_true", help="Output as JSON") - - args = parser.parse_args() - - classifier = RiskClassifier() - summary = classifier.get_validation_summary(args.spec_dir) - - if args.json: - print(json.dumps(summary, indent=2)) - else: - print(f"Risk Level: {summary['risk_level']}") - print(f"Complexity: {summary['complexity']}") - print(f"Skip Validation: {summary['skip_validation']}") - print(f"Minimal Mode: {summary['minimal_mode']}") - print(f"Test Types: {', '.join(summary['test_types'])}") - print(f"Security Scan: {summary['security_scan']}") - print(f"Staging Deployment: {summary['staging_deployment']}") - print(f"Confidence: {summary['confidence']:.2f}") - if summary.get("reasoning"): - print(f"Reasoning: {summary['reasoning']}") - - -if __name__ == "__main__": - main() +"""Backward compatibility shim - import from analysis.risk_classifier instead.""" +from analysis.risk_classifier import ( + RiskClassifier, + RiskAssessment, + ValidationRecommendations, + ComplexityAnalysis, + ScopeAnalysis, + IntegrationAnalysis, + InfrastructureAnalysis, + KnowledgeAnalysis, + RiskAnalysis, + AssessmentFlags, + load_risk_assessment, + get_validation_requirements, +) + +__all__ = [ + "RiskClassifier", + "RiskAssessment", + "ValidationRecommendations", + "ComplexityAnalysis", + "ScopeAnalysis", + "IntegrationAnalysis", + "InfrastructureAnalysis", + "KnowledgeAnalysis", + "RiskAnalysis", + "AssessmentFlags", + "load_risk_assessment", + "get_validation_requirements", +] diff --git a/auto-claude/runners/__init__.py b/auto-claude/runners/__init__.py new file mode 100644 index 00000000..ae49fb7f --- /dev/null +++ b/auto-claude/runners/__init__.py @@ -0,0 +1,21 @@ +""" +Runners Module +============== + +Standalone runners for various Auto Claude capabilities. +Each runner can be invoked from CLI or programmatically. +""" + +from .spec_runner import main as run_spec +from .roadmap_runner import main as run_roadmap +from .ideation_runner import main as run_ideation +from .insights_runner import main as run_insights +from .ai_analyzer_runner import main as run_ai_analyzer + +__all__ = [ + "run_spec", + "run_roadmap", + "run_ideation", + "run_insights", + "run_ai_analyzer", +] diff --git a/auto-claude/ai_analyzer/EXAMPLES.md b/auto-claude/runners/ai_analyzer/EXAMPLES.md similarity index 100% rename from auto-claude/ai_analyzer/EXAMPLES.md rename to auto-claude/runners/ai_analyzer/EXAMPLES.md diff --git a/auto-claude/ai_analyzer/README.md b/auto-claude/runners/ai_analyzer/README.md similarity index 100% rename from auto-claude/ai_analyzer/README.md rename to auto-claude/runners/ai_analyzer/README.md diff --git a/auto-claude/ai_analyzer/REFACTORING.md b/auto-claude/runners/ai_analyzer/REFACTORING.md similarity index 100% rename from auto-claude/ai_analyzer/REFACTORING.md rename to auto-claude/runners/ai_analyzer/REFACTORING.md diff --git a/auto-claude/ai_analyzer/__init__.py b/auto-claude/runners/ai_analyzer/__init__.py similarity index 100% rename from auto-claude/ai_analyzer/__init__.py rename to auto-claude/runners/ai_analyzer/__init__.py diff --git a/auto-claude/ai_analyzer/analyzers.py b/auto-claude/runners/ai_analyzer/analyzers.py similarity index 100% rename from auto-claude/ai_analyzer/analyzers.py rename to auto-claude/runners/ai_analyzer/analyzers.py diff --git a/auto-claude/ai_analyzer/cache_manager.py b/auto-claude/runners/ai_analyzer/cache_manager.py similarity index 100% rename from auto-claude/ai_analyzer/cache_manager.py rename to auto-claude/runners/ai_analyzer/cache_manager.py diff --git a/auto-claude/ai_analyzer/claude_client.py b/auto-claude/runners/ai_analyzer/claude_client.py similarity index 100% rename from auto-claude/ai_analyzer/claude_client.py rename to auto-claude/runners/ai_analyzer/claude_client.py diff --git a/auto-claude/ai_analyzer/cost_estimator.py b/auto-claude/runners/ai_analyzer/cost_estimator.py similarity index 100% rename from auto-claude/ai_analyzer/cost_estimator.py rename to auto-claude/runners/ai_analyzer/cost_estimator.py diff --git a/auto-claude/ai_analyzer/models.py b/auto-claude/runners/ai_analyzer/models.py similarity index 100% rename from auto-claude/ai_analyzer/models.py rename to auto-claude/runners/ai_analyzer/models.py diff --git a/auto-claude/ai_analyzer/result_parser.py b/auto-claude/runners/ai_analyzer/result_parser.py similarity index 100% rename from auto-claude/ai_analyzer/result_parser.py rename to auto-claude/runners/ai_analyzer/result_parser.py diff --git a/auto-claude/ai_analyzer/runner.py b/auto-claude/runners/ai_analyzer/runner.py similarity index 100% rename from auto-claude/ai_analyzer/runner.py rename to auto-claude/runners/ai_analyzer/runner.py diff --git a/auto-claude/ai_analyzer/summary_printer.py b/auto-claude/runners/ai_analyzer/summary_printer.py similarity index 100% rename from auto-claude/ai_analyzer/summary_printer.py rename to auto-claude/runners/ai_analyzer/summary_printer.py diff --git a/auto-claude/ai_analyzer_runner.py b/auto-claude/runners/ai_analyzer_runner.py similarity index 100% rename from auto-claude/ai_analyzer_runner.py rename to auto-claude/runners/ai_analyzer_runner.py diff --git a/auto-claude/ideation_runner.py b/auto-claude/runners/ideation_runner.py similarity index 100% rename from auto-claude/ideation_runner.py rename to auto-claude/runners/ideation_runner.py diff --git a/auto-claude/insights_runner.py b/auto-claude/runners/insights_runner.py similarity index 100% rename from auto-claude/insights_runner.py rename to auto-claude/runners/insights_runner.py diff --git a/auto-claude/roadmap/__init__.py b/auto-claude/runners/roadmap/__init__.py similarity index 100% rename from auto-claude/roadmap/__init__.py rename to auto-claude/runners/roadmap/__init__.py diff --git a/auto-claude/roadmap/competitor_analyzer.py b/auto-claude/runners/roadmap/competitor_analyzer.py similarity index 100% rename from auto-claude/roadmap/competitor_analyzer.py rename to auto-claude/runners/roadmap/competitor_analyzer.py diff --git a/auto-claude/roadmap/executor.py b/auto-claude/runners/roadmap/executor.py similarity index 100% rename from auto-claude/roadmap/executor.py rename to auto-claude/runners/roadmap/executor.py diff --git a/auto-claude/roadmap/graph_integration.py b/auto-claude/runners/roadmap/graph_integration.py similarity index 100% rename from auto-claude/roadmap/graph_integration.py rename to auto-claude/runners/roadmap/graph_integration.py diff --git a/auto-claude/roadmap/models.py b/auto-claude/runners/roadmap/models.py similarity index 100% rename from auto-claude/roadmap/models.py rename to auto-claude/runners/roadmap/models.py diff --git a/auto-claude/roadmap/orchestrator.py b/auto-claude/runners/roadmap/orchestrator.py similarity index 100% rename from auto-claude/roadmap/orchestrator.py rename to auto-claude/runners/roadmap/orchestrator.py diff --git a/auto-claude/roadmap/phases.py b/auto-claude/runners/roadmap/phases.py similarity index 100% rename from auto-claude/roadmap/phases.py rename to auto-claude/runners/roadmap/phases.py diff --git a/auto-claude/roadmap/project_index.json b/auto-claude/runners/roadmap/project_index.json similarity index 100% rename from auto-claude/roadmap/project_index.json rename to auto-claude/runners/roadmap/project_index.json diff --git a/auto-claude/roadmap_runner.py b/auto-claude/runners/roadmap_runner.py similarity index 100% rename from auto-claude/roadmap_runner.py rename to auto-claude/runners/roadmap_runner.py diff --git a/auto-claude/spec_runner.py b/auto-claude/runners/spec_runner.py similarity index 100% rename from auto-claude/spec_runner.py rename to auto-claude/runners/spec_runner.py diff --git a/auto-claude/scan_secrets.py b/auto-claude/scan_secrets.py index ab4c3138..abb89026 100644 --- a/auto-claude/scan_secrets.py +++ b/auto-claude/scan_secrets.py @@ -1,561 +1,2 @@ -#!/usr/bin/env python3 -""" -Secret Scanning Script for Auto-Build Framework -================================================ - -Scans staged git files for potential secrets before commit. -Designed to prevent accidental exposure of API keys, tokens, and credentials. - -Usage: - python scan_secrets.py [--staged-only] [--all-files] [--path PATH] - -Exit codes: - 0 - No secrets detected - 1 - Potential secrets found (commit should be blocked) - 2 - Error occurred during scanning -""" - -import argparse -import re -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path - -# ============================================================================= -# SECRET PATTERNS -# ============================================================================= - -# Generic high-entropy patterns that match common API key formats -GENERIC_PATTERNS = [ - # Generic API key patterns (32+ char alphanumeric strings assigned to variables) - ( - r'(?:api[_-]?key|apikey|api_secret|secret[_-]?key)\s*[:=]\s*["\']([a-zA-Z0-9_-]{32,})["\']', - "Generic API key assignment", - ), - # Generic token patterns - ( - r'(?:access[_-]?token|auth[_-]?token|bearer[_-]?token|token)\s*[:=]\s*["\']([a-zA-Z0-9_-]{32,})["\']', - "Generic access token", - ), - # Password patterns - ( - r'(?:password|passwd|pwd|pass)\s*[:=]\s*["\']([^"\']{8,})["\']', - "Password assignment", - ), - # Generic secret patterns - ( - r'(?:secret|client_secret|app_secret)\s*[:=]\s*["\']([a-zA-Z0-9_/+=]{16,})["\']', - "Secret assignment", - ), - # Bearer tokens in headers - (r'["\']?[Bb]earer\s+([a-zA-Z0-9_-]{20,})["\']?', "Bearer token"), - # Base64-encoded secrets (longer than typical, may be credentials) - (r'["\'][A-Za-z0-9+/]{64,}={0,2}["\']', "Potential base64-encoded secret"), -] - -# Service-specific patterns (known formats) -SERVICE_PATTERNS = [ - # OpenAI / Anthropic style keys - (r"sk-[a-zA-Z0-9]{20,}", "OpenAI/Anthropic-style API key"), - (r"sk-ant-[a-zA-Z0-9-]{20,}", "Anthropic API key"), - (r"sk-proj-[a-zA-Z0-9-]{20,}", "OpenAI project API key"), - # AWS - (r"AKIA[0-9A-Z]{16}", "AWS Access Key ID"), - ( - r'(?:aws_secret_access_key|aws_secret)\s*[:=]\s*["\']?([a-zA-Z0-9/+=]{40})["\']?', - "AWS Secret Access Key", - ), - # Google Cloud - (r"AIza[0-9A-Za-z_-]{35}", "Google API Key"), - (r'"type"\s*:\s*"service_account"', "Google Service Account JSON"), - # GitHub - (r"ghp_[a-zA-Z0-9]{36}", "GitHub Personal Access Token"), - (r"github_pat_[a-zA-Z0-9_]{22,}", "GitHub Fine-grained PAT"), - (r"gho_[a-zA-Z0-9]{36}", "GitHub OAuth Token"), - (r"ghs_[a-zA-Z0-9]{36}", "GitHub App Installation Token"), - (r"ghr_[a-zA-Z0-9]{36}", "GitHub Refresh Token"), - # Stripe - (r"sk_live_[0-9a-zA-Z]{24,}", "Stripe Live Secret Key"), - (r"sk_test_[0-9a-zA-Z]{24,}", "Stripe Test Secret Key"), - (r"pk_live_[0-9a-zA-Z]{24,}", "Stripe Live Publishable Key"), - (r"rk_live_[0-9a-zA-Z]{24,}", "Stripe Restricted Key"), - # Slack - (r"xox[baprs]-[0-9a-zA-Z-]{10,}", "Slack Token"), - (r"https://hooks\.slack\.com/services/[A-Z0-9/]+", "Slack Webhook URL"), - # Discord - (r"[MN][A-Za-z\d]{23,}\.[\w-]{6}\.[\w-]{27}", "Discord Bot Token"), - (r"https://discord(?:app)?\.com/api/webhooks/\d+/[\w-]+", "Discord Webhook URL"), - # Twilio - (r"SK[a-f0-9]{32}", "Twilio API Key"), - (r"AC[a-f0-9]{32}", "Twilio Account SID"), - # SendGrid - (r"SG\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9_-]{43}", "SendGrid API Key"), - # Mailchimp - (r"[a-f0-9]{32}-us\d+", "Mailchimp API Key"), - # NPM - (r"npm_[a-zA-Z0-9]{36}", "NPM Access Token"), - # PyPI - (r"pypi-[a-zA-Z0-9]{60,}", "PyPI API Token"), - # Supabase/JWT - (r"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[A-Za-z0-9_-]{50,}", "Supabase/JWT Token"), - # Linear - (r"lin_api_[a-zA-Z0-9]{40,}", "Linear API Key"), - # Vercel - (r"[a-zA-Z0-9]{24}_[a-zA-Z0-9]{28,}", "Potential Vercel Token"), - # Heroku - ( - r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}", - "Heroku API Key / UUID", - ), - # Doppler - (r"dp\.pt\.[a-zA-Z0-9]{40,}", "Doppler Service Token"), -] - -# Private key patterns -PRIVATE_KEY_PATTERNS = [ - (r"-----BEGIN\s+(RSA\s+)?PRIVATE\s+KEY-----", "RSA Private Key"), - (r"-----BEGIN\s+OPENSSH\s+PRIVATE\s+KEY-----", "OpenSSH Private Key"), - (r"-----BEGIN\s+DSA\s+PRIVATE\s+KEY-----", "DSA Private Key"), - (r"-----BEGIN\s+EC\s+PRIVATE\s+KEY-----", "EC Private Key"), - (r"-----BEGIN\s+PGP\s+PRIVATE\s+KEY\s+BLOCK-----", "PGP Private Key"), - (r"-----BEGIN\s+CERTIFICATE-----", "Certificate (may contain private key)"), -] - -# Database connection strings with embedded credentials -DATABASE_PATTERNS = [ - ( - r'mongodb(?:\+srv)?://[^"\s:]+:[^@"\s]+@[^\s"]+', - "MongoDB Connection String with credentials", - ), - ( - r'postgres(?:ql)?://[^"\s:]+:[^@"\s]+@[^\s"]+', - "PostgreSQL Connection String with credentials", - ), - (r'mysql://[^"\s:]+:[^@"\s]+@[^\s"]+', "MySQL Connection String with credentials"), - (r'redis://[^"\s:]+:[^@"\s]+@[^\s"]+', "Redis Connection String with credentials"), - ( - r'amqp://[^"\s:]+:[^@"\s]+@[^\s"]+', - "RabbitMQ Connection String with credentials", - ), -] - -# Combine all patterns -ALL_PATTERNS = ( - GENERIC_PATTERNS + SERVICE_PATTERNS + PRIVATE_KEY_PATTERNS + DATABASE_PATTERNS -) - - -# ============================================================================= -# DATA CLASSES -# ============================================================================= - - -@dataclass -class SecretMatch: - """A potential secret found in a file.""" - - file_path: str - line_number: int - pattern_name: str - matched_text: str - line_content: str - - -# ============================================================================= -# IGNORE LIST -# ============================================================================= - -# Files/directories to always skip -DEFAULT_IGNORE_PATTERNS = [ - r"\.git/", - r"node_modules/", - r"\.venv/", - r"venv/", - r"__pycache__/", - r"\.pyc$", - r"dist/", - r"build/", - r"\.egg-info/", - r"\.example$", - r"\.sample$", - r"\.template$", - r"\.md$", # Documentation files - r"\.rst$", - r"\.txt$", - r"package-lock\.json$", - r"yarn\.lock$", - r"pnpm-lock\.yaml$", - r"Cargo\.lock$", - r"poetry\.lock$", -] - -# Binary file extensions to skip -BINARY_EXTENSIONS = { - ".png", - ".jpg", - ".jpeg", - ".gif", - ".ico", - ".webp", - ".svg", - ".woff", - ".woff2", - ".ttf", - ".eot", - ".otf", - ".pdf", - ".doc", - ".docx", - ".xls", - ".xlsx", - ".zip", - ".tar", - ".gz", - ".bz2", - ".7z", - ".rar", - ".exe", - ".dll", - ".so", - ".dylib", - ".mp3", - ".mp4", - ".wav", - ".avi", - ".mov", - ".pyc", - ".pyo", - ".class", - ".o", -} - -# False positive patterns to filter out -FALSE_POSITIVE_PATTERNS = [ - r"process\.env\.", # Environment variable references - r"os\.environ", # Python env references - r"ENV\[", # Ruby/other env references - r"\$\{[A-Z_]+\}", # Shell variable substitution - r"your[-_]?api[-_]?key", # Placeholder values - r"xxx+", # Placeholder - r"placeholder", # Placeholder - r"example", # Example value - r"sample", # Sample value - r"test[-_]?key", # Test placeholder - r"<[A-Z_]+>", # Placeholder like - r"TODO", # Comment markers - r"FIXME", - r"CHANGEME", - r"INSERT[-_]?YOUR", - r"REPLACE[-_]?WITH", -] - - -# ============================================================================= -# CORE FUNCTIONS -# ============================================================================= - - -def load_secretsignore(project_dir: Path) -> list[str]: - """Load custom ignore patterns from .secretsignore file.""" - ignore_file = project_dir / ".secretsignore" - if not ignore_file.exists(): - return [] - - patterns = [] - try: - content = ignore_file.read_text() - for line in content.splitlines(): - line = line.strip() - # Skip comments and empty lines - if line and not line.startswith("#"): - patterns.append(line) - except OSError: - pass - - return patterns - - -def should_skip_file(file_path: str, custom_ignores: list[str]) -> bool: - """Check if a file should be skipped based on ignore patterns.""" - path = Path(file_path) - - # Check binary extensions - if path.suffix.lower() in BINARY_EXTENSIONS: - return True - - # Check default ignore patterns - for pattern in DEFAULT_IGNORE_PATTERNS: - if re.search(pattern, file_path): - return True - - # Check custom ignore patterns - for pattern in custom_ignores: - if re.search(pattern, file_path): - return True - - return False - - -def is_false_positive(line: str, matched_text: str) -> bool: - """Check if a match is likely a false positive.""" - line_lower = line.lower() - - for pattern in FALSE_POSITIVE_PATTERNS: - if re.search(pattern, line_lower): - return True - - # Check if it's just a variable name or type hint - if re.match(r"^[a-z_]+:\s*str\s*$", line.strip(), re.IGNORECASE): - return True - - # Check if it's in a comment - stripped = line.strip() - if ( - stripped.startswith("#") - or stripped.startswith("//") - or stripped.startswith("*") - ): - # But still flag if there's an actual long key-like string - if not re.search(r"[a-zA-Z0-9_-]{40,}", matched_text): - return True - - return False - - -def mask_secret(text: str, visible_chars: int = 8) -> str: - """Mask a secret, showing only first few characters.""" - if len(text) <= visible_chars: - return text - return text[:visible_chars] + "***" - - -def scan_content(content: str, file_path: str) -> list[SecretMatch]: - """Scan file content for potential secrets.""" - matches = [] - lines = content.splitlines() - - for line_num, line in enumerate(lines, 1): - for pattern, pattern_name in ALL_PATTERNS: - try: - for match in re.finditer(pattern, line, re.IGNORECASE): - matched_text = match.group(0) - - # Skip false positives - if is_false_positive(line, matched_text): - continue - - matches.append( - SecretMatch( - file_path=file_path, - line_number=line_num, - pattern_name=pattern_name, - matched_text=matched_text, - line_content=line.strip()[:100], # Truncate long lines - ) - ) - except re.error: - # Invalid regex, skip - continue - - return matches - - -def get_staged_files() -> list[str]: - """Get list of staged files from git (excluding deleted files).""" - try: - result = subprocess.run( - ["git", "diff", "--cached", "--name-only", "--diff-filter=ACM"], - capture_output=True, - text=True, - check=True, - ) - files = [f.strip() for f in result.stdout.splitlines() if f.strip()] - return files - except subprocess.CalledProcessError: - return [] - - -def get_all_tracked_files() -> list[str]: - """Get all tracked files in the repository.""" - try: - result = subprocess.run( - ["git", "ls-files"], - capture_output=True, - text=True, - check=True, - ) - files = [f.strip() for f in result.stdout.splitlines() if f.strip()] - return files - except subprocess.CalledProcessError: - return [] - - -def scan_files( - files: list[str], - project_dir: Path | None = None, -) -> list[SecretMatch]: - """Scan a list of files for secrets.""" - if project_dir is None: - project_dir = Path.cwd() - - custom_ignores = load_secretsignore(project_dir) - all_matches = [] - - for file_path in files: - # Skip files based on ignore patterns - if should_skip_file(file_path, custom_ignores): - continue - - full_path = project_dir / file_path - - # Skip if file doesn't exist or is a directory - if not full_path.exists() or full_path.is_dir(): - continue - - try: - content = full_path.read_text(encoding="utf-8", errors="ignore") - matches = scan_content(content, file_path) - all_matches.extend(matches) - except (OSError, UnicodeDecodeError): - # Skip files that can't be read - continue - - return all_matches - - -# ============================================================================= -# OUTPUT FORMATTING -# ============================================================================= - -# ANSI color codes -RED = "\033[0;31m" -GREEN = "\033[0;32m" -YELLOW = "\033[1;33m" -CYAN = "\033[0;36m" -NC = "\033[0m" # No Color - - -def print_results(matches: list[SecretMatch]) -> None: - """Print scan results in a formatted way.""" - if not matches: - print(f"{GREEN}No secrets detected. Commit allowed.{NC}") - return - - print(f"{RED}POTENTIAL SECRETS DETECTED!{NC}") - print(f"{RED}{'=' * 60}{NC}") - - # Group by file - files_with_matches: dict[str, list[SecretMatch]] = {} - for match in matches: - if match.file_path not in files_with_matches: - files_with_matches[match.file_path] = [] - files_with_matches[match.file_path].append(match) - - for file_path, file_matches in files_with_matches.items(): - print(f"\n{YELLOW}File: {file_path}{NC}") - for match in file_matches: - masked = mask_secret(match.matched_text) - print(f" Line {match.line_number}: [{match.pattern_name}]") - print(f" {CYAN}{masked}{NC}") - - print(f"\n{RED}{'=' * 60}{NC}") - print(f"\n{YELLOW}If these are false positives, you can:{NC}") - print(" 1. Add patterns to .secretsignore (create if needed)") - print(" 2. Use environment variables instead of hardcoded values") - print() - print(f"{RED}Commit blocked to protect against leaking secrets.{NC}") - - -def print_json_results(matches: list[SecretMatch]) -> None: - """Print scan results as JSON (for programmatic use).""" - import json - - results = { - "secrets_found": len(matches) > 0, - "count": len(matches), - "matches": [ - { - "file": m.file_path, - "line": m.line_number, - "type": m.pattern_name, - "preview": mask_secret(m.matched_text), - } - for m in matches - ], - } - print(json.dumps(results, indent=2)) - - -# ============================================================================= -# MAIN -# ============================================================================= - - -def main() -> int: - """Main entry point.""" - parser = argparse.ArgumentParser( - description="Scan files for potential secrets before commit" - ) - parser.add_argument( - "--staged-only", - "-s", - action="store_true", - default=True, - help="Only scan staged files (default)", - ) - parser.add_argument( - "--all-files", "-a", action="store_true", help="Scan all tracked files" - ) - parser.add_argument( - "--path", "-p", type=str, help="Scan a specific file or directory" - ) - parser.add_argument("--json", action="store_true", help="Output results as JSON") - parser.add_argument( - "--quiet", "-q", action="store_true", help="Only output if secrets are found" - ) - - args = parser.parse_args() - - project_dir = Path.cwd() - - # Determine which files to scan - if args.path: - path = Path(args.path) - if path.is_file(): - files = [str(path)] - elif path.is_dir(): - files = [ - str(f.relative_to(project_dir)) for f in path.rglob("*") if f.is_file() - ] - else: - print(f"{RED}Error: Path not found: {args.path}{NC}", file=sys.stderr) - return 2 - elif args.all_files: - files = get_all_tracked_files() - else: - files = get_staged_files() - - if not files: - if not args.quiet: - print(f"{GREEN}No files to scan.{NC}") - return 0 - - if not args.quiet and not args.json: - print(f"Scanning {len(files)} file(s) for secrets...") - - # Scan files - matches = scan_files(files, project_dir) - - # Output results - if args.json: - print_json_results(matches) - elif matches or not args.quiet: - print_results(matches) - - # Return exit code - return 1 if matches else 0 - - -if __name__ == "__main__": - sys.exit(main()) +"""Backward compatibility shim - import from security.scan_secrets instead.""" +from security.scan_secrets import * diff --git a/auto-claude/security.py b/auto-claude/security.py index 13364900..49caf200 100644 --- a/auto-claude/security.py +++ b/auto-claude/security.py @@ -1,94 +1,2 @@ -""" -Security Hooks for Auto-Build Framework -======================================= - -BACKWARD COMPATIBILITY FACADE - -This module maintains the original API for backward compatibility. -All functionality has been refactored into the security/ submodule: - -- security/validator.py - Command validation logic -- security/parser.py - Command parsing utilities -- security/profile.py - Security profile management -- security/hooks.py - Security hook implementations -- security/__init__.py - Public API exports - -See security/ directory for the actual implementation. - -The security system has three layers: -1. Base commands - Always allowed (core shell utilities) -2. Stack commands - Detected from project structure (frameworks, languages) -3. Custom commands - User-defined allowlist - -See project_analyzer.py for the detection logic. -""" - -# Import everything from the security module to maintain backward compatibility -from security import * # noqa: F401, F403 - -# Explicitly import commonly used items for clarity -from security import ( - BASE_COMMANDS, - VALIDATORS, - SecurityProfile, - bash_security_hook, - extract_commands, - get_command_for_validation, - get_security_profile, - is_command_allowed, - needs_validation, - reset_profile_cache, - split_command_segments, - validate_command, -) - -# Re-export for backward compatibility -__all__ = [ - "bash_security_hook", - "validate_command", - "get_security_profile", - "reset_profile_cache", - "extract_commands", - "split_command_segments", - "get_command_for_validation", - "VALIDATORS", - "SecurityProfile", - "is_command_allowed", - "needs_validation", - "BASE_COMMANDS", -] - - -# ============================================================================= -# CLI for testing (maintained for backward compatibility) -# ============================================================================= - -if __name__ == "__main__": - import sys - from pathlib import Path - - if len(sys.argv) < 2: - print("Usage: python security.py ") - print(" python security.py --list [project_dir]") - sys.exit(1) - - if sys.argv[1] == "--list": - # List all allowed commands for a project - project_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path.cwd() - profile = get_security_profile(project_dir) - - print("\nAllowed commands:") - for cmd in sorted(profile.get_all_allowed_commands()): - print(f" {cmd}") - - print(f"\nTotal: {len(profile.get_all_allowed_commands())} commands") - else: - # Validate a command - command = " ".join(sys.argv[1:]) - is_allowed, reason = validate_command(command) - - if is_allowed: - print(f"✓ ALLOWED: {command}") - else: - print(f"✗ BLOCKED: {command}") - print(f" Reason: {reason}") +"""Backward compatibility shim - import from security module instead.""" +from security import * diff --git a/auto-claude/security/main.py b/auto-claude/security/main.py new file mode 100644 index 00000000..13364900 --- /dev/null +++ b/auto-claude/security/main.py @@ -0,0 +1,94 @@ +""" +Security Hooks for Auto-Build Framework +======================================= + +BACKWARD COMPATIBILITY FACADE + +This module maintains the original API for backward compatibility. +All functionality has been refactored into the security/ submodule: + +- security/validator.py - Command validation logic +- security/parser.py - Command parsing utilities +- security/profile.py - Security profile management +- security/hooks.py - Security hook implementations +- security/__init__.py - Public API exports + +See security/ directory for the actual implementation. + +The security system has three layers: +1. Base commands - Always allowed (core shell utilities) +2. Stack commands - Detected from project structure (frameworks, languages) +3. Custom commands - User-defined allowlist + +See project_analyzer.py for the detection logic. +""" + +# Import everything from the security module to maintain backward compatibility +from security import * # noqa: F401, F403 + +# Explicitly import commonly used items for clarity +from security import ( + BASE_COMMANDS, + VALIDATORS, + SecurityProfile, + bash_security_hook, + extract_commands, + get_command_for_validation, + get_security_profile, + is_command_allowed, + needs_validation, + reset_profile_cache, + split_command_segments, + validate_command, +) + +# Re-export for backward compatibility +__all__ = [ + "bash_security_hook", + "validate_command", + "get_security_profile", + "reset_profile_cache", + "extract_commands", + "split_command_segments", + "get_command_for_validation", + "VALIDATORS", + "SecurityProfile", + "is_command_allowed", + "needs_validation", + "BASE_COMMANDS", +] + + +# ============================================================================= +# CLI for testing (maintained for backward compatibility) +# ============================================================================= + +if __name__ == "__main__": + import sys + from pathlib import Path + + if len(sys.argv) < 2: + print("Usage: python security.py ") + print(" python security.py --list [project_dir]") + sys.exit(1) + + if sys.argv[1] == "--list": + # List all allowed commands for a project + project_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path.cwd() + profile = get_security_profile(project_dir) + + print("\nAllowed commands:") + for cmd in sorted(profile.get_all_allowed_commands()): + print(f" {cmd}") + + print(f"\nTotal: {len(profile.get_all_allowed_commands())} commands") + else: + # Validate a command + command = " ".join(sys.argv[1:]) + is_allowed, reason = validate_command(command) + + if is_allowed: + print(f"✓ ALLOWED: {command}") + else: + print(f"✗ BLOCKED: {command}") + print(f" Reason: {reason}") diff --git a/auto-claude/security/scan_secrets.py b/auto-claude/security/scan_secrets.py new file mode 100644 index 00000000..ab4c3138 --- /dev/null +++ b/auto-claude/security/scan_secrets.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +""" +Secret Scanning Script for Auto-Build Framework +================================================ + +Scans staged git files for potential secrets before commit. +Designed to prevent accidental exposure of API keys, tokens, and credentials. + +Usage: + python scan_secrets.py [--staged-only] [--all-files] [--path PATH] + +Exit codes: + 0 - No secrets detected + 1 - Potential secrets found (commit should be blocked) + 2 - Error occurred during scanning +""" + +import argparse +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +# ============================================================================= +# SECRET PATTERNS +# ============================================================================= + +# Generic high-entropy patterns that match common API key formats +GENERIC_PATTERNS = [ + # Generic API key patterns (32+ char alphanumeric strings assigned to variables) + ( + r'(?:api[_-]?key|apikey|api_secret|secret[_-]?key)\s*[:=]\s*["\']([a-zA-Z0-9_-]{32,})["\']', + "Generic API key assignment", + ), + # Generic token patterns + ( + r'(?:access[_-]?token|auth[_-]?token|bearer[_-]?token|token)\s*[:=]\s*["\']([a-zA-Z0-9_-]{32,})["\']', + "Generic access token", + ), + # Password patterns + ( + r'(?:password|passwd|pwd|pass)\s*[:=]\s*["\']([^"\']{8,})["\']', + "Password assignment", + ), + # Generic secret patterns + ( + r'(?:secret|client_secret|app_secret)\s*[:=]\s*["\']([a-zA-Z0-9_/+=]{16,})["\']', + "Secret assignment", + ), + # Bearer tokens in headers + (r'["\']?[Bb]earer\s+([a-zA-Z0-9_-]{20,})["\']?', "Bearer token"), + # Base64-encoded secrets (longer than typical, may be credentials) + (r'["\'][A-Za-z0-9+/]{64,}={0,2}["\']', "Potential base64-encoded secret"), +] + +# Service-specific patterns (known formats) +SERVICE_PATTERNS = [ + # OpenAI / Anthropic style keys + (r"sk-[a-zA-Z0-9]{20,}", "OpenAI/Anthropic-style API key"), + (r"sk-ant-[a-zA-Z0-9-]{20,}", "Anthropic API key"), + (r"sk-proj-[a-zA-Z0-9-]{20,}", "OpenAI project API key"), + # AWS + (r"AKIA[0-9A-Z]{16}", "AWS Access Key ID"), + ( + r'(?:aws_secret_access_key|aws_secret)\s*[:=]\s*["\']?([a-zA-Z0-9/+=]{40})["\']?', + "AWS Secret Access Key", + ), + # Google Cloud + (r"AIza[0-9A-Za-z_-]{35}", "Google API Key"), + (r'"type"\s*:\s*"service_account"', "Google Service Account JSON"), + # GitHub + (r"ghp_[a-zA-Z0-9]{36}", "GitHub Personal Access Token"), + (r"github_pat_[a-zA-Z0-9_]{22,}", "GitHub Fine-grained PAT"), + (r"gho_[a-zA-Z0-9]{36}", "GitHub OAuth Token"), + (r"ghs_[a-zA-Z0-9]{36}", "GitHub App Installation Token"), + (r"ghr_[a-zA-Z0-9]{36}", "GitHub Refresh Token"), + # Stripe + (r"sk_live_[0-9a-zA-Z]{24,}", "Stripe Live Secret Key"), + (r"sk_test_[0-9a-zA-Z]{24,}", "Stripe Test Secret Key"), + (r"pk_live_[0-9a-zA-Z]{24,}", "Stripe Live Publishable Key"), + (r"rk_live_[0-9a-zA-Z]{24,}", "Stripe Restricted Key"), + # Slack + (r"xox[baprs]-[0-9a-zA-Z-]{10,}", "Slack Token"), + (r"https://hooks\.slack\.com/services/[A-Z0-9/]+", "Slack Webhook URL"), + # Discord + (r"[MN][A-Za-z\d]{23,}\.[\w-]{6}\.[\w-]{27}", "Discord Bot Token"), + (r"https://discord(?:app)?\.com/api/webhooks/\d+/[\w-]+", "Discord Webhook URL"), + # Twilio + (r"SK[a-f0-9]{32}", "Twilio API Key"), + (r"AC[a-f0-9]{32}", "Twilio Account SID"), + # SendGrid + (r"SG\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9_-]{43}", "SendGrid API Key"), + # Mailchimp + (r"[a-f0-9]{32}-us\d+", "Mailchimp API Key"), + # NPM + (r"npm_[a-zA-Z0-9]{36}", "NPM Access Token"), + # PyPI + (r"pypi-[a-zA-Z0-9]{60,}", "PyPI API Token"), + # Supabase/JWT + (r"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[A-Za-z0-9_-]{50,}", "Supabase/JWT Token"), + # Linear + (r"lin_api_[a-zA-Z0-9]{40,}", "Linear API Key"), + # Vercel + (r"[a-zA-Z0-9]{24}_[a-zA-Z0-9]{28,}", "Potential Vercel Token"), + # Heroku + ( + r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}", + "Heroku API Key / UUID", + ), + # Doppler + (r"dp\.pt\.[a-zA-Z0-9]{40,}", "Doppler Service Token"), +] + +# Private key patterns +PRIVATE_KEY_PATTERNS = [ + (r"-----BEGIN\s+(RSA\s+)?PRIVATE\s+KEY-----", "RSA Private Key"), + (r"-----BEGIN\s+OPENSSH\s+PRIVATE\s+KEY-----", "OpenSSH Private Key"), + (r"-----BEGIN\s+DSA\s+PRIVATE\s+KEY-----", "DSA Private Key"), + (r"-----BEGIN\s+EC\s+PRIVATE\s+KEY-----", "EC Private Key"), + (r"-----BEGIN\s+PGP\s+PRIVATE\s+KEY\s+BLOCK-----", "PGP Private Key"), + (r"-----BEGIN\s+CERTIFICATE-----", "Certificate (may contain private key)"), +] + +# Database connection strings with embedded credentials +DATABASE_PATTERNS = [ + ( + r'mongodb(?:\+srv)?://[^"\s:]+:[^@"\s]+@[^\s"]+', + "MongoDB Connection String with credentials", + ), + ( + r'postgres(?:ql)?://[^"\s:]+:[^@"\s]+@[^\s"]+', + "PostgreSQL Connection String with credentials", + ), + (r'mysql://[^"\s:]+:[^@"\s]+@[^\s"]+', "MySQL Connection String with credentials"), + (r'redis://[^"\s:]+:[^@"\s]+@[^\s"]+', "Redis Connection String with credentials"), + ( + r'amqp://[^"\s:]+:[^@"\s]+@[^\s"]+', + "RabbitMQ Connection String with credentials", + ), +] + +# Combine all patterns +ALL_PATTERNS = ( + GENERIC_PATTERNS + SERVICE_PATTERNS + PRIVATE_KEY_PATTERNS + DATABASE_PATTERNS +) + + +# ============================================================================= +# DATA CLASSES +# ============================================================================= + + +@dataclass +class SecretMatch: + """A potential secret found in a file.""" + + file_path: str + line_number: int + pattern_name: str + matched_text: str + line_content: str + + +# ============================================================================= +# IGNORE LIST +# ============================================================================= + +# Files/directories to always skip +DEFAULT_IGNORE_PATTERNS = [ + r"\.git/", + r"node_modules/", + r"\.venv/", + r"venv/", + r"__pycache__/", + r"\.pyc$", + r"dist/", + r"build/", + r"\.egg-info/", + r"\.example$", + r"\.sample$", + r"\.template$", + r"\.md$", # Documentation files + r"\.rst$", + r"\.txt$", + r"package-lock\.json$", + r"yarn\.lock$", + r"pnpm-lock\.yaml$", + r"Cargo\.lock$", + r"poetry\.lock$", +] + +# Binary file extensions to skip +BINARY_EXTENSIONS = { + ".png", + ".jpg", + ".jpeg", + ".gif", + ".ico", + ".webp", + ".svg", + ".woff", + ".woff2", + ".ttf", + ".eot", + ".otf", + ".pdf", + ".doc", + ".docx", + ".xls", + ".xlsx", + ".zip", + ".tar", + ".gz", + ".bz2", + ".7z", + ".rar", + ".exe", + ".dll", + ".so", + ".dylib", + ".mp3", + ".mp4", + ".wav", + ".avi", + ".mov", + ".pyc", + ".pyo", + ".class", + ".o", +} + +# False positive patterns to filter out +FALSE_POSITIVE_PATTERNS = [ + r"process\.env\.", # Environment variable references + r"os\.environ", # Python env references + r"ENV\[", # Ruby/other env references + r"\$\{[A-Z_]+\}", # Shell variable substitution + r"your[-_]?api[-_]?key", # Placeholder values + r"xxx+", # Placeholder + r"placeholder", # Placeholder + r"example", # Example value + r"sample", # Sample value + r"test[-_]?key", # Test placeholder + r"<[A-Z_]+>", # Placeholder like + r"TODO", # Comment markers + r"FIXME", + r"CHANGEME", + r"INSERT[-_]?YOUR", + r"REPLACE[-_]?WITH", +] + + +# ============================================================================= +# CORE FUNCTIONS +# ============================================================================= + + +def load_secretsignore(project_dir: Path) -> list[str]: + """Load custom ignore patterns from .secretsignore file.""" + ignore_file = project_dir / ".secretsignore" + if not ignore_file.exists(): + return [] + + patterns = [] + try: + content = ignore_file.read_text() + for line in content.splitlines(): + line = line.strip() + # Skip comments and empty lines + if line and not line.startswith("#"): + patterns.append(line) + except OSError: + pass + + return patterns + + +def should_skip_file(file_path: str, custom_ignores: list[str]) -> bool: + """Check if a file should be skipped based on ignore patterns.""" + path = Path(file_path) + + # Check binary extensions + if path.suffix.lower() in BINARY_EXTENSIONS: + return True + + # Check default ignore patterns + for pattern in DEFAULT_IGNORE_PATTERNS: + if re.search(pattern, file_path): + return True + + # Check custom ignore patterns + for pattern in custom_ignores: + if re.search(pattern, file_path): + return True + + return False + + +def is_false_positive(line: str, matched_text: str) -> bool: + """Check if a match is likely a false positive.""" + line_lower = line.lower() + + for pattern in FALSE_POSITIVE_PATTERNS: + if re.search(pattern, line_lower): + return True + + # Check if it's just a variable name or type hint + if re.match(r"^[a-z_]+:\s*str\s*$", line.strip(), re.IGNORECASE): + return True + + # Check if it's in a comment + stripped = line.strip() + if ( + stripped.startswith("#") + or stripped.startswith("//") + or stripped.startswith("*") + ): + # But still flag if there's an actual long key-like string + if not re.search(r"[a-zA-Z0-9_-]{40,}", matched_text): + return True + + return False + + +def mask_secret(text: str, visible_chars: int = 8) -> str: + """Mask a secret, showing only first few characters.""" + if len(text) <= visible_chars: + return text + return text[:visible_chars] + "***" + + +def scan_content(content: str, file_path: str) -> list[SecretMatch]: + """Scan file content for potential secrets.""" + matches = [] + lines = content.splitlines() + + for line_num, line in enumerate(lines, 1): + for pattern, pattern_name in ALL_PATTERNS: + try: + for match in re.finditer(pattern, line, re.IGNORECASE): + matched_text = match.group(0) + + # Skip false positives + if is_false_positive(line, matched_text): + continue + + matches.append( + SecretMatch( + file_path=file_path, + line_number=line_num, + pattern_name=pattern_name, + matched_text=matched_text, + line_content=line.strip()[:100], # Truncate long lines + ) + ) + except re.error: + # Invalid regex, skip + continue + + return matches + + +def get_staged_files() -> list[str]: + """Get list of staged files from git (excluding deleted files).""" + try: + result = subprocess.run( + ["git", "diff", "--cached", "--name-only", "--diff-filter=ACM"], + capture_output=True, + text=True, + check=True, + ) + files = [f.strip() for f in result.stdout.splitlines() if f.strip()] + return files + except subprocess.CalledProcessError: + return [] + + +def get_all_tracked_files() -> list[str]: + """Get all tracked files in the repository.""" + try: + result = subprocess.run( + ["git", "ls-files"], + capture_output=True, + text=True, + check=True, + ) + files = [f.strip() for f in result.stdout.splitlines() if f.strip()] + return files + except subprocess.CalledProcessError: + return [] + + +def scan_files( + files: list[str], + project_dir: Path | None = None, +) -> list[SecretMatch]: + """Scan a list of files for secrets.""" + if project_dir is None: + project_dir = Path.cwd() + + custom_ignores = load_secretsignore(project_dir) + all_matches = [] + + for file_path in files: + # Skip files based on ignore patterns + if should_skip_file(file_path, custom_ignores): + continue + + full_path = project_dir / file_path + + # Skip if file doesn't exist or is a directory + if not full_path.exists() or full_path.is_dir(): + continue + + try: + content = full_path.read_text(encoding="utf-8", errors="ignore") + matches = scan_content(content, file_path) + all_matches.extend(matches) + except (OSError, UnicodeDecodeError): + # Skip files that can't be read + continue + + return all_matches + + +# ============================================================================= +# OUTPUT FORMATTING +# ============================================================================= + +# ANSI color codes +RED = "\033[0;31m" +GREEN = "\033[0;32m" +YELLOW = "\033[1;33m" +CYAN = "\033[0;36m" +NC = "\033[0m" # No Color + + +def print_results(matches: list[SecretMatch]) -> None: + """Print scan results in a formatted way.""" + if not matches: + print(f"{GREEN}No secrets detected. Commit allowed.{NC}") + return + + print(f"{RED}POTENTIAL SECRETS DETECTED!{NC}") + print(f"{RED}{'=' * 60}{NC}") + + # Group by file + files_with_matches: dict[str, list[SecretMatch]] = {} + for match in matches: + if match.file_path not in files_with_matches: + files_with_matches[match.file_path] = [] + files_with_matches[match.file_path].append(match) + + for file_path, file_matches in files_with_matches.items(): + print(f"\n{YELLOW}File: {file_path}{NC}") + for match in file_matches: + masked = mask_secret(match.matched_text) + print(f" Line {match.line_number}: [{match.pattern_name}]") + print(f" {CYAN}{masked}{NC}") + + print(f"\n{RED}{'=' * 60}{NC}") + print(f"\n{YELLOW}If these are false positives, you can:{NC}") + print(" 1. Add patterns to .secretsignore (create if needed)") + print(" 2. Use environment variables instead of hardcoded values") + print() + print(f"{RED}Commit blocked to protect against leaking secrets.{NC}") + + +def print_json_results(matches: list[SecretMatch]) -> None: + """Print scan results as JSON (for programmatic use).""" + import json + + results = { + "secrets_found": len(matches) > 0, + "count": len(matches), + "matches": [ + { + "file": m.file_path, + "line": m.line_number, + "type": m.pattern_name, + "preview": mask_secret(m.matched_text), + } + for m in matches + ], + } + print(json.dumps(results, indent=2)) + + +# ============================================================================= +# MAIN +# ============================================================================= + + +def main() -> int: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Scan files for potential secrets before commit" + ) + parser.add_argument( + "--staged-only", + "-s", + action="store_true", + default=True, + help="Only scan staged files (default)", + ) + parser.add_argument( + "--all-files", "-a", action="store_true", help="Scan all tracked files" + ) + parser.add_argument( + "--path", "-p", type=str, help="Scan a specific file or directory" + ) + parser.add_argument("--json", action="store_true", help="Output results as JSON") + parser.add_argument( + "--quiet", "-q", action="store_true", help="Only output if secrets are found" + ) + + args = parser.parse_args() + + project_dir = Path.cwd() + + # Determine which files to scan + if args.path: + path = Path(args.path) + if path.is_file(): + files = [str(path)] + elif path.is_dir(): + files = [ + str(f.relative_to(project_dir)) for f in path.rglob("*") if f.is_file() + ] + else: + print(f"{RED}Error: Path not found: {args.path}{NC}", file=sys.stderr) + return 2 + elif args.all_files: + files = get_all_tracked_files() + else: + files = get_staged_files() + + if not files: + if not args.quiet: + print(f"{GREEN}No files to scan.{NC}") + return 0 + + if not args.quiet and not args.json: + print(f"Scanning {len(files)} file(s) for secrets...") + + # Scan files + matches = scan_files(files, project_dir) + + # Output results + if args.json: + print_json_results(matches) + elif matches or not args.quiet: + print_results(matches) + + # Return exit code + return 1 if matches else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/auto-claude/security_scanner.py b/auto-claude/security_scanner.py index 8cabb51d..503bd1b2 100644 --- a/auto-claude/security_scanner.py +++ b/auto-claude/security_scanner.py @@ -1,597 +1,2 @@ -#!/usr/bin/env python3 -""" -Security Scanner Module -======================= - -Consolidates security scanning including secrets detection and SAST tools. -This module integrates the existing scan_secrets.py and provides a unified -interface for all security scanning. - -The security scanner is used by: -- QA Agent: To verify no secrets are committed -- Validation Strategy: To run security scans for high-risk changes - -Usage: - from security_scanner import SecurityScanner - - scanner = SecurityScanner() - results = scanner.scan(project_dir, spec_dir) - - if results.has_critical_issues: - print("Security issues found - blocking QA approval") -""" - -import json -import subprocess -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -# Import the existing secrets scanner -try: - from scan_secrets import SecretMatch, get_all_tracked_files, scan_files - - HAS_SECRETS_SCANNER = True -except ImportError: - HAS_SECRETS_SCANNER = False - SecretMatch = None - - -# ============================================================================= -# DATA CLASSES -# ============================================================================= - - -@dataclass -class SecurityVulnerability: - """ - Represents a security vulnerability found during scanning. - - Attributes: - severity: Severity level (critical, high, medium, low, info) - source: Which scanner found this (secrets, bandit, npm_audit, etc.) - title: Short title of the vulnerability - description: Detailed description - file: File where vulnerability was found (if applicable) - line: Line number (if applicable) - cwe: CWE identifier if available - """ - - severity: str # critical, high, medium, low, info - source: str # secrets, bandit, npm_audit, semgrep, etc. - title: str - description: str - file: str | None = None - line: int | None = None - cwe: str | None = None - - -@dataclass -class SecurityScanResult: - """ - Result of a security scan. - - Attributes: - secrets: List of detected secrets - vulnerabilities: List of security vulnerabilities - scan_errors: List of errors during scanning - has_critical_issues: Whether any critical issues were found - should_block_qa: Whether these results should block QA approval - """ - - secrets: list[dict[str, Any]] = field(default_factory=list) - vulnerabilities: list[SecurityVulnerability] = field(default_factory=list) - scan_errors: list[str] = field(default_factory=list) - has_critical_issues: bool = False - should_block_qa: bool = False - - -# ============================================================================= -# SECURITY SCANNER -# ============================================================================= - - -class SecurityScanner: - """ - Consolidates all security scanning operations. - - Integrates: - - scan_secrets.py for secrets detection - - Bandit for Python SAST (if available) - - npm audit for JavaScript vulnerabilities (if applicable) - """ - - def __init__(self) -> None: - """Initialize the security scanner.""" - self._bandit_available: bool | None = None - self._npm_available: bool | None = None - - def scan( - self, - project_dir: Path, - spec_dir: Path | None = None, - changed_files: list[str] | None = None, - run_secrets: bool = True, - run_sast: bool = True, - run_dependency_audit: bool = True, - ) -> SecurityScanResult: - """ - Run all applicable security scans. - - Args: - project_dir: Path to the project root - spec_dir: Path to the spec directory (for storing results) - changed_files: Optional list of files to scan (if None, scans all) - run_secrets: Whether to run secrets scanning - run_sast: Whether to run SAST tools - run_dependency_audit: Whether to run dependency audits - - Returns: - SecurityScanResult with all findings - """ - project_dir = Path(project_dir) - result = SecurityScanResult() - - # Run secrets scan - if run_secrets: - self._run_secrets_scan(project_dir, changed_files, result) - - # Run SAST based on project type - if run_sast: - self._run_sast_scans(project_dir, result) - - # Run dependency audits - if run_dependency_audit: - self._run_dependency_audits(project_dir, result) - - # Determine if should block QA - result.has_critical_issues = ( - any(v.severity in ["critical", "high"] for v in result.vulnerabilities) - or len(result.secrets) > 0 - ) - - # Any secrets always block, critical vulnerabilities block - result.should_block_qa = len(result.secrets) > 0 or any( - v.severity == "critical" for v in result.vulnerabilities - ) - - # Save results if spec_dir provided - if spec_dir: - self._save_results(spec_dir, result) - - return result - - def _run_secrets_scan( - self, - project_dir: Path, - changed_files: list[str] | None, - result: SecurityScanResult, - ) -> None: - """Run secrets scanning using scan_secrets.py.""" - if not HAS_SECRETS_SCANNER: - result.scan_errors.append("scan_secrets module not available") - return - - try: - # Get files to scan - if changed_files: - files_to_scan = changed_files - else: - files_to_scan = get_all_tracked_files() - - # Run scan - matches = scan_files(files_to_scan, project_dir) - - # Convert matches to result format - for match in matches: - result.secrets.append( - { - "file": match.file_path, - "line": match.line_number, - "pattern": match.pattern_name, - "matched_text": self._redact_secret(match.matched_text), - } - ) - - # Also add as vulnerability - result.vulnerabilities.append( - SecurityVulnerability( - severity="critical", - source="secrets", - title=f"Potential secret: {match.pattern_name}", - description=f"Found potential {match.pattern_name} in file", - file=match.file_path, - line=match.line_number, - ) - ) - - except Exception as e: - result.scan_errors.append(f"Secrets scan error: {str(e)}") - - def _run_sast_scans(self, project_dir: Path, result: SecurityScanResult) -> None: - """Run SAST tools based on project type.""" - # Python SAST with Bandit - if self._is_python_project(project_dir): - self._run_bandit(project_dir, result) - - # JavaScript/Node.js - npm audit - # (handled in dependency audits for Node projects) - - def _run_bandit(self, project_dir: Path, result: SecurityScanResult) -> None: - """Run Bandit security scanner for Python projects.""" - if not self._check_bandit_available(): - return - - try: - # Find Python source directories - src_dirs = [] - for candidate in ["src", "app", project_dir.name, "."]: - candidate_path = project_dir / candidate - if ( - candidate_path.exists() - and (candidate_path / "__init__.py").exists() - ): - src_dirs.append(str(candidate_path)) - - if not src_dirs: - # Try to find any Python files - py_files = list(project_dir.glob("**/*.py")) - if not py_files: - return - src_dirs = ["."] - - # Run bandit - cmd = [ - "bandit", - "-r", - *src_dirs, - "-f", - "json", - "--exit-zero", # Don't fail on findings - ] - - proc = subprocess.run( - cmd, - cwd=project_dir, - capture_output=True, - text=True, - timeout=120, - ) - - if proc.stdout: - try: - bandit_output = json.loads(proc.stdout) - for finding in bandit_output.get("results", []): - severity = finding.get("issue_severity", "MEDIUM").lower() - if severity == "high": - severity = "high" - elif severity == "medium": - severity = "medium" - else: - severity = "low" - - result.vulnerabilities.append( - SecurityVulnerability( - severity=severity, - source="bandit", - title=finding.get("issue_text", "Unknown issue"), - description=finding.get("issue_text", ""), - file=finding.get("filename"), - line=finding.get("line_number"), - cwe=finding.get("issue_cwe", {}).get("id"), - ) - ) - except json.JSONDecodeError: - result.scan_errors.append("Failed to parse Bandit output") - - except subprocess.TimeoutExpired: - result.scan_errors.append("Bandit scan timed out") - except FileNotFoundError: - result.scan_errors.append("Bandit not found") - except Exception as e: - result.scan_errors.append(f"Bandit error: {str(e)}") - - def _run_dependency_audits( - self, project_dir: Path, result: SecurityScanResult - ) -> None: - """Run dependency vulnerability audits.""" - # npm audit for JavaScript projects - if (project_dir / "package.json").exists(): - self._run_npm_audit(project_dir, result) - - # pip-audit for Python projects (if available) - if self._is_python_project(project_dir): - self._run_pip_audit(project_dir, result) - - def _run_npm_audit(self, project_dir: Path, result: SecurityScanResult) -> None: - """Run npm audit for JavaScript projects.""" - try: - cmd = ["npm", "audit", "--json"] - - proc = subprocess.run( - cmd, - cwd=project_dir, - capture_output=True, - text=True, - timeout=120, - ) - - if proc.stdout: - try: - audit_output = json.loads(proc.stdout) - - # npm audit v2+ format - vulnerabilities = audit_output.get("vulnerabilities", {}) - for pkg_name, vuln_info in vulnerabilities.items(): - severity = vuln_info.get("severity", "moderate") - if severity == "critical": - severity = "critical" - elif severity == "high": - severity = "high" - elif severity == "moderate": - severity = "medium" - else: - severity = "low" - - result.vulnerabilities.append( - SecurityVulnerability( - severity=severity, - source="npm_audit", - title=f"Vulnerable dependency: {pkg_name}", - description=vuln_info.get("via", [{}])[0].get( - "title", "" - ) - if isinstance(vuln_info.get("via"), list) - and vuln_info.get("via") - else str(vuln_info.get("via", "")), - file="package.json", - ) - ) - except json.JSONDecodeError: - pass # npm audit may return invalid JSON on no findings - - except subprocess.TimeoutExpired: - result.scan_errors.append("npm audit timed out") - except FileNotFoundError: - pass # npm not available - except Exception as e: - result.scan_errors.append(f"npm audit error: {str(e)}") - - def _run_pip_audit(self, project_dir: Path, result: SecurityScanResult) -> None: - """Run pip-audit for Python projects (if available).""" - try: - cmd = ["pip-audit", "--format", "json"] - - proc = subprocess.run( - cmd, - cwd=project_dir, - capture_output=True, - text=True, - timeout=120, - ) - - if proc.stdout: - try: - audit_output = json.loads(proc.stdout) - for vuln in audit_output: - severity = "high" if vuln.get("fix_versions") else "medium" - - result.vulnerabilities.append( - SecurityVulnerability( - severity=severity, - source="pip_audit", - title=f"Vulnerable package: {vuln.get('name')}", - description=vuln.get("description", ""), - cwe=vuln.get("aliases", [""])[0] - if vuln.get("aliases") - else None, - ) - ) - except json.JSONDecodeError: - pass - - except FileNotFoundError: - pass # pip-audit not available - except subprocess.TimeoutExpired: - pass - except Exception: - pass - - def _is_python_project(self, project_dir: Path) -> bool: - """Check if this is a Python project.""" - indicators = [ - project_dir / "pyproject.toml", - project_dir / "requirements.txt", - project_dir / "setup.py", - project_dir / "setup.cfg", - ] - return any(p.exists() for p in indicators) - - def _check_bandit_available(self) -> bool: - """Check if Bandit is available.""" - if self._bandit_available is None: - try: - subprocess.run( - ["bandit", "--version"], - capture_output=True, - timeout=5, - ) - self._bandit_available = True - except (FileNotFoundError, subprocess.TimeoutExpired): - self._bandit_available = False - return self._bandit_available - - def _redact_secret(self, text: str) -> str: - """Redact a secret for safe logging.""" - if len(text) <= 8: - return "*" * len(text) - return text[:4] + "*" * (len(text) - 8) + text[-4:] - - def _save_results(self, spec_dir: Path, result: SecurityScanResult) -> None: - """Save scan results to spec directory.""" - spec_dir = Path(spec_dir) - spec_dir.mkdir(parents=True, exist_ok=True) - - output_file = spec_dir / "security_scan_results.json" - output_data = self.to_dict(result) - - with open(output_file, "w", encoding="utf-8") as f: - json.dump(output_data, f, indent=2) - - def to_dict(self, result: SecurityScanResult) -> dict[str, Any]: - """Convert result to dictionary for JSON serialization.""" - return { - "secrets": result.secrets, - "vulnerabilities": [ - { - "severity": v.severity, - "source": v.source, - "title": v.title, - "description": v.description, - "file": v.file, - "line": v.line, - "cwe": v.cwe, - } - for v in result.vulnerabilities - ], - "scan_errors": result.scan_errors, - "has_critical_issues": result.has_critical_issues, - "should_block_qa": result.should_block_qa, - "summary": { - "total_secrets": len(result.secrets), - "total_vulnerabilities": len(result.vulnerabilities), - "critical_count": sum( - 1 for v in result.vulnerabilities if v.severity == "critical" - ), - "high_count": sum( - 1 for v in result.vulnerabilities if v.severity == "high" - ), - "medium_count": sum( - 1 for v in result.vulnerabilities if v.severity == "medium" - ), - "low_count": sum( - 1 for v in result.vulnerabilities if v.severity == "low" - ), - }, - } - - -# ============================================================================= -# CONVENIENCE FUNCTIONS -# ============================================================================= - - -def scan_for_security_issues( - project_dir: Path, - spec_dir: Path | None = None, - changed_files: list[str] | None = None, -) -> SecurityScanResult: - """ - Convenience function to run security scan. - - Args: - project_dir: Path to project root - spec_dir: Optional spec directory to save results - changed_files: Optional list of files to scan - - Returns: - SecurityScanResult with all findings - """ - scanner = SecurityScanner() - return scanner.scan(project_dir, spec_dir, changed_files) - - -def has_security_issues(project_dir: Path) -> bool: - """ - Quick check if project has security issues. - - Args: - project_dir: Path to project root - - Returns: - True if any critical/high issues found - """ - scanner = SecurityScanner() - result = scanner.scan(project_dir, run_sast=False, run_dependency_audit=False) - return result.has_critical_issues - - -def scan_secrets_only( - project_dir: Path, - changed_files: list[str] | None = None, -) -> list[dict[str, Any]]: - """ - Scan only for secrets (quick scan). - - Args: - project_dir: Path to project root - changed_files: Optional list of files to scan - - Returns: - List of detected secrets - """ - scanner = SecurityScanner() - result = scanner.scan( - project_dir, - changed_files=changed_files, - run_sast=False, - run_dependency_audit=False, - ) - return result.secrets - - -# ============================================================================= -# CLI -# ============================================================================= - - -def main() -> None: - """CLI entry point for testing.""" - import argparse - - parser = argparse.ArgumentParser(description="Run security scans") - parser.add_argument("project_dir", type=Path, help="Path to project root") - parser.add_argument("--spec-dir", type=Path, help="Path to spec directory") - parser.add_argument( - "--secrets-only", action="store_true", help="Only scan for secrets" - ) - parser.add_argument("--json", action="store_true", help="Output as JSON") - - args = parser.parse_args() - - scanner = SecurityScanner() - result = scanner.scan( - args.project_dir, - spec_dir=args.spec_dir, - run_sast=not args.secrets_only, - run_dependency_audit=not args.secrets_only, - ) - - if args.json: - print(json.dumps(scanner.to_dict(result), indent=2)) - else: - print(f"Secrets Found: {len(result.secrets)}") - print(f"Vulnerabilities: {len(result.vulnerabilities)}") - print(f"Has Critical Issues: {result.has_critical_issues}") - print(f"Should Block QA: {result.should_block_qa}") - - if result.secrets: - print("\nSecrets Detected:") - for secret in result.secrets: - print(f" - {secret['pattern']} in {secret['file']}:{secret['line']}") - - if result.vulnerabilities: - print(f"\nVulnerabilities ({len(result.vulnerabilities)}):") - for v in result.vulnerabilities: - print(f" [{v.severity.upper()}] {v.title}") - if v.file: - print(f" File: {v.file}:{v.line or ''}") - - if result.scan_errors: - print(f"\nScan Errors ({len(result.scan_errors)}):") - for error in result.scan_errors: - print(f" - {error}") - - -if __name__ == "__main__": - main() +"""Backward compatibility shim - import from analysis.security_scanner instead.""" +from analysis.security_scanner import * diff --git a/auto-claude/service_orchestrator.py b/auto-claude/service_orchestrator.py index 671d050e..3eb0262e 100644 --- a/auto-claude/service_orchestrator.py +++ b/auto-claude/service_orchestrator.py @@ -1,610 +1,18 @@ -#!/usr/bin/env python3 -""" -Service Orchestrator Module -=========================== - -Orchestrates multi-service environments for testing. -Handles docker-compose, monorepo service discovery, and health checks. - -The service orchestrator is used by: -- QA Agent: To start services before integration/e2e tests -- Validation Strategy: To determine if multi-service orchestration is needed - -Usage: - from service_orchestrator import ServiceOrchestrator - - orchestrator = ServiceOrchestrator(project_dir) - if orchestrator.is_multi_service(): - orchestrator.start_services() - # run tests - orchestrator.stop_services() -""" - -import json -import subprocess -import time -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -# ============================================================================= -# DATA CLASSES -# ============================================================================= - - -@dataclass -class ServiceConfig: - """ - Configuration for a single service. - - Attributes: - name: Name of the service - path: Path to the service (relative to project root) - port: Port the service runs on - type: Type of service (docker, local, mock) - health_check_url: URL for health check - startup_command: Command to start the service - startup_timeout: Timeout in seconds for startup - """ - - name: str - path: str | None = None - port: int | None = None - type: str = "docker" # docker, local, mock - health_check_url: str | None = None - startup_command: str | None = None - startup_timeout: int = 120 - - -@dataclass -class OrchestrationResult: - """ - Result of service orchestration. - - Attributes: - success: Whether all services started successfully - services_started: List of services that were started - services_failed: List of services that failed to start - errors: List of error messages - """ - - success: bool = False - services_started: list[str] = field(default_factory=list) - services_failed: list[str] = field(default_factory=list) - errors: list[str] = field(default_factory=list) - - -# ============================================================================= -# SERVICE ORCHESTRATOR -# ============================================================================= - - -class ServiceOrchestrator: - """ - Orchestrates multi-service environments. - - Supports: - - Docker Compose for containerized services - - Monorepo service discovery - - Health check waiting - """ - - def __init__(self, project_dir: Path) -> None: - """ - Initialize the service orchestrator. - - Args: - project_dir: Path to the project root - """ - self.project_dir = Path(project_dir) - self._compose_file: Path | None = None - self._services: list[ServiceConfig] = [] - self._processes: dict[str, subprocess.Popen] = {} - self._discover_services() - - def _discover_services(self) -> None: - """Discover services in the project.""" - # Check for docker-compose - self._compose_file = self._find_compose_file() - - if self._compose_file: - self._parse_compose_services() - else: - # Check for monorepo structure - self._discover_monorepo_services() - - def _find_compose_file(self) -> Path | None: - """Find docker-compose configuration file.""" - candidates = [ - "docker-compose.yml", - "docker-compose.yaml", - "compose.yml", - "compose.yaml", - "docker-compose.dev.yml", - "docker-compose.dev.yaml", - ] - - for candidate in candidates: - path = self.project_dir / candidate - if path.exists(): - return path - - return None - - def _parse_compose_services(self) -> None: - """Parse services from docker-compose file.""" - if not self._compose_file: - return - - try: - # Try to import yaml - import yaml - - HAS_YAML = True - except ImportError: - HAS_YAML = False - - if not HAS_YAML: - # Basic parsing without yaml module - content = self._compose_file.read_text() - if "services:" in content: - # Very basic service name extraction - lines = content.split("\n") - in_services = False - for line in lines: - if line.strip() == "services:": - in_services = True - continue - if ( - in_services - and line.startswith(" ") - and not line.startswith(" ") - ): - service_name = line.strip().rstrip(":") - if service_name: - self._services.append(ServiceConfig(name=service_name)) - return - - try: - with open(self._compose_file, encoding="utf-8") as f: - compose_data = yaml.safe_load(f) - - services = compose_data.get("services", {}) - for name, config in services.items(): - if not isinstance(config, dict): - continue - - # Extract port mapping - ports = config.get("ports", []) - port = None - if ports: - port_mapping = str(ports[0]) - if ":" in port_mapping: - port = int(port_mapping.split(":")[0]) - - # Determine health check URL - health_url = None - if port: - health_url = f"http://localhost:{port}/health" - - self._services.append( - ServiceConfig( - name=name, - port=port, - type="docker", - health_check_url=health_url, - ) - ) - except Exception: - pass - - def _discover_monorepo_services(self) -> None: - """Discover services in a monorepo structure.""" - # Common monorepo patterns - service_dirs = [ - "services", - "packages", - "apps", - "microservices", - ] - - for service_dir in service_dirs: - dir_path = self.project_dir / service_dir - if dir_path.exists() and dir_path.is_dir(): - for item in dir_path.iterdir(): - if item.is_dir() and self._is_service_directory(item): - self._services.append( - ServiceConfig( - name=item.name, - path=str(item.relative_to(self.project_dir)), - type="local", - ) - ) - - def _is_service_directory(self, path: Path) -> bool: - """Check if a directory contains a service.""" - # Look for indicators of a service - indicators = [ - "package.json", - "pyproject.toml", - "requirements.txt", - "Dockerfile", - "main.py", - "app.py", - "index.ts", - "index.js", - "main.go", - "Cargo.toml", - ] - - return any((path / indicator).exists() for indicator in indicators) - - def is_multi_service(self) -> bool: - """ - Check if this is a multi-service project. - - Returns: - True if multiple services are detected - """ - return len(self._services) > 1 or self._compose_file is not None - - def has_docker_compose(self) -> bool: - """ - Check if project has docker-compose configuration. - - Returns: - True if docker-compose file exists - """ - return self._compose_file is not None - - def get_services(self) -> list[ServiceConfig]: - """ - Get list of discovered services. - - Returns: - List of ServiceConfig objects - """ - return self._services.copy() - - def start_services(self, timeout: int = 120) -> OrchestrationResult: - """ - Start all services. - - Args: - timeout: Timeout in seconds for all services to start - - Returns: - OrchestrationResult with status - """ - result = OrchestrationResult() - - if self._compose_file: - return self._start_docker_compose(timeout) - else: - return self._start_local_services(timeout) - - def _start_docker_compose(self, timeout: int) -> OrchestrationResult: - """Start services using docker-compose.""" - result = OrchestrationResult() - - try: - # Check if docker-compose is available - docker_cmd = self._get_docker_compose_cmd() - if not docker_cmd: - result.errors.append("docker-compose not found") - return result - - # Start services - cmd = docker_cmd + ["up", "-d"] - - proc = subprocess.run( - cmd, - cwd=self.project_dir, - capture_output=True, - text=True, - timeout=timeout, - ) - - if proc.returncode != 0: - result.errors.append(f"docker-compose up failed: {proc.stderr}") - return result - - # Wait for health checks - if self._wait_for_health(timeout): - result.success = True - result.services_started = [s.name for s in self._services] - else: - result.errors.append("Services did not become healthy in time") - result.services_failed = [s.name for s in self._services] - - except subprocess.TimeoutExpired: - result.errors.append("docker-compose startup timed out") - except Exception as e: - result.errors.append(f"Error starting services: {str(e)}") - - return result - - def _start_local_services(self, timeout: int) -> OrchestrationResult: - """Start local services (non-docker).""" - result = OrchestrationResult() - - for service in self._services: - if service.startup_command: - try: - proc = subprocess.Popen( - service.startup_command, - shell=True, - cwd=self.project_dir / service.path - if service.path - else self.project_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - self._processes[service.name] = proc - result.services_started.append(service.name) - except Exception as e: - result.errors.append(f"Failed to start {service.name}: {str(e)}") - result.services_failed.append(service.name) - - # Wait for services to be ready - if result.services_started: - if self._wait_for_health(timeout): - result.success = True - else: - result.errors.append("Services did not become healthy in time") - - return result - - def stop_services(self) -> None: - """Stop all running services.""" - if self._compose_file: - self._stop_docker_compose() - else: - self._stop_local_services() - - def _stop_docker_compose(self) -> None: - """Stop services using docker-compose.""" - try: - docker_cmd = self._get_docker_compose_cmd() - if docker_cmd: - subprocess.run( - docker_cmd + ["down"], - cwd=self.project_dir, - capture_output=True, - timeout=60, - ) - except Exception: - pass - - def _stop_local_services(self) -> None: - """Stop local services.""" - for name, proc in self._processes.items(): - try: - proc.terminate() - proc.wait(timeout=10) - except Exception: - try: - proc.kill() - except Exception: - pass - self._processes.clear() - - def _get_docker_compose_cmd(self) -> list[str] | None: - """Get the docker-compose command (v1 or v2).""" - # Try docker compose v2 first - try: - proc = subprocess.run( - ["docker", "compose", "version"], - capture_output=True, - timeout=5, - ) - if proc.returncode == 0: - return ["docker", "compose", "-f", str(self._compose_file)] - except Exception: - pass - - # Try docker-compose v1 - try: - proc = subprocess.run( - ["docker-compose", "version"], - capture_output=True, - timeout=5, - ) - if proc.returncode == 0: - return ["docker-compose", "-f", str(self._compose_file)] - except Exception: - pass - - return None - - def _wait_for_health(self, timeout: int) -> bool: - """ - Wait for all services to become healthy. - - Args: - timeout: Maximum time to wait in seconds - - Returns: - True if all services became healthy - """ - start_time = time.time() - - while time.time() - start_time < timeout: - all_healthy = True - - for service in self._services: - if service.port: - if not self._check_port(service.port): - all_healthy = False - break - - if all_healthy: - return True - - time.sleep(2) - - return False - - def _check_port(self, port: int) -> bool: - """Check if a port is responding.""" - import socket - - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(1) - result = s.connect_ex(("localhost", port)) - return result == 0 - except Exception: - return False - - def to_dict(self) -> dict[str, Any]: - """Convert orchestration config to dictionary.""" - return { - "is_multi_service": self.is_multi_service(), - "has_docker_compose": self.has_docker_compose(), - "compose_file": str(self._compose_file) if self._compose_file else None, - "services": [ - { - "name": s.name, - "path": s.path, - "port": s.port, - "type": s.type, - "health_check_url": s.health_check_url, - } - for s in self._services - ], - } - - -# ============================================================================= -# CONVENIENCE FUNCTIONS -# ============================================================================= - - -def is_multi_service_project(project_dir: Path) -> bool: - """ - Check if project is multi-service. - - Args: - project_dir: Path to project root - - Returns: - True if multi-service project - """ - orchestrator = ServiceOrchestrator(project_dir) - return orchestrator.is_multi_service() - - -def get_service_config(project_dir: Path) -> dict[str, Any]: - """ - Get service configuration for project. - - Args: - project_dir: Path to project root - - Returns: - Dictionary with service configuration - """ - orchestrator = ServiceOrchestrator(project_dir) - return orchestrator.to_dict() - - -# ============================================================================= -# CONTEXT MANAGER -# ============================================================================= - - -class ServiceContext: - """ - Context manager for service orchestration. - - Usage: - with ServiceContext(project_dir) as services: - # Services are running - run_tests() - # Services are stopped - """ - - def __init__(self, project_dir: Path, timeout: int = 120) -> None: - """Initialize service context.""" - self.orchestrator = ServiceOrchestrator(project_dir) - self.timeout = timeout - self.result: OrchestrationResult | None = None - - def __enter__(self) -> "ServiceContext": - """Start services on context entry.""" - if self.orchestrator.is_multi_service(): - self.result = self.orchestrator.start_services(self.timeout) - return self - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - """Stop services on context exit.""" - self.orchestrator.stop_services() - - @property - def success(self) -> bool: - """Check if services started successfully.""" - if self.result: - return self.result.success - return True # No services to start - - -# ============================================================================= -# CLI -# ============================================================================= - - -def main() -> None: - """CLI entry point for testing.""" - import argparse - - parser = argparse.ArgumentParser(description="Service orchestration") - parser.add_argument("project_dir", type=Path, help="Path to project root") - parser.add_argument("--start", action="store_true", help="Start services") - parser.add_argument("--stop", action="store_true", help="Stop services") - parser.add_argument("--status", action="store_true", help="Show service status") - parser.add_argument("--json", action="store_true", help="Output as JSON") - - args = parser.parse_args() - - orchestrator = ServiceOrchestrator(args.project_dir) - - if args.start: - result = orchestrator.start_services() - if args.json: - print( - json.dumps( - { - "success": result.success, - "services_started": result.services_started, - "errors": result.errors, - }, - indent=2, - ) - ) - else: - print(f"Started: {result.services_started}") - if result.errors: - print(f"Errors: {result.errors}") - elif args.stop: - orchestrator.stop_services() - print("Services stopped") - else: - # Default: show status - config = orchestrator.to_dict() - - if args.json: - print(json.dumps(config, indent=2)) - else: - print(f"Multi-service: {config['is_multi_service']}") - print(f"Docker Compose: {config['has_docker_compose']}") - if config["compose_file"]: - print(f"Compose File: {config['compose_file']}") - print(f"\nServices ({len(config['services'])}):") - for service in config["services"]: - port_info = f":{service['port']}" if service["port"] else "" - print(f" - {service['name']} ({service['type']}){port_info}") - - -if __name__ == "__main__": - main() +"""Backward compatibility shim - import from services.orchestrator instead.""" +from services.orchestrator import ( + ServiceConfig, + OrchestrationResult, + ServiceOrchestrator, + ServiceContext, + is_multi_service_project, + get_service_config, +) + +__all__ = [ + "ServiceConfig", + "OrchestrationResult", + "ServiceOrchestrator", + "ServiceContext", + "is_multi_service_project", + "get_service_config", +] diff --git a/auto-claude/services/__init__.py b/auto-claude/services/__init__.py new file mode 100644 index 00000000..7b6fa8d2 --- /dev/null +++ b/auto-claude/services/__init__.py @@ -0,0 +1,16 @@ +""" +Services Module +=============== + +Background services and orchestration for Auto Claude. +""" + +from .context import ServiceContext +from .orchestrator import ServiceOrchestrator +from .recovery import RecoveryManager + +__all__ = [ + "ServiceContext", + "ServiceOrchestrator", + "RecoveryManager", +] diff --git a/auto-claude/service_context.py b/auto-claude/services/context.py similarity index 100% rename from auto-claude/service_context.py rename to auto-claude/services/context.py diff --git a/auto-claude/services/orchestrator.py b/auto-claude/services/orchestrator.py new file mode 100644 index 00000000..671d050e --- /dev/null +++ b/auto-claude/services/orchestrator.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python3 +""" +Service Orchestrator Module +=========================== + +Orchestrates multi-service environments for testing. +Handles docker-compose, monorepo service discovery, and health checks. + +The service orchestrator is used by: +- QA Agent: To start services before integration/e2e tests +- Validation Strategy: To determine if multi-service orchestration is needed + +Usage: + from service_orchestrator import ServiceOrchestrator + + orchestrator = ServiceOrchestrator(project_dir) + if orchestrator.is_multi_service(): + orchestrator.start_services() + # run tests + orchestrator.stop_services() +""" + +import json +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# ============================================================================= +# DATA CLASSES +# ============================================================================= + + +@dataclass +class ServiceConfig: + """ + Configuration for a single service. + + Attributes: + name: Name of the service + path: Path to the service (relative to project root) + port: Port the service runs on + type: Type of service (docker, local, mock) + health_check_url: URL for health check + startup_command: Command to start the service + startup_timeout: Timeout in seconds for startup + """ + + name: str + path: str | None = None + port: int | None = None + type: str = "docker" # docker, local, mock + health_check_url: str | None = None + startup_command: str | None = None + startup_timeout: int = 120 + + +@dataclass +class OrchestrationResult: + """ + Result of service orchestration. + + Attributes: + success: Whether all services started successfully + services_started: List of services that were started + services_failed: List of services that failed to start + errors: List of error messages + """ + + success: bool = False + services_started: list[str] = field(default_factory=list) + services_failed: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) + + +# ============================================================================= +# SERVICE ORCHESTRATOR +# ============================================================================= + + +class ServiceOrchestrator: + """ + Orchestrates multi-service environments. + + Supports: + - Docker Compose for containerized services + - Monorepo service discovery + - Health check waiting + """ + + def __init__(self, project_dir: Path) -> None: + """ + Initialize the service orchestrator. + + Args: + project_dir: Path to the project root + """ + self.project_dir = Path(project_dir) + self._compose_file: Path | None = None + self._services: list[ServiceConfig] = [] + self._processes: dict[str, subprocess.Popen] = {} + self._discover_services() + + def _discover_services(self) -> None: + """Discover services in the project.""" + # Check for docker-compose + self._compose_file = self._find_compose_file() + + if self._compose_file: + self._parse_compose_services() + else: + # Check for monorepo structure + self._discover_monorepo_services() + + def _find_compose_file(self) -> Path | None: + """Find docker-compose configuration file.""" + candidates = [ + "docker-compose.yml", + "docker-compose.yaml", + "compose.yml", + "compose.yaml", + "docker-compose.dev.yml", + "docker-compose.dev.yaml", + ] + + for candidate in candidates: + path = self.project_dir / candidate + if path.exists(): + return path + + return None + + def _parse_compose_services(self) -> None: + """Parse services from docker-compose file.""" + if not self._compose_file: + return + + try: + # Try to import yaml + import yaml + + HAS_YAML = True + except ImportError: + HAS_YAML = False + + if not HAS_YAML: + # Basic parsing without yaml module + content = self._compose_file.read_text() + if "services:" in content: + # Very basic service name extraction + lines = content.split("\n") + in_services = False + for line in lines: + if line.strip() == "services:": + in_services = True + continue + if ( + in_services + and line.startswith(" ") + and not line.startswith(" ") + ): + service_name = line.strip().rstrip(":") + if service_name: + self._services.append(ServiceConfig(name=service_name)) + return + + try: + with open(self._compose_file, encoding="utf-8") as f: + compose_data = yaml.safe_load(f) + + services = compose_data.get("services", {}) + for name, config in services.items(): + if not isinstance(config, dict): + continue + + # Extract port mapping + ports = config.get("ports", []) + port = None + if ports: + port_mapping = str(ports[0]) + if ":" in port_mapping: + port = int(port_mapping.split(":")[0]) + + # Determine health check URL + health_url = None + if port: + health_url = f"http://localhost:{port}/health" + + self._services.append( + ServiceConfig( + name=name, + port=port, + type="docker", + health_check_url=health_url, + ) + ) + except Exception: + pass + + def _discover_monorepo_services(self) -> None: + """Discover services in a monorepo structure.""" + # Common monorepo patterns + service_dirs = [ + "services", + "packages", + "apps", + "microservices", + ] + + for service_dir in service_dirs: + dir_path = self.project_dir / service_dir + if dir_path.exists() and dir_path.is_dir(): + for item in dir_path.iterdir(): + if item.is_dir() and self._is_service_directory(item): + self._services.append( + ServiceConfig( + name=item.name, + path=str(item.relative_to(self.project_dir)), + type="local", + ) + ) + + def _is_service_directory(self, path: Path) -> bool: + """Check if a directory contains a service.""" + # Look for indicators of a service + indicators = [ + "package.json", + "pyproject.toml", + "requirements.txt", + "Dockerfile", + "main.py", + "app.py", + "index.ts", + "index.js", + "main.go", + "Cargo.toml", + ] + + return any((path / indicator).exists() for indicator in indicators) + + def is_multi_service(self) -> bool: + """ + Check if this is a multi-service project. + + Returns: + True if multiple services are detected + """ + return len(self._services) > 1 or self._compose_file is not None + + def has_docker_compose(self) -> bool: + """ + Check if project has docker-compose configuration. + + Returns: + True if docker-compose file exists + """ + return self._compose_file is not None + + def get_services(self) -> list[ServiceConfig]: + """ + Get list of discovered services. + + Returns: + List of ServiceConfig objects + """ + return self._services.copy() + + def start_services(self, timeout: int = 120) -> OrchestrationResult: + """ + Start all services. + + Args: + timeout: Timeout in seconds for all services to start + + Returns: + OrchestrationResult with status + """ + result = OrchestrationResult() + + if self._compose_file: + return self._start_docker_compose(timeout) + else: + return self._start_local_services(timeout) + + def _start_docker_compose(self, timeout: int) -> OrchestrationResult: + """Start services using docker-compose.""" + result = OrchestrationResult() + + try: + # Check if docker-compose is available + docker_cmd = self._get_docker_compose_cmd() + if not docker_cmd: + result.errors.append("docker-compose not found") + return result + + # Start services + cmd = docker_cmd + ["up", "-d"] + + proc = subprocess.run( + cmd, + cwd=self.project_dir, + capture_output=True, + text=True, + timeout=timeout, + ) + + if proc.returncode != 0: + result.errors.append(f"docker-compose up failed: {proc.stderr}") + return result + + # Wait for health checks + if self._wait_for_health(timeout): + result.success = True + result.services_started = [s.name for s in self._services] + else: + result.errors.append("Services did not become healthy in time") + result.services_failed = [s.name for s in self._services] + + except subprocess.TimeoutExpired: + result.errors.append("docker-compose startup timed out") + except Exception as e: + result.errors.append(f"Error starting services: {str(e)}") + + return result + + def _start_local_services(self, timeout: int) -> OrchestrationResult: + """Start local services (non-docker).""" + result = OrchestrationResult() + + for service in self._services: + if service.startup_command: + try: + proc = subprocess.Popen( + service.startup_command, + shell=True, + cwd=self.project_dir / service.path + if service.path + else self.project_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self._processes[service.name] = proc + result.services_started.append(service.name) + except Exception as e: + result.errors.append(f"Failed to start {service.name}: {str(e)}") + result.services_failed.append(service.name) + + # Wait for services to be ready + if result.services_started: + if self._wait_for_health(timeout): + result.success = True + else: + result.errors.append("Services did not become healthy in time") + + return result + + def stop_services(self) -> None: + """Stop all running services.""" + if self._compose_file: + self._stop_docker_compose() + else: + self._stop_local_services() + + def _stop_docker_compose(self) -> None: + """Stop services using docker-compose.""" + try: + docker_cmd = self._get_docker_compose_cmd() + if docker_cmd: + subprocess.run( + docker_cmd + ["down"], + cwd=self.project_dir, + capture_output=True, + timeout=60, + ) + except Exception: + pass + + def _stop_local_services(self) -> None: + """Stop local services.""" + for name, proc in self._processes.items(): + try: + proc.terminate() + proc.wait(timeout=10) + except Exception: + try: + proc.kill() + except Exception: + pass + self._processes.clear() + + def _get_docker_compose_cmd(self) -> list[str] | None: + """Get the docker-compose command (v1 or v2).""" + # Try docker compose v2 first + try: + proc = subprocess.run( + ["docker", "compose", "version"], + capture_output=True, + timeout=5, + ) + if proc.returncode == 0: + return ["docker", "compose", "-f", str(self._compose_file)] + except Exception: + pass + + # Try docker-compose v1 + try: + proc = subprocess.run( + ["docker-compose", "version"], + capture_output=True, + timeout=5, + ) + if proc.returncode == 0: + return ["docker-compose", "-f", str(self._compose_file)] + except Exception: + pass + + return None + + def _wait_for_health(self, timeout: int) -> bool: + """ + Wait for all services to become healthy. + + Args: + timeout: Maximum time to wait in seconds + + Returns: + True if all services became healthy + """ + start_time = time.time() + + while time.time() - start_time < timeout: + all_healthy = True + + for service in self._services: + if service.port: + if not self._check_port(service.port): + all_healthy = False + break + + if all_healthy: + return True + + time.sleep(2) + + return False + + def _check_port(self, port: int) -> bool: + """Check if a port is responding.""" + import socket + + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(1) + result = s.connect_ex(("localhost", port)) + return result == 0 + except Exception: + return False + + def to_dict(self) -> dict[str, Any]: + """Convert orchestration config to dictionary.""" + return { + "is_multi_service": self.is_multi_service(), + "has_docker_compose": self.has_docker_compose(), + "compose_file": str(self._compose_file) if self._compose_file else None, + "services": [ + { + "name": s.name, + "path": s.path, + "port": s.port, + "type": s.type, + "health_check_url": s.health_check_url, + } + for s in self._services + ], + } + + +# ============================================================================= +# CONVENIENCE FUNCTIONS +# ============================================================================= + + +def is_multi_service_project(project_dir: Path) -> bool: + """ + Check if project is multi-service. + + Args: + project_dir: Path to project root + + Returns: + True if multi-service project + """ + orchestrator = ServiceOrchestrator(project_dir) + return orchestrator.is_multi_service() + + +def get_service_config(project_dir: Path) -> dict[str, Any]: + """ + Get service configuration for project. + + Args: + project_dir: Path to project root + + Returns: + Dictionary with service configuration + """ + orchestrator = ServiceOrchestrator(project_dir) + return orchestrator.to_dict() + + +# ============================================================================= +# CONTEXT MANAGER +# ============================================================================= + + +class ServiceContext: + """ + Context manager for service orchestration. + + Usage: + with ServiceContext(project_dir) as services: + # Services are running + run_tests() + # Services are stopped + """ + + def __init__(self, project_dir: Path, timeout: int = 120) -> None: + """Initialize service context.""" + self.orchestrator = ServiceOrchestrator(project_dir) + self.timeout = timeout + self.result: OrchestrationResult | None = None + + def __enter__(self) -> "ServiceContext": + """Start services on context entry.""" + if self.orchestrator.is_multi_service(): + self.result = self.orchestrator.start_services(self.timeout) + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + """Stop services on context exit.""" + self.orchestrator.stop_services() + + @property + def success(self) -> bool: + """Check if services started successfully.""" + if self.result: + return self.result.success + return True # No services to start + + +# ============================================================================= +# CLI +# ============================================================================= + + +def main() -> None: + """CLI entry point for testing.""" + import argparse + + parser = argparse.ArgumentParser(description="Service orchestration") + parser.add_argument("project_dir", type=Path, help="Path to project root") + parser.add_argument("--start", action="store_true", help="Start services") + parser.add_argument("--stop", action="store_true", help="Stop services") + parser.add_argument("--status", action="store_true", help="Show service status") + parser.add_argument("--json", action="store_true", help="Output as JSON") + + args = parser.parse_args() + + orchestrator = ServiceOrchestrator(args.project_dir) + + if args.start: + result = orchestrator.start_services() + if args.json: + print( + json.dumps( + { + "success": result.success, + "services_started": result.services_started, + "errors": result.errors, + }, + indent=2, + ) + ) + else: + print(f"Started: {result.services_started}") + if result.errors: + print(f"Errors: {result.errors}") + elif args.stop: + orchestrator.stop_services() + print("Services stopped") + else: + # Default: show status + config = orchestrator.to_dict() + + if args.json: + print(json.dumps(config, indent=2)) + else: + print(f"Multi-service: {config['is_multi_service']}") + print(f"Docker Compose: {config['has_docker_compose']}") + if config["compose_file"]: + print(f"Compose File: {config['compose_file']}") + print(f"\nServices ({len(config['services'])}):") + for service in config["services"]: + port_info = f":{service['port']}" if service["port"] else "" + print(f" - {service['name']} ({service['type']}){port_info}") + + +if __name__ == "__main__": + main() diff --git a/auto-claude/services/recovery.py b/auto-claude/services/recovery.py new file mode 100644 index 00000000..af6fb66c --- /dev/null +++ b/auto-claude/services/recovery.py @@ -0,0 +1,606 @@ +""" +Smart Rollback and Recovery System +=================================== + +Automatic recovery from build failures, stuck loops, and broken builds. +Enables true "walk away" automation by detecting and recovering from common failure modes. + +Key Features: +- Automatic rollback to last working state +- Circular fix detection (prevents infinite loops) +- Attempt history tracking across sessions +- Smart retry with different approaches +- Escalation to human when stuck +""" + +import json +import subprocess +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from pathlib import Path + + +class FailureType(Enum): + """Types of failures that can occur during autonomous builds.""" + + BROKEN_BUILD = "broken_build" # Code doesn't compile/run + VERIFICATION_FAILED = "verification_failed" # Subtask verification failed + CIRCULAR_FIX = "circular_fix" # Same fix attempted multiple times + CONTEXT_EXHAUSTED = "context_exhausted" # Ran out of context mid-subtask + UNKNOWN = "unknown" + + +@dataclass +class RecoveryAction: + """Action to take in response to a failure.""" + + action: str # "rollback", "retry", "skip", "escalate" + target: str # commit hash, subtask id, or message + reason: str + + +class RecoveryManager: + """ + Manages recovery from build failures. + + Responsibilities: + - Track attempt history across sessions + - Classify failures and determine recovery actions + - Rollback to working states + - Detect circular fixes (same approach repeatedly) + - Escalate stuck subtasks for human intervention + """ + + def __init__(self, spec_dir: Path, project_dir: Path): + """ + Initialize recovery manager. + + Args: + spec_dir: Spec directory containing memory/ + project_dir: Root project directory for git operations + """ + self.spec_dir = spec_dir + self.project_dir = project_dir + self.memory_dir = spec_dir / "memory" + self.attempt_history_file = self.memory_dir / "attempt_history.json" + self.build_commits_file = self.memory_dir / "build_commits.json" + + # Ensure memory directory exists + self.memory_dir.mkdir(parents=True, exist_ok=True) + + # Initialize files if they don't exist + if not self.attempt_history_file.exists(): + self._init_attempt_history() + + if not self.build_commits_file.exists(): + self._init_build_commits() + + def _init_attempt_history(self) -> None: + """Initialize the attempt history file.""" + initial_data = { + "subtasks": {}, + "stuck_subtasks": [], + "metadata": { + "created_at": datetime.now().isoformat(), + "last_updated": datetime.now().isoformat(), + }, + } + with open(self.attempt_history_file, "w") as f: + json.dump(initial_data, f, indent=2) + + def _init_build_commits(self) -> None: + """Initialize the build commits tracking file.""" + initial_data = { + "commits": [], + "last_good_commit": None, + "metadata": { + "created_at": datetime.now().isoformat(), + "last_updated": datetime.now().isoformat(), + }, + } + with open(self.build_commits_file, "w") as f: + json.dump(initial_data, f, indent=2) + + def _load_attempt_history(self) -> dict: + """Load attempt history from JSON file.""" + try: + with open(self.attempt_history_file) as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + self._init_attempt_history() + with open(self.attempt_history_file) as f: + return json.load(f) + + def _save_attempt_history(self, data: dict) -> None: + """Save attempt history to JSON file.""" + data["metadata"]["last_updated"] = datetime.now().isoformat() + with open(self.attempt_history_file, "w") as f: + json.dump(data, f, indent=2) + + def _load_build_commits(self) -> dict: + """Load build commits from JSON file.""" + try: + with open(self.build_commits_file) as f: + return json.load(f) + except (OSError, json.JSONDecodeError): + self._init_build_commits() + with open(self.build_commits_file) as f: + return json.load(f) + + def _save_build_commits(self, data: dict) -> None: + """Save build commits to JSON file.""" + data["metadata"]["last_updated"] = datetime.now().isoformat() + with open(self.build_commits_file, "w") as f: + json.dump(data, f, indent=2) + + def classify_failure(self, error: str, subtask_id: str) -> FailureType: + """ + Classify what type of failure occurred. + + Args: + error: Error message or description + subtask_id: ID of the subtask that failed + + Returns: + FailureType enum value + """ + error_lower = error.lower() + + # Check for broken build indicators + build_errors = [ + "syntax error", + "compilation error", + "module not found", + "import error", + "cannot find module", + "unexpected token", + "indentation error", + "parse error", + ] + if any(be in error_lower for be in build_errors): + return FailureType.BROKEN_BUILD + + # Check for verification failures + verification_errors = [ + "verification failed", + "expected", + "assertion", + "test failed", + "status code", + ] + if any(ve in error_lower for ve in verification_errors): + return FailureType.VERIFICATION_FAILED + + # Check for context exhaustion + context_errors = ["context", "token limit", "maximum length"] + if any(ce in error_lower for ce in context_errors): + return FailureType.CONTEXT_EXHAUSTED + + # Check for circular fixes (will be determined by attempt history) + if self.is_circular_fix(subtask_id, error): + return FailureType.CIRCULAR_FIX + + return FailureType.UNKNOWN + + def get_attempt_count(self, subtask_id: str) -> int: + """ + Get how many times this subtask has been attempted. + + Args: + subtask_id: ID of the subtask + + Returns: + Number of attempts + """ + history = self._load_attempt_history() + subtask_data = history["subtasks"].get(subtask_id, {}) + return len(subtask_data.get("attempts", [])) + + def record_attempt( + self, + subtask_id: str, + session: int, + success: bool, + approach: str, + error: str | None = None, + ) -> None: + """ + Record an attempt at a subtask. + + Args: + subtask_id: ID of the subtask + session: Session number + success: Whether the attempt succeeded + approach: Description of the approach taken + error: Error message if failed + """ + history = self._load_attempt_history() + + # Initialize subtask entry if it doesn't exist + if subtask_id not in history["subtasks"]: + history["subtasks"][subtask_id] = {"attempts": [], "status": "pending"} + + # Add the attempt + attempt = { + "session": session, + "timestamp": datetime.now().isoformat(), + "approach": approach, + "success": success, + "error": error, + } + history["subtasks"][subtask_id]["attempts"].append(attempt) + + # Update status + if success: + history["subtasks"][subtask_id]["status"] = "completed" + else: + history["subtasks"][subtask_id]["status"] = "failed" + + self._save_attempt_history(history) + + def is_circular_fix(self, subtask_id: str, current_approach: str) -> bool: + """ + Detect if we're trying the same approach repeatedly. + + Args: + subtask_id: ID of the subtask + current_approach: Description of current approach + + Returns: + True if this appears to be a circular fix attempt + """ + history = self._load_attempt_history() + subtask_data = history["subtasks"].get(subtask_id, {}) + attempts = subtask_data.get("attempts", []) + + if len(attempts) < 2: + return False + + # Check if last 3 attempts used similar approaches + # Simple similarity check: look for repeated keywords + recent_attempts = attempts[-3:] if len(attempts) >= 3 else attempts + + # Extract key terms from current approach (ignore common words) + stop_words = { + "with", + "using", + "the", + "a", + "an", + "and", + "or", + "but", + "in", + "on", + "at", + "to", + "for", + "trying", + } + current_keywords = set( + word for word in current_approach.lower().split() if word not in stop_words + ) + + similar_count = 0 + for attempt in recent_attempts: + attempt_keywords = set( + word + for word in attempt["approach"].lower().split() + if word not in stop_words + ) + + # Calculate Jaccard similarity (intersection over union) + overlap = len(current_keywords & attempt_keywords) + total = len(current_keywords | attempt_keywords) + + if total > 0: + similarity = overlap / total + # If >30% of meaningful words overlap, consider it similar + # This catches key technical terms appearing repeatedly + # (e.g., "async await" across multiple attempts) + if similarity > 0.3: + similar_count += 1 + + # If 2+ recent attempts were similar to current approach, it's circular + return similar_count >= 2 + + def determine_recovery_action( + self, failure_type: FailureType, subtask_id: str + ) -> RecoveryAction: + """ + Decide what to do based on failure type and history. + + Args: + failure_type: Type of failure that occurred + subtask_id: ID of the subtask that failed + + Returns: + RecoveryAction describing what to do + """ + attempt_count = self.get_attempt_count(subtask_id) + + if failure_type == FailureType.BROKEN_BUILD: + # Broken build: rollback to last good state + last_good = self.get_last_good_commit() + if last_good: + return RecoveryAction( + action="rollback", + target=last_good, + reason=f"Build broken in subtask {subtask_id}, rolling back to working state", + ) + else: + return RecoveryAction( + action="escalate", + target=subtask_id, + reason="Build broken and no good commit found to rollback to", + ) + + elif failure_type == FailureType.VERIFICATION_FAILED: + # Verification failed: retry with different approach if < 3 attempts + if attempt_count < 3: + return RecoveryAction( + action="retry", + target=subtask_id, + reason=f"Verification failed, retry with different approach (attempt {attempt_count + 1}/3)", + ) + else: + return RecoveryAction( + action="skip", + target=subtask_id, + reason=f"Verification failed after {attempt_count} attempts, marking as stuck", + ) + + elif failure_type == FailureType.CIRCULAR_FIX: + # Circular fix detected: skip and escalate + return RecoveryAction( + action="skip", + target=subtask_id, + reason="Circular fix detected - same approach tried multiple times", + ) + + elif failure_type == FailureType.CONTEXT_EXHAUSTED: + # Context exhausted: commit current progress and continue + return RecoveryAction( + action="continue", + target=subtask_id, + reason="Context exhausted, will commit progress and continue in next session", + ) + + else: # UNKNOWN + # Unknown error: retry once, then escalate + if attempt_count < 2: + return RecoveryAction( + action="retry", + target=subtask_id, + reason=f"Unknown error, retrying (attempt {attempt_count + 1}/2)", + ) + else: + return RecoveryAction( + action="escalate", + target=subtask_id, + reason=f"Unknown error persists after {attempt_count} attempts", + ) + + def get_last_good_commit(self) -> str | None: + """ + Find the most recent commit where build was working. + + Returns: + Commit hash or None + """ + commits = self._load_build_commits() + return commits.get("last_good_commit") + + def record_good_commit(self, commit_hash: str, subtask_id: str) -> None: + """ + Record a commit where the build was working. + + Args: + commit_hash: Git commit hash + subtask_id: Subtask that was successfully completed + """ + commits = self._load_build_commits() + + commit_record = { + "hash": commit_hash, + "subtask_id": subtask_id, + "timestamp": datetime.now().isoformat(), + } + + commits["commits"].append(commit_record) + commits["last_good_commit"] = commit_hash + + self._save_build_commits(commits) + + def rollback_to_commit(self, commit_hash: str) -> bool: + """ + Rollback to a specific commit. + + Args: + commit_hash: Git commit hash to rollback to + + Returns: + True if successful, False otherwise + """ + try: + # Use git reset --hard to rollback + result = subprocess.run( + ["git", "reset", "--hard", commit_hash], + cwd=self.project_dir, + capture_output=True, + text=True, + check=True, + ) + return True + except subprocess.CalledProcessError as e: + print(f"Error rolling back to {commit_hash}: {e.stderr}") + return False + + def mark_subtask_stuck(self, subtask_id: str, reason: str) -> None: + """ + Mark a subtask as needing human intervention. + + Args: + subtask_id: ID of the subtask + reason: Why it's stuck + """ + history = self._load_attempt_history() + + stuck_entry = { + "subtask_id": subtask_id, + "reason": reason, + "escalated_at": datetime.now().isoformat(), + "attempt_count": self.get_attempt_count(subtask_id), + } + + # Check if already in stuck list + existing = [ + s for s in history["stuck_subtasks"] if s["subtask_id"] == subtask_id + ] + if not existing: + history["stuck_subtasks"].append(stuck_entry) + + # Update subtask status + if subtask_id in history["subtasks"]: + history["subtasks"][subtask_id]["status"] = "stuck" + + self._save_attempt_history(history) + + def get_stuck_subtasks(self) -> list[dict]: + """ + Get all subtasks marked as stuck. + + Returns: + List of stuck subtask entries + """ + history = self._load_attempt_history() + return history.get("stuck_subtasks", []) + + def get_subtask_history(self, subtask_id: str) -> dict: + """ + Get the attempt history for a specific subtask. + + Args: + subtask_id: ID of the subtask + + Returns: + Subtask history dict with attempts + """ + history = self._load_attempt_history() + return history["subtasks"].get( + subtask_id, {"attempts": [], "status": "pending"} + ) + + def get_recovery_hints(self, subtask_id: str) -> list[str]: + """ + Get hints for recovery based on previous attempts. + + Args: + subtask_id: ID of the subtask + + Returns: + List of hint strings + """ + subtask_history = self.get_subtask_history(subtask_id) + attempts = subtask_history.get("attempts", []) + + if not attempts: + return ["This is the first attempt at this subtask"] + + hints = [f"Previous attempts: {len(attempts)}"] + + # Add info about what was tried + for i, attempt in enumerate(attempts[-3:], 1): + hints.append( + f"Attempt {i}: {attempt['approach']} - " + f"{'SUCCESS' if attempt['success'] else 'FAILED'}" + ) + if attempt.get("error"): + hints.append(f" Error: {attempt['error'][:100]}") + + # Add guidance + if len(attempts) >= 2: + hints.append( + "\n⚠️ IMPORTANT: Try a DIFFERENT approach than previous attempts" + ) + hints.append( + "Consider: different library, different pattern, or simpler implementation" + ) + + return hints + + def clear_stuck_subtasks(self) -> None: + """Clear all stuck subtasks (for manual resolution).""" + history = self._load_attempt_history() + history["stuck_subtasks"] = [] + self._save_attempt_history(history) + + def reset_subtask(self, subtask_id: str) -> None: + """ + Reset a subtask's attempt history. + + Args: + subtask_id: ID of the subtask to reset + """ + history = self._load_attempt_history() + + # Clear attempt history + if subtask_id in history["subtasks"]: + history["subtasks"][subtask_id] = {"attempts": [], "status": "pending"} + + # Remove from stuck subtasks + history["stuck_subtasks"] = [ + s for s in history["stuck_subtasks"] if s["subtask_id"] != subtask_id + ] + + self._save_attempt_history(history) + + +# Utility functions for integration with agent.py + + +def check_and_recover( + spec_dir: Path, project_dir: Path, subtask_id: str, error: str | None = None +) -> RecoveryAction | None: + """ + Check if recovery is needed and return appropriate action. + + Args: + spec_dir: Spec directory + project_dir: Project directory + subtask_id: Current subtask ID + error: Error message if any + + Returns: + RecoveryAction if recovery needed, None otherwise + """ + if not error: + return None + + manager = RecoveryManager(spec_dir, project_dir) + failure_type = manager.classify_failure(error, subtask_id) + + return manager.determine_recovery_action(failure_type, subtask_id) + + +def get_recovery_context(spec_dir: Path, project_dir: Path, subtask_id: str) -> dict: + """ + Get recovery context for a subtask (for prompt generation). + + Args: + spec_dir: Spec directory + project_dir: Project directory + subtask_id: Subtask ID + + Returns: + Dict with recovery hints and history + """ + manager = RecoveryManager(spec_dir, project_dir) + + return { + "attempt_count": manager.get_attempt_count(subtask_id), + "hints": manager.get_recovery_hints(subtask_id), + "subtask_history": manager.get_subtask_history(subtask_id), + "stuck_subtasks": manager.get_stuck_subtasks(), + } diff --git a/auto-claude/spec/critique.py b/auto-claude/spec/critique.py new file mode 100644 index 00000000..3308db84 --- /dev/null +++ b/auto-claude/spec/critique.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +""" +Self-Critique System +==================== + +Implements a self-critique loop that agents must run before marking subtasks complete. +This helps catch quality issues early, before verification stage. + +The critique system ensures: +- Code follows patterns from reference files +- All required files were modified/created +- Error handling is present +- No debugging artifacts left behind +- Implementation matches subtask requirements +""" + +import re +from dataclasses import dataclass, field + + +@dataclass +class CritiqueResult: + """Result of a self-critique evaluation.""" + + passes: bool + issues: list[str] = field(default_factory=list) + improvements_made: list[str] = field(default_factory=list) + recommendations: list[str] = field(default_factory=list) + + def to_dict(self) -> dict: + """Convert to dictionary for storage.""" + return { + "passes": self.passes, + "issues": self.issues, + "improvements_made": self.improvements_made, + "recommendations": self.recommendations, + } + + @classmethod + def from_dict(cls, data: dict) -> "CritiqueResult": + """Load from dictionary.""" + return cls( + passes=data.get("passes", False), + issues=data.get("issues", []), + improvements_made=data.get("improvements_made", []), + recommendations=data.get("recommendations", []), + ) + + +def generate_critique_prompt( + subtask: dict, files_modified: list[str], patterns_from: list[str] +) -> str: + """ + Generate a critique prompt for the agent to self-evaluate. + + Args: + subtask: The subtask being implemented + files_modified: List of files actually modified + patterns_from: List of pattern files to compare against + + Returns: + Formatted prompt for self-critique + """ + subtask_id = subtask.get("id", "unknown") + subtask_desc = subtask.get("description", "No description") + service = subtask.get("service", "all services") + files_to_modify = subtask.get("files_to_modify", []) + files_to_create = subtask.get("files_to_create", []) + + prompt = f"""## MANDATORY Self-Critique: {subtask_id} + +**Subtask Description:** {subtask_desc} +**Service:** {service} + +Before marking this subtask as complete, you MUST perform a thorough self-critique. +This is NOT optional - it's a required quality gate. + +### STEP 1: Code Quality Checklist + +Review your implementation against these criteria: + +**Pattern Adherence:** +- [ ] Follows patterns from reference files exactly: {", ".join(patterns_from) if patterns_from else "N/A"} +- [ ] Variable naming matches codebase conventions +- [ ] Imports organized correctly (grouped, sorted) +- [ ] Code style consistent with existing files + +**Error Handling:** +- [ ] Try-catch blocks where operations can fail +- [ ] Meaningful error messages +- [ ] Proper error propagation +- [ ] Edge cases considered + +**Code Cleanliness:** +- [ ] No console.log/print statements for debugging +- [ ] No commented-out code blocks +- [ ] No TODO comments without context +- [ ] No hardcoded values that should be configurable + +**Best Practices:** +- [ ] Functions are focused and single-purpose +- [ ] No code duplication +- [ ] Appropriate use of constants +- [ ] Documentation/comments where needed + +### STEP 2: Implementation Completeness + +**Files Modified:** +Expected: {", ".join(files_to_modify) if files_to_modify else "None"} +Actual: {", ".join(files_modified) if files_modified else "None"} +- [ ] All files_to_modify were actually modified +- [ ] No unexpected files were modified + +**Files Created:** +Expected: {", ".join(files_to_create) if files_to_create else "None"} +- [ ] All files_to_create were actually created +- [ ] Files follow naming conventions + +**Requirements:** +- [ ] Subtask description requirements fully met +- [ ] All acceptance criteria from spec considered +- [ ] No scope creep - stayed within subtask boundaries + +### STEP 3: Potential Issues Analysis + +List any concerns, limitations, or potential problems with your implementation: + +1. [Issue 1, or "None identified"] +2. [Issue 2, if any] +3. [Issue 3, if any] + +Be honest. Finding issues now is better than discovering them during verification. + +### STEP 4: Improvements Made + +If you identified issues in your critique, list what you fixed: + +1. [Improvement 1, or "No fixes needed"] +2. [Improvement 2, if applicable] +3. [Improvement 3, if applicable] + +### STEP 5: Final Verdict + +**PROCEED:** [YES/NO - Only YES if all critical items pass] + +**REASON:** [Brief explanation of your decision] + +**CONFIDENCE:** [High/Medium/Low - How confident are you in this implementation?] + +--- + +## Instructions for Agent + +1. Work through each section methodically +2. Check each box honestly - don't skip items +3. If you find issues, FIX THEM before continuing +4. Re-run this critique after fixes +5. Only mark the subtask complete when verdict is YES with High confidence +6. Document your critique results in your response + +Remember: The next session has no context. Quality issues you miss now will be harder to fix later. +""" + + return prompt + + +def parse_critique_response(response: str) -> CritiqueResult: + """ + Parse the agent's critique response into structured data. + + Args: + response: The agent's response to the critique prompt + + Returns: + CritiqueResult with parsed information + """ + issues = [] + improvements = [] + recommendations = [] + passes = False + + # Extract PROCEED verdict + proceed_match = re.search( + r"\*\*PROCEED:\*\*\s*\[?\s*(YES|NO)", response, re.IGNORECASE + ) + if proceed_match: + passes = proceed_match.group(1).upper() == "YES" + + # Extract issues from Step 3 + issues_section = re.search( + r"### STEP 3:.*?Potential Issues.*?\n\n(.*?)(?=###|\Z)", + response, + re.DOTALL | re.IGNORECASE, + ) + if issues_section: + issue_lines = issues_section.group(1).strip().split("\n") + for line in issue_lines: + line = line.strip() + if not line or line.startswith("---"): + continue + # Remove list markers + issue = re.sub(r"^\d+\.\s*|\*\s*|-\s*", "", line).strip() + # Skip if it's a placeholder or indicates no issues + if ( + issue + and issue.lower() + not in ["none", "none identified", "no issues", "no concerns"] + and issue + not in [ + '[Issue 1, or "None identified"]', + "[Issue 2, if any]", + "[Issue 3, if any]", + ] + ): + issues.append(issue) + + # Extract improvements from Step 4 + improvements_section = re.search( + r"### STEP 4:.*?Improvements Made.*?\n\n(.*?)(?=###|\Z)", + response, + re.DOTALL | re.IGNORECASE, + ) + if improvements_section: + improvement_lines = improvements_section.group(1).strip().split("\n") + for line in improvement_lines: + line = line.strip() + if not line or line.startswith("---"): + continue + # Remove list markers + improvement = re.sub(r"^\d+\.\s*|\*\s*|-\s*", "", line).strip() + # Skip if it's a placeholder or indicates no improvements + if ( + improvement + and improvement.lower() + not in ["none", "no fixes needed", "no improvements", "n/a"] + and improvement + not in [ + '[Improvement 1, or "No fixes needed"]', + "[Improvement 2, if applicable]", + "[Improvement 3, if applicable]", + ] + ): + improvements.append(improvement) + + # Extract confidence level as recommendation + confidence_match = re.search( + r"\*\*CONFIDENCE:\*\*\s*\[?\s*(High|Medium|Low)", response, re.IGNORECASE + ) + if confidence_match: + confidence = confidence_match.group(1) + if confidence.lower() != "high": + recommendations.append( + f"Confidence level: {confidence} - consider additional review" + ) + + return CritiqueResult( + passes=passes, + issues=issues, + improvements_made=improvements, + recommendations=recommendations, + ) + + +def should_proceed(result: CritiqueResult) -> bool: + """ + Determine if the subtask should be marked complete based on critique. + + Args: + result: The critique result + + Returns: + True if subtask can be marked complete, False otherwise + """ + # Must pass the critique + if not result.passes: + return False + + # If there are unresolved issues, don't proceed + if result.issues: + return False + + return True + + +def format_critique_summary(result: CritiqueResult) -> str: + """ + Format a critique result as a human-readable summary. + + Args: + result: The critique result + + Returns: + Formatted summary string + """ + lines = ["## Critique Summary"] + lines.append("") + lines.append(f"**Status:** {'PASSED ✓' if result.passes else 'FAILED ✗'}") + lines.append("") + + if result.issues: + lines.append("**Issues Identified:**") + for i, issue in enumerate(result.issues, 1): + lines.append(f"{i}. {issue}") + lines.append("") + + if result.improvements_made: + lines.append("**Improvements Made:**") + for i, improvement in enumerate(result.improvements_made, 1): + lines.append(f"{i}. {improvement}") + lines.append("") + + if result.recommendations: + lines.append("**Recommendations:**") + for i, rec in enumerate(result.recommendations, 1): + lines.append(f"{i}. {rec}") + lines.append("") + + if should_proceed(result): + lines.append("**Decision:** Subtask is ready to be marked complete.") + else: + lines.append("**Decision:** Subtask needs more work before completion.") + + return "\n".join(lines) + + +# Example usage for testing +if __name__ == "__main__": + # Demo subtask + subtask = { + "id": "auth-middleware", + "description": "Add JWT authentication middleware", + "service": "backend", + "files_to_modify": ["app/middleware/auth.py"], + "patterns_from": ["app/middleware/cors.py"], + } + + files_modified = ["app/middleware/auth.py"] + + # Generate prompt + prompt = generate_critique_prompt(subtask, files_modified, subtask["patterns_from"]) + print(prompt) + print("\n" + "=" * 80 + "\n") + + # Simulate a critique response + sample_response = """ +### STEP 3: Potential Issues Analysis + +1. Token expiration edge case not fully tested +2. None + +### STEP 4: Improvements Made + +1. Added comprehensive error handling for invalid tokens +2. Improved logging for debugging +3. Added input validation for JWT format + +### STEP 5: Final Verdict + +**PROCEED:** YES + +**REASON:** All critical items verified, patterns followed, error handling complete + +**CONFIDENCE:** High +""" + + # Parse response + result = parse_critique_response(sample_response) + print(format_critique_summary(result)) + print(f"\nShould proceed: {should_proceed(result)}") diff --git a/auto-claude/spec/phases/planning_phases.py b/auto-claude/spec/phases/planning_phases.py index b19125b4..314387d6 100644 --- a/auto-claude/spec/phases/planning_phases.py +++ b/auto-claude/spec/phases/planning_phases.py @@ -22,7 +22,7 @@ class PlanningPhaseMixin: async def phase_planning(self) -> PhaseResult: """Create the implementation plan.""" - from validate_spec.auto_fix import auto_fix_plan + from ..validate_pkg.auto_fix import auto_fix_plan plan_file = self.spec_dir / "implementation_plan.json" diff --git a/auto-claude/spec/pipeline/__init__.py b/auto-claude/spec/pipeline/__init__.py index e4476fd0..515c5f1c 100644 --- a/auto-claude/spec/pipeline/__init__.py +++ b/auto-claude/spec/pipeline/__init__.py @@ -10,10 +10,12 @@ Components: - orchestrator: Main SpecOrchestrator class """ +from init import init_auto_claude_dir from .models import get_specs_dir from .orchestrator import SpecOrchestrator __all__ = [ "SpecOrchestrator", "get_specs_dir", + "init_auto_claude_dir", ] diff --git a/auto-claude/spec/pipeline/orchestrator.py b/auto-claude/spec/pipeline/orchestrator.py index df8df91a..0c81977f 100644 --- a/auto-claude/spec/pipeline/orchestrator.py +++ b/auto-claude/spec/pipeline/orchestrator.py @@ -25,7 +25,7 @@ from ui import ( print_section, print_status, ) -from validate_spec.spec_validator import SpecValidator +from ..validate_pkg.spec_validator import SpecValidator from .. import complexity, phases, requirements from .agent_runner import AgentRunner @@ -547,3 +547,41 @@ class SpecOrchestrator: return False return True + + # Backward compatibility methods for tests + def _generate_spec_name(self, task_description: str) -> str: + """Generate a spec name from task description (backward compatibility). + + This method is kept for backward compatibility with existing tests. + The functionality has been moved to models.generate_spec_name. + + Args: + task_description: The task description + + Returns: + Generated spec name + """ + from .models import generate_spec_name + + return generate_spec_name(task_description) + + def _rename_spec_dir_from_requirements(self) -> bool: + """Rename spec directory from requirements (backward compatibility). + + This method is kept for backward compatibility with existing tests. + The functionality has been moved to models.rename_spec_dir_from_requirements. + + Returns: + True if successful or not needed, False on error + """ + result = rename_spec_dir_from_requirements(self.spec_dir) + # Update self.spec_dir if it was renamed + if result and self.spec_dir.name.endswith("-pending"): + # Find the renamed directory + parent = self.spec_dir.parent + prefix = self.spec_dir.name[:4] # e.g., "001-" + for candidate in parent.iterdir(): + if candidate.name.startswith(prefix) and "pending" not in candidate.name: + self.spec_dir = candidate + break + return result diff --git a/auto-claude/validate_spec/MIGRATION.md b/auto-claude/spec/validate_pkg/MIGRATION.md similarity index 100% rename from auto-claude/validate_spec/MIGRATION.md rename to auto-claude/spec/validate_pkg/MIGRATION.md diff --git a/auto-claude/validate_spec/README.md b/auto-claude/spec/validate_pkg/README.md similarity index 100% rename from auto-claude/validate_spec/README.md rename to auto-claude/spec/validate_pkg/README.md diff --git a/auto-claude/spec/validate_pkg/__init__.py b/auto-claude/spec/validate_pkg/__init__.py new file mode 100644 index 00000000..9f4061e9 --- /dev/null +++ b/auto-claude/spec/validate_pkg/__init__.py @@ -0,0 +1,19 @@ +""" +Spec Validation System +====================== + +Validates spec outputs at each checkpoint to ensure reliability. +This is the enforcement layer that catches errors before they propagate. + +The spec creation process has mandatory checkpoints: +1. Prerequisites (project_index.json exists) +2. Context (context.json created with required fields) +3. Spec document (spec.md with required sections) +4. Implementation plan (implementation_plan.json with valid schema) +""" + +from .auto_fix import auto_fix_plan +from .models import ValidationResult +from .spec_validator import SpecValidator + +__all__ = ["SpecValidator", "ValidationResult", "auto_fix_plan"] diff --git a/auto-claude/spec/validate_pkg/auto_fix.py b/auto-claude/spec/validate_pkg/auto_fix.py new file mode 100644 index 00000000..e89e8ca6 --- /dev/null +++ b/auto-claude/spec/validate_pkg/auto_fix.py @@ -0,0 +1,80 @@ +""" +Auto-Fix Utilities +================== + +Automated fixes for common implementation plan issues. +""" + +import json +from pathlib import Path + + +def auto_fix_plan(spec_dir: Path) -> bool: + """Attempt to auto-fix common implementation_plan.json issues. + + Args: + spec_dir: Path to the spec directory + + Returns: + True if fixes were applied, False otherwise + """ + plan_file = spec_dir / "implementation_plan.json" + + if not plan_file.exists(): + return False + + try: + with open(plan_file) as f: + plan = json.load(f) + except json.JSONDecodeError: + return False + + fixed = False + + # Fix missing top-level fields + if "feature" not in plan: + plan["feature"] = "Unnamed Feature" + fixed = True + + if "workflow_type" not in plan: + plan["workflow_type"] = "feature" + fixed = True + + if "phases" not in plan: + plan["phases"] = [] + fixed = True + + # Fix phases + for i, phase in enumerate(plan.get("phases", [])): + if "phase" not in phase: + phase["phase"] = i + 1 + fixed = True + + if "name" not in phase: + phase["name"] = f"Phase {i + 1}" + fixed = True + + if "subtasks" not in phase: + phase["subtasks"] = [] + fixed = True + + # Fix subtasks + for j, subtask in enumerate(phase.get("subtasks", [])): + if "id" not in subtask: + subtask["id"] = f"subtask-{i + 1}-{j + 1}" + fixed = True + + if "description" not in subtask: + subtask["description"] = "No description" + fixed = True + + if "status" not in subtask: + subtask["status"] = "pending" + fixed = True + + if fixed: + with open(plan_file, "w") as f: + json.dump(plan, f, indent=2) + print(f"Auto-fixed: {plan_file}") + + return fixed diff --git a/auto-claude/validate_spec/models.py b/auto-claude/spec/validate_pkg/models.py similarity index 100% rename from auto-claude/validate_spec/models.py rename to auto-claude/spec/validate_pkg/models.py diff --git a/auto-claude/validate_spec/schemas.py b/auto-claude/spec/validate_pkg/schemas.py similarity index 100% rename from auto-claude/validate_spec/schemas.py rename to auto-claude/spec/validate_pkg/schemas.py diff --git a/auto-claude/spec/validate_pkg/spec_validator.py b/auto-claude/spec/validate_pkg/spec_validator.py new file mode 100644 index 00000000..1b8064de --- /dev/null +++ b/auto-claude/spec/validate_pkg/spec_validator.py @@ -0,0 +1,80 @@ +""" +Spec Validator +============== + +Main validator class that orchestrates all validation checkpoints. +""" + +from pathlib import Path + +from .models import ValidationResult +from .validators import ( + ContextValidator, + ImplementationPlanValidator, + PrereqsValidator, + SpecDocumentValidator, +) + + +class SpecValidator: + """Validates spec outputs at each checkpoint.""" + + def __init__(self, spec_dir: Path): + """Initialize the spec validator. + + Args: + spec_dir: Path to the spec directory + """ + self.spec_dir = Path(spec_dir) + + # Initialize individual validators + self._prereqs_validator = PrereqsValidator(self.spec_dir) + self._context_validator = ContextValidator(self.spec_dir) + self._spec_document_validator = SpecDocumentValidator(self.spec_dir) + self._implementation_plan_validator = ImplementationPlanValidator(self.spec_dir) + + def validate_all(self) -> list[ValidationResult]: + """Run all validations. + + Returns: + List of validation results for all checkpoints + """ + results = [ + self.validate_prereqs(), + self.validate_context(), + self.validate_spec_document(), + self.validate_implementation_plan(), + ] + return results + + def validate_prereqs(self) -> ValidationResult: + """Validate prerequisites exist. + + Returns: + ValidationResult for prerequisites checkpoint + """ + return self._prereqs_validator.validate() + + def validate_context(self) -> ValidationResult: + """Validate context.json exists and has required structure. + + Returns: + ValidationResult for context checkpoint + """ + return self._context_validator.validate() + + def validate_spec_document(self) -> ValidationResult: + """Validate spec.md exists and has required sections. + + Returns: + ValidationResult for spec document checkpoint + """ + return self._spec_document_validator.validate() + + def validate_implementation_plan(self) -> ValidationResult: + """Validate implementation_plan.json exists and has valid schema. + + Returns: + ValidationResult for implementation plan checkpoint + """ + return self._implementation_plan_validator.validate() diff --git a/auto-claude/validate_spec/validators/__init__.py b/auto-claude/spec/validate_pkg/validators/__init__.py similarity index 100% rename from auto-claude/validate_spec/validators/__init__.py rename to auto-claude/spec/validate_pkg/validators/__init__.py diff --git a/auto-claude/validate_spec/validators/context_validator.py b/auto-claude/spec/validate_pkg/validators/context_validator.py similarity index 100% rename from auto-claude/validate_spec/validators/context_validator.py rename to auto-claude/spec/validate_pkg/validators/context_validator.py diff --git a/auto-claude/validate_spec/validators/implementation_plan_validator.py b/auto-claude/spec/validate_pkg/validators/implementation_plan_validator.py similarity index 100% rename from auto-claude/validate_spec/validators/implementation_plan_validator.py rename to auto-claude/spec/validate_pkg/validators/implementation_plan_validator.py diff --git a/auto-claude/validate_spec/validators/prereqs_validator.py b/auto-claude/spec/validate_pkg/validators/prereqs_validator.py similarity index 100% rename from auto-claude/validate_spec/validators/prereqs_validator.py rename to auto-claude/spec/validate_pkg/validators/prereqs_validator.py diff --git a/auto-claude/validate_spec/validators/spec_document_validator.py b/auto-claude/spec/validate_pkg/validators/spec_document_validator.py similarity index 100% rename from auto-claude/validate_spec/validators/spec_document_validator.py rename to auto-claude/spec/validate_pkg/validators/spec_document_validator.py diff --git a/auto-claude/validate_spec.py b/auto-claude/spec/validate_spec.py similarity index 98% rename from auto-claude/validate_spec.py rename to auto-claude/spec/validate_spec.py index 22475367..5b5cdaba 100644 --- a/auto-claude/validate_spec.py +++ b/auto-claude/spec/validate_spec.py @@ -19,7 +19,7 @@ import json import sys from pathlib import Path -from validate_spec import SpecValidator, auto_fix_plan +from validate_pkg import SpecValidator, auto_fix_plan def main() -> None: diff --git a/auto-claude/spec/validation_strategy.py b/auto-claude/spec/validation_strategy.py new file mode 100644 index 00000000..cad467e2 --- /dev/null +++ b/auto-claude/spec/validation_strategy.py @@ -0,0 +1,1033 @@ +#!/usr/bin/env python3 +""" +Validation Strategy Module +========================== + +Builds validation strategies based on project type and risk level. +This module determines how the QA agent should validate implementations. + +The validation strategy is used by: +- Planner Agent: To define verification requirements in the implementation plan +- QA Agent: To determine what tests to create and run + +Usage: + from validation_strategy import ValidationStrategyBuilder + + builder = ValidationStrategyBuilder() + strategy = builder.build_strategy(project_dir, spec_dir, "medium") + + for step in strategy: + print(f"Run: {step.command}") +""" + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from risk_classifier import RiskClassifier + +# ============================================================================= +# DATA CLASSES +# ============================================================================= + + +@dataclass +class ValidationStep: + """ + A single validation step to execute. + + Attributes: + name: Human-readable name of the step + command: Command to execute (or "manual" for manual steps) + expected_outcome: Description of what success looks like + step_type: Type of validation (test, visual, api, security, manual) + required: Whether this step is mandatory + blocking: Whether failure blocks approval + """ + + name: str + command: str + expected_outcome: str + step_type: str # test, visual, api, security, manual + required: bool = True + blocking: bool = True + + +@dataclass +class ValidationStrategy: + """ + Complete validation strategy for a task. + + Attributes: + risk_level: Risk level (trivial, low, medium, high, critical) + project_type: Detected project type + steps: List of validation steps to execute + test_types_required: List of test types to create + security_scan_required: Whether security scanning is needed + staging_deployment_required: Whether staging deployment is needed + skip_validation: Whether validation can be skipped entirely + reasoning: Explanation of the strategy + """ + + risk_level: str + project_type: str + steps: list[ValidationStep] = field(default_factory=list) + test_types_required: list[str] = field(default_factory=list) + security_scan_required: bool = False + staging_deployment_required: bool = False + skip_validation: bool = False + reasoning: str = "" + + +# ============================================================================= +# PROJECT TYPE DETECTION +# ============================================================================= + + +# Project type indicators +PROJECT_TYPE_INDICATORS = { + "html_css": { + "files": ["index.html", "style.css", "styles.css"], + "extensions": [".html", ".css"], + "no_package_manager": True, + }, + "react_spa": { + "dependencies": ["react", "react-dom"], + "files": ["package.json"], + }, + "vue_spa": { + "dependencies": ["vue"], + "files": ["package.json"], + }, + "nextjs": { + "dependencies": ["next"], + "files": ["next.config.js", "next.config.mjs", "next.config.ts"], + }, + "nodejs": { + "files": ["package.json"], + "not_dependencies": ["react", "vue", "next", "angular"], + }, + "python_api": { + "dependencies_python": ["fastapi", "flask", "django"], + "files": ["pyproject.toml", "setup.py", "requirements.txt"], + }, + "python_cli": { + "files": ["pyproject.toml", "setup.py"], + "entry_points": True, + }, + "rust": { + "files": ["Cargo.toml"], + }, + "go": { + "files": ["go.mod"], + }, + "ruby": { + "files": ["Gemfile"], + }, +} + + +def detect_project_type(project_dir: Path) -> str: + """ + Detect the project type based on files and dependencies. + + Args: + project_dir: Path to the project directory + + Returns: + Project type string (e.g., "react_spa", "python_api", "nodejs") + """ + project_dir = Path(project_dir) + + # Check for specific frameworks first + package_json = project_dir / "package.json" + if package_json.exists(): + try: + with open(package_json, encoding="utf-8") as f: + pkg = json.load(f) + deps = pkg.get("dependencies", {}) + dev_deps = pkg.get("devDependencies", {}) + all_deps = {**deps, **dev_deps} + + if "electron" in all_deps: + return "electron" + if "next" in all_deps: + return "nextjs" + if "react" in all_deps: + return "react_spa" + if "vue" in all_deps: + return "vue_spa" + if "@angular/core" in all_deps: + return "angular_spa" + return "nodejs" + except (OSError, json.JSONDecodeError): + return "nodejs" + + # Check for Python projects + pyproject = project_dir / "pyproject.toml" + requirements = project_dir / "requirements.txt" + if pyproject.exists() or requirements.exists(): + # Try to detect API framework + deps_text = "" + if requirements.exists(): + deps_text = requirements.read_text().lower() + if pyproject.exists(): + deps_text += pyproject.read_text().lower() + + if "fastapi" in deps_text or "flask" in deps_text or "django" in deps_text: + return "python_api" + if "click" in deps_text or "typer" in deps_text or "argparse" in deps_text: + return "python_cli" + return "python" + + # Check for other languages + if (project_dir / "Cargo.toml").exists(): + return "rust" + if (project_dir / "go.mod").exists(): + return "go" + if (project_dir / "Gemfile").exists(): + return "ruby" + + # Check for simple HTML/CSS + html_files = list(project_dir.glob("*.html")) + if html_files: + return "html_css" + + return "unknown" + + +# ============================================================================= +# VALIDATION STRATEGY BUILDER +# ============================================================================= + + +class ValidationStrategyBuilder: + """ + Builds validation strategies based on project type and risk level. + + The builder uses the risk assessment from complexity_assessment.json + and adapts the validation strategy to the detected project type. + """ + + def __init__(self) -> None: + """Initialize the strategy builder.""" + self._risk_classifier = RiskClassifier() + + def build_strategy( + self, + project_dir: Path, + spec_dir: Path, + risk_level: str | None = None, + ) -> ValidationStrategy: + """ + Build a validation strategy for the given project and spec. + + Args: + project_dir: Path to the project root + spec_dir: Path to the spec directory + risk_level: Override risk level (if not provided, reads from assessment) + + Returns: + ValidationStrategy with appropriate steps + """ + project_dir = Path(project_dir) + spec_dir = Path(spec_dir) + + # Get risk level from assessment if not provided + if risk_level is None: + assessment = self._risk_classifier.load_assessment(spec_dir) + if assessment: + risk_level = assessment.validation.risk_level + else: + risk_level = "medium" # Default to medium + + # Detect project type + project_type = detect_project_type(project_dir) + + # Build strategy based on project type + strategy_builders = { + "html_css": self._strategy_for_html_css, + "react_spa": self._strategy_for_spa, + "vue_spa": self._strategy_for_spa, + "angular_spa": self._strategy_for_spa, + "nextjs": self._strategy_for_fullstack, + "nodejs": self._strategy_for_nodejs, + "electron": self._strategy_for_electron, + "python_api": self._strategy_for_python_api, + "python_cli": self._strategy_for_cli, + "python": self._strategy_for_python, + "rust": self._strategy_for_rust, + "go": self._strategy_for_go, + "ruby": self._strategy_for_ruby, + } + + builder_func = strategy_builders.get(project_type, self._strategy_default) + strategy = builder_func(project_dir, risk_level) + + # Add security scanning for high+ risk + if risk_level in ["high", "critical"]: + strategy = self._add_security_steps(strategy, project_type) + + # Set common properties + strategy.risk_level = risk_level + strategy.project_type = project_type + strategy.skip_validation = risk_level == "trivial" + + return strategy + + def _strategy_for_html_css( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for simple HTML/CSS projects. + + Focus on visual verification and accessibility. + """ + steps = [ + ValidationStep( + name="Start HTTP Server", + command="python -m http.server 8000 &", + expected_outcome="Server running on port 8000", + step_type="setup", + required=True, + blocking=True, + ), + ValidationStep( + name="Visual Verification", + command="npx playwright screenshot http://localhost:8000 screenshot.png", + expected_outcome="Screenshot captured without errors", + step_type="visual", + required=True, + blocking=False, + ), + ValidationStep( + name="Console Error Check", + command="npx playwright test --grep 'console-errors'", + expected_outcome="No JavaScript console errors", + step_type="test", + required=True, + blocking=True, + ), + ] + + # Add Lighthouse for medium+ risk + if risk_level in ["medium", "high", "critical"]: + steps.append( + ValidationStep( + name="Lighthouse Audit", + command="npx lighthouse http://localhost:8000 --output=json --output-path=lighthouse.json", + expected_outcome="Performance > 90, Accessibility > 90", + step_type="visual", + required=True, + blocking=risk_level in ["high", "critical"], + ) + ) + + return ValidationStrategy( + risk_level=risk_level, + project_type="html_css", + steps=steps, + test_types_required=["visual"] if risk_level != "trivial" else [], + reasoning="HTML/CSS project requires visual verification and accessibility checks.", + ) + + def _strategy_for_spa( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for Single Page Applications (React, Vue, Angular). + + Focus on component tests and E2E testing. + """ + steps = [] + + # Unit/component tests for all non-trivial + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Unit/Component Tests", + command="npm test", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + # E2E tests for medium+ risk + if risk_level in ["medium", "high", "critical"]: + steps.append( + ValidationStep( + name="E2E Tests", + command="npx playwright test", + expected_outcome="All E2E tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + # Browser console check + steps.append( + ValidationStep( + name="Console Error Check", + command="npm run dev & sleep 5 && npx playwright test --grep 'no-console-errors'", + expected_outcome="No console errors in browser", + step_type="test", + required=True, + blocking=risk_level in ["high", "critical"], + ) + ) + + # Determine test types + test_types = ["unit"] + if risk_level in ["medium", "high", "critical"]: + test_types.append("integration") + if risk_level in ["high", "critical"]: + test_types.append("e2e") + + return ValidationStrategy( + risk_level=risk_level, + project_type="spa", + steps=steps, + test_types_required=test_types, + reasoning="SPA requires component tests for logic and E2E for user flows.", + ) + + def _strategy_for_fullstack( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for fullstack frameworks (Next.js, Rails, Django). + + Focus on API tests, frontend tests, and integration. + """ + steps = [] + + # Unit tests + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Unit Tests", + command="npm test", + expected_outcome="All unit tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + # API tests for medium+ risk + if risk_level in ["medium", "high", "critical"]: + steps.append( + ValidationStep( + name="API Integration Tests", + command="npm run test:api", + expected_outcome="All API tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + # E2E tests for high+ risk + if risk_level in ["high", "critical"]: + steps.append( + ValidationStep( + name="E2E Tests", + command="npm run test:e2e", + expected_outcome="All E2E tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + # Database migration check + steps.append( + ValidationStep( + name="Database Migration Check", + command="npm run db:migrate:status", + expected_outcome="All migrations applied successfully", + step_type="api", + required=risk_level in ["medium", "high", "critical"], + blocking=True, + ) + ) + + # Determine test types + test_types = ["unit"] + if risk_level in ["medium", "high", "critical"]: + test_types.append("integration") + if risk_level in ["high", "critical"]: + test_types.append("e2e") + + return ValidationStrategy( + risk_level=risk_level, + project_type="fullstack", + steps=steps, + test_types_required=test_types, + reasoning="Fullstack requires API tests, frontend tests, and DB migration checks.", + ) + + def _strategy_for_nodejs( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for Node.js backend projects. + """ + steps = [] + + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Unit Tests", + command="npm test", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + if risk_level in ["medium", "high", "critical"]: + steps.append( + ValidationStep( + name="Integration Tests", + command="npm run test:integration", + expected_outcome="All integration tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + test_types = ["unit"] + if risk_level in ["medium", "high", "critical"]: + test_types.append("integration") + + return ValidationStrategy( + risk_level=risk_level, + project_type="nodejs", + steps=steps, + test_types_required=test_types, + reasoning="Node.js backend requires unit and integration tests.", + ) + + def _strategy_for_python_api( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for Python API projects (FastAPI, Flask, Django). + """ + steps = [] + + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Unit Tests", + command="pytest tests/ -v", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + if risk_level in ["medium", "high", "critical"]: + steps.append( + ValidationStep( + name="API Tests", + command="pytest tests/api/ -v", + expected_outcome="All API tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + steps.append( + ValidationStep( + name="Coverage Check", + command="pytest --cov=src --cov-report=term-missing", + expected_outcome="Coverage >= 80%", + step_type="test", + required=True, + blocking=risk_level == "critical", + ) + ) + + if risk_level in ["high", "critical"]: + steps.append( + ValidationStep( + name="Database Migration Check", + command="alembic current && alembic check", + expected_outcome="Migrations are current and valid", + step_type="api", + required=True, + blocking=True, + ) + ) + + test_types = ["unit"] + if risk_level in ["medium", "high", "critical"]: + test_types.append("integration") + if risk_level in ["high", "critical"]: + test_types.append("e2e") + + return ValidationStrategy( + risk_level=risk_level, + project_type="python_api", + steps=steps, + test_types_required=test_types, + reasoning="Python API requires pytest tests and migration checks.", + ) + + def _strategy_for_cli( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for CLI tools. + """ + steps = [] + + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Unit Tests", + command="pytest tests/ -v", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + steps.append( + ValidationStep( + name="CLI Help Check", + command="python -m module_name --help", + expected_outcome="Help text displays without errors", + step_type="test", + required=True, + blocking=True, + ) + ) + + if risk_level in ["medium", "high", "critical"]: + steps.append( + ValidationStep( + name="CLI Output Verification", + command="python -m module_name --version", + expected_outcome="Version displays correctly", + step_type="test", + required=True, + blocking=False, + ) + ) + + return ValidationStrategy( + risk_level=risk_level, + project_type="python_cli", + steps=steps, + test_types_required=["unit"], + reasoning="CLI tools require output verification and unit tests.", + ) + + def _strategy_for_python( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for generic Python projects. + """ + steps = [] + + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Unit Tests", + command="pytest tests/ -v", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + test_types = ["unit"] + if risk_level in ["medium", "high", "critical"]: + test_types.append("integration") + + return ValidationStrategy( + risk_level=risk_level, + project_type="python", + steps=steps, + test_types_required=test_types, + reasoning="Python project requires pytest unit tests.", + ) + + def _strategy_for_rust( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for Rust projects. + """ + steps = [] + + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Cargo Test", + command="cargo test", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + steps.append( + ValidationStep( + name="Cargo Clippy", + command="cargo clippy -- -D warnings", + expected_outcome="No clippy warnings", + step_type="test", + required=True, + blocking=risk_level in ["high", "critical"], + ) + ) + + return ValidationStrategy( + risk_level=risk_level, + project_type="rust", + steps=steps, + test_types_required=["unit"], + reasoning="Rust project requires cargo test and clippy checks.", + ) + + def _strategy_for_go( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for Go projects. + """ + steps = [] + + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Go Test", + command="go test ./...", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + steps.append( + ValidationStep( + name="Go Vet", + command="go vet ./...", + expected_outcome="No issues found", + step_type="test", + required=True, + blocking=risk_level in ["high", "critical"], + ) + ) + + return ValidationStrategy( + risk_level=risk_level, + project_type="go", + steps=steps, + test_types_required=["unit"], + reasoning="Go project requires go test and vet checks.", + ) + + def _strategy_for_ruby( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for Ruby projects. + """ + steps = [] + + if risk_level != "trivial": + steps.append( + ValidationStep( + name="RSpec Tests", + command="bundle exec rspec", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + return ValidationStrategy( + risk_level=risk_level, + project_type="ruby", + steps=steps, + test_types_required=["unit"], + reasoning="Ruby project requires RSpec tests.", + ) + + def _strategy_for_electron( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Validation strategy for Electron desktop applications. + + Focus on main/renderer process tests, E2E testing, and app packaging. + """ + steps = [] + + # Unit tests for all non-trivial + if risk_level != "trivial": + steps.append( + ValidationStep( + name="Unit Tests", + command="npm test", + expected_outcome="All tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + # E2E tests for medium+ risk (Electron apps need GUI testing) + if risk_level in ["medium", "high", "critical"]: + steps.append( + ValidationStep( + name="E2E Tests", + command="npm run test:e2e", + expected_outcome="All E2E tests pass", + step_type="test", + required=True, + blocking=True, + ) + ) + + # App build/package verification for medium+ risk + if risk_level in ["medium", "high", "critical"]: + steps.append( + ValidationStep( + name="Build Verification", + command="npm run build", + expected_outcome="App builds without errors", + step_type="test", + required=True, + blocking=True, + ) + ) + + # Console error check for high+ risk + if risk_level in ["high", "critical"]: + steps.append( + ValidationStep( + name="Console Error Check", + command="npm run test:console", + expected_outcome="No console errors in main or renderer process", + step_type="test", + required=True, + blocking=True, + ) + ) + + # Determine test types + test_types = ["unit"] + if risk_level in ["medium", "high", "critical"]: + test_types.append("integration") + test_types.append("e2e") + + return ValidationStrategy( + risk_level=risk_level, + project_type="electron", + steps=steps, + test_types_required=test_types, + reasoning="Electron app requires unit tests, E2E tests for GUI, and build verification.", + ) + + def _strategy_default( + self, project_dir: Path, risk_level: str + ) -> ValidationStrategy: + """ + Default validation strategy for unknown project types. + """ + steps = [ + ValidationStep( + name="Manual Verification", + command="manual", + expected_outcome="Code changes reviewed and tested manually", + step_type="manual", + required=True, + blocking=True, + ), + ] + + return ValidationStrategy( + risk_level=risk_level, + project_type="unknown", + steps=steps, + test_types_required=[], + reasoning="Unknown project type - manual verification required.", + ) + + def _add_security_steps( + self, strategy: ValidationStrategy, project_type: str + ) -> ValidationStrategy: + """ + Add security scanning steps to a strategy. + """ + security_steps = [] + + # Secrets scanning (always for high+ risk) + security_steps.append( + ValidationStep( + name="Secrets Scan", + command="python auto-claude/scan_secrets.py --all-files --json", + expected_outcome="No secrets detected", + step_type="security", + required=True, + blocking=True, + ) + ) + + # Language-specific SAST + if project_type in ["python", "python_api", "python_cli"]: + security_steps.append( + ValidationStep( + name="Bandit Security Scan", + command="bandit -r src/ -f json", + expected_outcome="No high severity issues", + step_type="security", + required=True, + blocking=True, + ) + ) + + if project_type in ["nodejs", "react_spa", "vue_spa", "nextjs"]: + security_steps.append( + ValidationStep( + name="npm audit", + command="npm audit --json", + expected_outcome="No critical vulnerabilities", + step_type="security", + required=True, + blocking=True, + ) + ) + + strategy.steps.extend(security_steps) + strategy.security_scan_required = True + + return strategy + + def to_dict(self, strategy: ValidationStrategy) -> dict[str, Any]: + """ + Convert a ValidationStrategy to a dictionary for JSON serialization. + """ + return { + "risk_level": strategy.risk_level, + "project_type": strategy.project_type, + "skip_validation": strategy.skip_validation, + "test_types_required": strategy.test_types_required, + "security_scan_required": strategy.security_scan_required, + "staging_deployment_required": strategy.staging_deployment_required, + "reasoning": strategy.reasoning, + "steps": [ + { + "name": step.name, + "command": step.command, + "expected_outcome": step.expected_outcome, + "type": step.step_type, + "required": step.required, + "blocking": step.blocking, + } + for step in strategy.steps + ], + } + + +# ============================================================================= +# CONVENIENCE FUNCTIONS +# ============================================================================= + + +def build_validation_strategy( + project_dir: Path, + spec_dir: Path, + risk_level: str | None = None, +) -> ValidationStrategy: + """ + Convenience function to build a validation strategy. + + Args: + project_dir: Path to project root + spec_dir: Path to spec directory + risk_level: Optional override for risk level + + Returns: + ValidationStrategy object + """ + builder = ValidationStrategyBuilder() + return builder.build_strategy(project_dir, spec_dir, risk_level) + + +def get_strategy_as_dict( + project_dir: Path, + spec_dir: Path, + risk_level: str | None = None, +) -> dict[str, Any]: + """ + Get validation strategy as a dictionary. + + Args: + project_dir: Path to project root + spec_dir: Path to spec directory + risk_level: Optional override for risk level + + Returns: + Dictionary representation of strategy + """ + builder = ValidationStrategyBuilder() + strategy = builder.build_strategy(project_dir, spec_dir, risk_level) + return builder.to_dict(strategy) + + +# ============================================================================= +# CLI +# ============================================================================= + + +def main() -> None: + """CLI entry point for testing.""" + import argparse + + parser = argparse.ArgumentParser(description="Build validation strategy") + parser.add_argument("project_dir", type=Path, help="Path to project root") + parser.add_argument("--spec-dir", type=Path, help="Path to spec directory") + parser.add_argument("--risk-level", type=str, help="Override risk level") + parser.add_argument("--json", action="store_true", help="Output as JSON") + + args = parser.parse_args() + + spec_dir = args.spec_dir or args.project_dir + builder = ValidationStrategyBuilder() + strategy = builder.build_strategy(args.project_dir, spec_dir, args.risk_level) + + if args.json: + print(json.dumps(builder.to_dict(strategy), indent=2)) + else: + print(f"Project Type: {strategy.project_type}") + print(f"Risk Level: {strategy.risk_level}") + print(f"Skip Validation: {strategy.skip_validation}") + print(f"Test Types: {', '.join(strategy.test_types_required)}") + print(f"Security Scan: {strategy.security_scan_required}") + print(f"Reasoning: {strategy.reasoning}") + print(f"\nValidation Steps ({len(strategy.steps)}):") + for i, step in enumerate(strategy.steps, 1): + print(f" {i}. {step.name}") + print(f" Command: {step.command}") + print(f" Expected: {step.expected_outcome}") + + +if __name__ == "__main__": + main() diff --git a/auto-claude/task_logger.py.backup b/auto-claude/task_logger.py.backup deleted file mode 100644 index 1a827c35..00000000 --- a/auto-claude/task_logger.py.backup +++ /dev/null @@ -1,818 +0,0 @@ -""" -Task Logger -============ - -Persistent logging system for Auto Claude tasks. -Logs are organized by phase (planning, coding, validation) and stored in the spec directory. - -Key features: -- Phase-based log organization (collapsible in UI) -- Streaming markers for real-time UI updates (similar to insights_runner.py) -- Persistent storage in JSON format for easy frontend consumption -- Tool usage tracking with start/end markers -""" - -import json -import sys -from dataclasses import asdict, dataclass -from datetime import datetime, timezone -from enum import Enum -from pathlib import Path - - -class LogPhase(str, Enum): - """Log phases matching the execution flow.""" - - PLANNING = "planning" - CODING = "coding" - VALIDATION = "validation" - - -class LogEntryType(str, Enum): - """Types of log entries.""" - - TEXT = "text" - TOOL_START = "tool_start" - TOOL_END = "tool_end" - PHASE_START = "phase_start" - PHASE_END = "phase_end" - ERROR = "error" - SUCCESS = "success" - INFO = "info" - - -@dataclass -class LogEntry: - """A single log entry.""" - - timestamp: str - type: str - content: str - phase: str - tool_name: str | None = None - tool_input: str | None = None - subtask_id: str | None = None - session: int | None = None - # New fields for expandable detail view - detail: str | None = ( - None # Full content that can be expanded (e.g., file contents, command output) - ) - subphase: str | None = ( - None # Subphase grouping (e.g., "PROJECT DISCOVERY", "CONTEXT GATHERING") - ) - collapsed: bool | None = None # Whether to show collapsed by default in UI - - def to_dict(self) -> dict: - """Convert to dictionary, excluding None values.""" - return {k: v for k, v in asdict(self).items() if v is not None} - - -@dataclass -class PhaseLog: - """Logs for a single phase.""" - - phase: str - status: str # "pending", "active", "completed", "failed" - started_at: str | None = None - completed_at: str | None = None - entries: list = None - - def __post_init__(self): - if self.entries is None: - self.entries = [] - - def to_dict(self) -> dict: - return { - "phase": self.phase, - "status": self.status, - "started_at": self.started_at, - "completed_at": self.completed_at, - "entries": self.entries, - } - - -class TaskLogger: - """ - Logger for a specific task/spec. - - Handles persistent storage of logs and emits streaming markers - for real-time UI updates. - - Usage: - logger = TaskLogger(spec_dir) - logger.start_phase(LogPhase.CODING) - logger.log("Starting implementation...") - logger.tool_start("Read", "/path/to/file.py") - logger.tool_end("Read") - logger.log("File read complete") - logger.end_phase(LogPhase.CODING, success=True) - """ - - LOG_FILE = "task_logs.json" - - def __init__(self, spec_dir: Path, emit_markers: bool = True): - """ - Initialize the task logger. - - Args: - spec_dir: Path to the spec directory - emit_markers: Whether to emit streaming markers to stdout - """ - self.spec_dir = Path(spec_dir) - self.log_file = self.spec_dir / self.LOG_FILE - self.emit_markers = emit_markers - self.current_phase: LogPhase | None = None - self.current_session: int | None = None - self.current_subtask: str | None = None - self._data: dict = self._load_or_create() - - def _load_or_create(self) -> dict: - """Load existing logs or create new structure.""" - if self.log_file.exists(): - try: - with open(self.log_file) as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - pass - - return { - "spec_id": self.spec_dir.name, - "created_at": self._timestamp(), - "updated_at": self._timestamp(), - "phases": { - LogPhase.PLANNING.value: { - "phase": LogPhase.PLANNING.value, - "status": "pending", - "started_at": None, - "completed_at": None, - "entries": [], - }, - LogPhase.CODING.value: { - "phase": LogPhase.CODING.value, - "status": "pending", - "started_at": None, - "completed_at": None, - "entries": [], - }, - LogPhase.VALIDATION.value: { - "phase": LogPhase.VALIDATION.value, - "status": "pending", - "started_at": None, - "completed_at": None, - "entries": [], - }, - }, - } - - def _save(self): - """Save logs to file.""" - self._data["updated_at"] = self._timestamp() - try: - self.spec_dir.mkdir(parents=True, exist_ok=True) - with open(self.log_file, "w") as f: - json.dump(self._data, f, indent=2) - except OSError as e: - print(f"Warning: Failed to save task logs: {e}", file=sys.stderr) - - def _timestamp(self) -> str: - """Get current timestamp in ISO format.""" - return datetime.now(timezone.utc).isoformat() - - def _emit(self, marker_type: str, data: dict): - """Emit a streaming marker to stdout for UI consumption.""" - if not self.emit_markers: - return - try: - marker = f"__TASK_LOG_{marker_type.upper()}__:{json.dumps(data)}" - print(marker, flush=True) - except Exception: - pass # Don't let marker emission break logging - - def _add_entry(self, entry: LogEntry): - """Add an entry to the current phase.""" - phase_key = entry.phase - if phase_key not in self._data["phases"]: - # Create phase if it doesn't exist - self._data["phases"][phase_key] = { - "phase": phase_key, - "status": "active", - "started_at": self._timestamp(), - "completed_at": None, - "entries": [], - } - - self._data["phases"][phase_key]["entries"].append(entry.to_dict()) - self._save() - - def set_session(self, session: int): - """Set the current session number.""" - self.current_session = session - - def set_subtask(self, subtask_id: str | None): - """Set the current subtask being processed.""" - self.current_subtask = subtask_id - - def start_phase(self, phase: LogPhase, message: str | None = None): - """ - Start a new phase, auto-closing any stale active phases. - - This handles restart/recovery scenarios where a previous run was interrupted - before properly closing a phase. When starting a new phase, any other phases - that are still marked as "active" will be auto-closed. - - Args: - phase: The phase to start - message: Optional message to log at phase start - """ - self.current_phase = phase - phase_key = phase.value - - # Auto-close any other active phases (handles restart/recovery scenarios) - for other_phase_key, phase_data in self._data["phases"].items(): - if other_phase_key != phase_key and phase_data.get("status") == "active": - # Auto-close stale phase from previous interrupted run - phase_data["status"] = "completed" - phase_data["completed_at"] = self._timestamp() - # Add a log entry noting the auto-close - auto_close_entry = LogEntry( - timestamp=self._timestamp(), - type=LogEntryType.PHASE_END.value, - content=f"{other_phase_key} phase auto-closed on resume", - phase=other_phase_key, - session=self.current_session, - ) - self._data["phases"][other_phase_key]["entries"].append( - auto_close_entry.to_dict() - ) - - # Update phase status - if phase_key in self._data["phases"]: - self._data["phases"][phase_key]["status"] = "active" - self._data["phases"][phase_key]["started_at"] = self._timestamp() - - # Emit marker for UI - self._emit("PHASE_START", {"phase": phase_key, "timestamp": self._timestamp()}) - - # Add phase start entry - entry = LogEntry( - timestamp=self._timestamp(), - type=LogEntryType.PHASE_START.value, - content=message or f"Starting {phase_key} phase", - phase=phase_key, - session=self.current_session, - ) - self._add_entry(entry) - - # Also print the message - if message: - print(message, flush=True) - - def end_phase( - self, phase: LogPhase, success: bool = True, message: str | None = None - ): - """ - End a phase. - - Args: - phase: The phase to end - success: Whether the phase completed successfully - message: Optional message to log at phase end - """ - phase_key = phase.value - - # Update phase status - if phase_key in self._data["phases"]: - self._data["phases"][phase_key]["status"] = ( - "completed" if success else "failed" - ) - self._data["phases"][phase_key]["completed_at"] = self._timestamp() - - # Emit marker for UI - self._emit( - "PHASE_END", - {"phase": phase_key, "success": success, "timestamp": self._timestamp()}, - ) - - # Add phase end entry - entry = LogEntry( - timestamp=self._timestamp(), - type=LogEntryType.PHASE_END.value, - content=message - or f"{'Completed' if success else 'Failed'} {phase_key} phase", - phase=phase_key, - session=self.current_session, - ) - self._add_entry(entry) - - if message: - print(message, flush=True) - - if phase == self.current_phase: - self.current_phase = None - - self._save() - - def log( - self, - content: str, - entry_type: LogEntryType = LogEntryType.TEXT, - phase: LogPhase | None = None, - print_to_console: bool = True, - ): - """ - Log a message. - - Args: - content: The message to log - entry_type: Type of entry (text, error, success, info) - phase: Optional phase override (uses current_phase if not specified) - print_to_console: Whether to also print to stdout (default True) - """ - phase_key = (phase or self.current_phase or LogPhase.CODING).value - - entry = LogEntry( - timestamp=self._timestamp(), - type=entry_type.value, - content=content, - phase=phase_key, - subtask_id=self.current_subtask, - session=self.current_session, - ) - self._add_entry(entry) - - # Emit streaming marker - self._emit( - "TEXT", - { - "content": content, - "phase": phase_key, - "type": entry_type.value, - "subtask_id": self.current_subtask, - "timestamp": self._timestamp(), - }, - ) - - # Also print to console (unless caller handles printing) - if print_to_console: - print(content, flush=True) - - def log_error(self, content: str, phase: LogPhase | None = None): - """Log an error message.""" - self.log(content, LogEntryType.ERROR, phase) - - def log_success(self, content: str, phase: LogPhase | None = None): - """Log a success message.""" - self.log(content, LogEntryType.SUCCESS, phase) - - def log_info(self, content: str, phase: LogPhase | None = None): - """Log an info message.""" - self.log(content, LogEntryType.INFO, phase) - - def log_with_detail( - self, - content: str, - detail: str, - entry_type: LogEntryType = LogEntryType.TEXT, - phase: LogPhase | None = None, - subphase: str | None = None, - collapsed: bool = True, - print_to_console: bool = True, - ): - """ - Log a message with expandable detail content. - - Args: - content: Brief summary shown by default - detail: Full content shown when expanded (e.g., file contents, command output) - entry_type: Type of entry (text, error, success, info) - phase: Optional phase override - subphase: Optional subphase grouping (e.g., "PROJECT DISCOVERY") - collapsed: Whether detail should be collapsed by default (default True) - print_to_console: Whether to print summary to stdout (default True) - """ - phase_key = (phase or self.current_phase or LogPhase.CODING).value - - entry = LogEntry( - timestamp=self._timestamp(), - type=entry_type.value, - content=content, - phase=phase_key, - subtask_id=self.current_subtask, - session=self.current_session, - detail=detail, - subphase=subphase, - collapsed=collapsed, - ) - self._add_entry(entry) - - # Emit streaming marker with detail indicator - self._emit( - "TEXT", - { - "content": content, - "phase": phase_key, - "type": entry_type.value, - "subtask_id": self.current_subtask, - "timestamp": self._timestamp(), - "has_detail": True, - "subphase": subphase, - }, - ) - - if print_to_console: - print(content, flush=True) - - def start_subphase( - self, - subphase: str, - phase: LogPhase | None = None, - print_to_console: bool = True, - ): - """ - Mark the start of a subphase within the current phase. - - Args: - subphase: Name of the subphase (e.g., "PROJECT DISCOVERY", "CONTEXT GATHERING") - phase: Optional phase override - print_to_console: Whether to print to stdout - """ - phase_key = (phase or self.current_phase or LogPhase.CODING).value - - entry = LogEntry( - timestamp=self._timestamp(), - type=LogEntryType.INFO.value, - content=f"Starting {subphase}", - phase=phase_key, - subtask_id=self.current_subtask, - session=self.current_session, - subphase=subphase, - ) - self._add_entry(entry) - - # Emit streaming marker - self._emit( - "SUBPHASE_START", - {"subphase": subphase, "phase": phase_key, "timestamp": self._timestamp()}, - ) - - if print_to_console: - print(f"\n--- {subphase} ---", flush=True) - - def tool_start( - self, - tool_name: str, - tool_input: str | None = None, - phase: LogPhase | None = None, - print_to_console: bool = True, - ): - """ - Log the start of a tool execution. - - Args: - tool_name: Name of the tool (e.g., "Read", "Write", "Bash") - tool_input: Brief description of tool input - phase: Optional phase override - print_to_console: Whether to also print to stdout (default True) - """ - phase_key = (phase or self.current_phase or LogPhase.CODING).value - - # Truncate long inputs for display - display_input = tool_input - if display_input and len(display_input) > 100: - display_input = display_input[:97] + "..." - - entry = LogEntry( - timestamp=self._timestamp(), - type=LogEntryType.TOOL_START.value, - content=f"[{tool_name}] {display_input or ''}".strip(), - phase=phase_key, - tool_name=tool_name, - tool_input=display_input, - subtask_id=self.current_subtask, - session=self.current_session, - ) - self._add_entry(entry) - - # Emit streaming marker (same format as insights_runner.py) - self._emit( - "TOOL_START", - {"name": tool_name, "input": display_input, "phase": phase_key}, - ) - - if print_to_console: - print(f"\n[Tool: {tool_name}]", flush=True) - - def tool_end( - self, - tool_name: str, - success: bool = True, - result: str | None = None, - detail: str | None = None, - phase: LogPhase | None = None, - print_to_console: bool = False, - ): - """ - Log the end of a tool execution. - - Args: - tool_name: Name of the tool - success: Whether the tool succeeded - result: Optional brief result description (shown in summary) - detail: Optional full result content (expandable in UI, e.g., file contents, command output) - phase: Optional phase override - print_to_console: Whether to also print to stdout (default False for tool_end) - """ - phase_key = (phase or self.current_phase or LogPhase.CODING).value - - # Truncate long results for display - display_result = result - if display_result and len(display_result) > 100: - display_result = display_result[:97] + "..." - - status = "Done" if success else "Error" - content = f"[{tool_name}] {status}" - if display_result: - content += f": {display_result}" - - # Truncate detail for storage (max 10KB to avoid bloating JSON) - stored_detail = detail - if stored_detail and len(stored_detail) > 10240: - stored_detail = ( - stored_detail[:10240] - + f"\n\n... [truncated - full output was {len(detail)} chars]" - ) - - entry = LogEntry( - timestamp=self._timestamp(), - type=LogEntryType.TOOL_END.value, - content=content, - phase=phase_key, - tool_name=tool_name, - subtask_id=self.current_subtask, - session=self.current_session, - detail=stored_detail, - collapsed=True, - ) - self._add_entry(entry) - - # Emit streaming marker - self._emit( - "TOOL_END", - { - "name": tool_name, - "success": success, - "phase": phase_key, - "has_detail": detail is not None, - }, - ) - - if print_to_console: - if result: - print(f" [{status}] {display_result}", flush=True) - else: - print(f" [{status}]", flush=True) - - def get_logs(self) -> dict: - """Get all logs.""" - return self._data - - def get_phase_logs(self, phase: LogPhase) -> dict: - """Get logs for a specific phase.""" - return self._data["phases"].get(phase.value, {}) - - def clear(self): - """Clear all logs (useful for testing).""" - self._data = self._load_or_create() - self._save() - - -def load_task_logs(spec_dir: Path) -> dict | None: - """ - Load task logs from a spec directory. - - Args: - spec_dir: Path to the spec directory - - Returns: - Logs dictionary or None if not found - """ - log_file = spec_dir / TaskLogger.LOG_FILE - if not log_file.exists(): - return None - - try: - with open(log_file) as f: - return json.load(f) - except (OSError, json.JSONDecodeError): - return None - - -def get_active_phase(spec_dir: Path) -> str | None: - """ - Get the currently active phase for a spec. - - Args: - spec_dir: Path to the spec directory - - Returns: - Phase name or None if no active phase - """ - logs = load_task_logs(spec_dir) - if not logs: - return None - - for phase_name, phase_data in logs.get("phases", {}).items(): - if phase_data.get("status") == "active": - return phase_name - - return None - - -# Global logger instance for easy access -_current_logger: TaskLogger | None = None - - -def get_task_logger( - spec_dir: Path | None = None, emit_markers: bool = True -) -> TaskLogger | None: - """ - Get or create a task logger for the given spec directory. - - Args: - spec_dir: Path to the spec directory (creates new logger if different from current) - emit_markers: Whether to emit streaming markers - - Returns: - TaskLogger instance or None if no spec_dir - """ - global _current_logger - - if spec_dir is None: - return _current_logger - - if _current_logger is None or _current_logger.spec_dir != spec_dir: - _current_logger = TaskLogger(spec_dir, emit_markers) - - return _current_logger - - -def clear_task_logger(): - """Clear the global task logger.""" - global _current_logger - _current_logger = None - - -def update_task_logger_path(new_spec_dir: Path): - """ - Update the global task logger's spec directory after a rename. - - This should be called after renaming a spec directory to ensure - the logger continues writing to the correct location. - - Args: - new_spec_dir: The new path to the spec directory - """ - global _current_logger - - if _current_logger is None: - return - - # Update the logger's internal paths - _current_logger.spec_dir = Path(new_spec_dir) - _current_logger.log_file = _current_logger.spec_dir / TaskLogger.LOG_FILE - - # Update spec_id in the data - _current_logger._data["spec_id"] = new_spec_dir.name - - # Save to the new location - _current_logger._save() - - -class StreamingLogCapture: - """ - Context manager to capture streaming output and log it. - - Usage: - with StreamingLogCapture(logger, phase) as capture: - # Run agent session - async for msg in client.receive_response(): - capture.process_message(msg) - """ - - def __init__(self, logger: TaskLogger, phase: LogPhase | None = None): - self.logger = logger - self.phase = phase - self.current_tool: str | None = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - # End any active tool - if self.current_tool: - self.logger.tool_end( - self.current_tool, success=exc_type is None, phase=self.phase - ) - self.current_tool = None - return False - - def process_text(self, text: str): - """Process text output from the agent.""" - if text.strip(): - self.logger.log(text, phase=self.phase) - - def process_tool_start(self, tool_name: str, tool_input: str | None = None): - """Process tool start.""" - # End previous tool if any - if self.current_tool: - self.logger.tool_end(self.current_tool, success=True, phase=self.phase) - - self.current_tool = tool_name - self.logger.tool_start(tool_name, tool_input, phase=self.phase) - - def process_tool_end( - self, - tool_name: str, - success: bool = True, - result: str | None = None, - detail: str | None = None, - ): - """Process tool end.""" - self.logger.tool_end( - tool_name, success, result, detail=detail, phase=self.phase - ) - if self.current_tool == tool_name: - self.current_tool = None - - def process_message(self, msg, verbose: bool = False, capture_detail: bool = True): - """ - Process a message from the Claude SDK stream. - - Args: - msg: Message from client.receive_response() - verbose: Whether to show detailed tool results - capture_detail: Whether to capture full tool output for expandable detail view - """ - msg_type = type(msg).__name__ - - if msg_type == "AssistantMessage" and hasattr(msg, "content"): - for block in msg.content: - block_type = type(block).__name__ - - if block_type == "TextBlock" and hasattr(block, "text"): - # Text is already logged by the agent session - pass - elif block_type == "ToolUseBlock" and hasattr(block, "name"): - tool_input = None - if hasattr(block, "input") and block.input: - inp = block.input - if isinstance(inp, dict): - # Extract meaningful input description - if "pattern" in inp: - tool_input = f"pattern: {inp['pattern']}" - elif "file_path" in inp: - fp = inp["file_path"] - if len(fp) > 50: - fp = "..." + fp[-47:] - tool_input = fp - elif "command" in inp: - cmd = inp["command"] - if len(cmd) > 50: - cmd = cmd[:47] + "..." - tool_input = cmd - elif "path" in inp: - tool_input = inp["path"] - self.process_tool_start(block.name, tool_input) - - elif msg_type == "UserMessage" and hasattr(msg, "content"): - for block in msg.content: - block_type = type(block).__name__ - - if block_type == "ToolResultBlock": - is_error = getattr(block, "is_error", False) - result_content = getattr(block, "content", "") - - if self.current_tool: - result_str = None - if verbose and result_content: - result_str = str(result_content)[:100] - - # Capture full detail for expandable view - detail_content = None - if capture_detail and self.current_tool in ( - "Read", - "Grep", - "Bash", - "Edit", - "Write", - ): - full_result = str(result_content) - if len(full_result) < 50000: # 50KB max - detail_content = full_result - - self.process_tool_end( - self.current_tool, - success=not is_error, - result=result_str, - detail=detail_content, - ) diff --git a/auto-claude/task_logger.py b/auto-claude/task_logger/main.py similarity index 100% rename from auto-claude/task_logger.py rename to auto-claude/task_logger/main.py diff --git a/auto-claude/test_discovery.py b/auto-claude/test_discovery.py index ffed6307..e4a45a7a 100644 --- a/auto-claude/test_discovery.py +++ b/auto-claude/test_discovery.py @@ -1,682 +1,20 @@ -#!/usr/bin/env python3 -""" -Test Discovery Module -===================== - -Detects test frameworks, test commands, and test directories in a project. -This module analyzes project configuration files to discover how tests -should be run. - -The test discovery results are used by: -- QA Agent: To determine what test commands to run -- Test Creator: To know what framework to use when creating tests -- Planner: To include correct test commands in verification strategy - -Usage: - from test_discovery import TestDiscovery - - discovery = TestDiscovery() - result = discovery.discover(project_dir) - - print(f"Test frameworks: {result['frameworks']}") - print(f"Test command: {result['test_command']}") -""" - -import json -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -# ============================================================================= -# DATA CLASSES -# ============================================================================= - - -@dataclass -class TestFramework: - """ - Represents a detected test framework. - - Attributes: - name: Name of the framework (e.g., "pytest", "jest", "vitest") - type: Type of testing (unit, integration, e2e, all) - command: Command to run tests - config_file: Configuration file if found - version: Version if detected - coverage_command: Command for coverage if available - """ - - name: str - type: str # unit, integration, e2e, all - command: str - config_file: str | None = None - version: str | None = None - coverage_command: str | None = None - - -@dataclass -class TestDiscoveryResult: - """ - Result of test framework discovery. - - Attributes: - frameworks: List of detected test frameworks - test_command: Primary test command to run - test_directories: Discovered test directories - package_manager: Detected package manager - has_tests: Whether any test files were found - coverage_command: Command for coverage if available - """ - - frameworks: list[TestFramework] = field(default_factory=list) - test_command: str = "" - test_directories: list[str] = field(default_factory=list) - package_manager: str = "" - has_tests: bool = False - coverage_command: str | None = None - - -# ============================================================================= -# FRAMEWORK DETECTORS -# ============================================================================= - - -# Pattern-based framework detection -FRAMEWORK_PATTERNS = { - # JavaScript/TypeScript - "jest": { - "config_files": [ - "jest.config.js", - "jest.config.ts", - "jest.config.mjs", - "jest.config.cjs", - ], - "package_key": "jest", - "type": "unit", - "command": "npx jest", - "coverage_command": "npx jest --coverage", - }, - "vitest": { - "config_files": ["vitest.config.js", "vitest.config.ts", "vitest.config.mjs"], - "package_key": "vitest", - "type": "unit", - "command": "npx vitest run", - "coverage_command": "npx vitest run --coverage", - }, - "mocha": { - "config_files": [ - ".mocharc.js", - ".mocharc.json", - ".mocharc.yaml", - ".mocharc.yml", - ], - "package_key": "mocha", - "type": "unit", - "command": "npx mocha", - "coverage_command": "npx nyc mocha", - }, - "playwright": { - "config_files": ["playwright.config.js", "playwright.config.ts"], - "package_key": "@playwright/test", - "type": "e2e", - "command": "npx playwright test", - "coverage_command": None, - }, - "cypress": { - "config_files": ["cypress.config.js", "cypress.config.ts", "cypress.json"], - "package_key": "cypress", - "type": "e2e", - "command": "npx cypress run", - "coverage_command": None, - }, - # Python - "pytest": { - "config_files": ["pytest.ini", "pyproject.toml", "setup.cfg", "conftest.py"], - "pyproject_key": "pytest", - "requirements_key": "pytest", - "type": "all", - "command": "pytest", - "coverage_command": "pytest --cov", - }, - "unittest": { - "config_files": [], - "type": "unit", - "command": "python -m unittest discover", - "coverage_command": "coverage run -m unittest discover", - }, - # Rust - "cargo_test": { - "config_files": ["Cargo.toml"], - "type": "all", - "command": "cargo test", - "coverage_command": "cargo tarpaulin", - }, - # Go - "go_test": { - "config_files": ["go.mod"], - "type": "all", - "command": "go test ./...", - "coverage_command": "go test -cover ./...", - }, - # Ruby - "rspec": { - "config_files": [".rspec", "spec/spec_helper.rb"], - "gemfile_key": "rspec", - "type": "all", - "command": "bundle exec rspec", - "coverage_command": "bundle exec rspec --format documentation", - }, - "minitest": { - "config_files": [], - "gemfile_key": "minitest", - "type": "unit", - "command": "bundle exec rake test", - "coverage_command": None, - }, -} - - -# ============================================================================= -# TEST DISCOVERY -# ============================================================================= - - -class TestDiscovery: - """ - Discovers test frameworks and configurations in a project. - - Analyzes: - - Package files (package.json, pyproject.toml, Cargo.toml, etc.) - - Configuration files (jest.config.js, pytest.ini, etc.) - - Directory structure (tests/, spec/, __tests__/) - """ - - def __init__(self) -> None: - """Initialize the test discovery.""" - self._cache: dict[str, TestDiscoveryResult] = {} - - def discover(self, project_dir: Path) -> TestDiscoveryResult: - """ - Discover test frameworks and configuration in the project. - - Args: - project_dir: Path to the project root - - Returns: - TestDiscoveryResult with detected frameworks and commands - """ - project_dir = Path(project_dir) - cache_key = str(project_dir.resolve()) - - if cache_key in self._cache: - return self._cache[cache_key] - - result = TestDiscoveryResult() - - # Detect package manager - result.package_manager = self._detect_package_manager(project_dir) - - # Discover frameworks based on project type - if (project_dir / "package.json").exists(): - self._discover_js_frameworks(project_dir, result) - - # Check for Python project indicators - python_indicators = [ - project_dir / "pyproject.toml", - project_dir / "requirements.txt", - project_dir / "setup.py", - project_dir / "pytest.ini", - project_dir / "conftest.py", - project_dir / "tests" / "conftest.py", - ] - if any(p.exists() for p in python_indicators): - self._discover_python_frameworks(project_dir, result) - - if (project_dir / "Cargo.toml").exists(): - self._discover_rust_frameworks(project_dir, result) - if (project_dir / "go.mod").exists(): - self._discover_go_frameworks(project_dir, result) - if (project_dir / "Gemfile").exists(): - self._discover_ruby_frameworks(project_dir, result) - - # Find test directories - result.test_directories = self._find_test_directories(project_dir) - - # Check if tests exist - result.has_tests = self._has_test_files(project_dir, result.test_directories) - - # Set primary test command - if result.frameworks: - result.test_command = result.frameworks[0].command - - # Set coverage command from first framework that has one - if not result.coverage_command: - for framework in result.frameworks: - if framework.coverage_command: - result.coverage_command = framework.coverage_command - break - - self._cache[cache_key] = result - return result - - def _detect_package_manager(self, project_dir: Path) -> str: - """Detect the package manager used by the project.""" - if (project_dir / "pnpm-lock.yaml").exists(): - return "pnpm" - if (project_dir / "yarn.lock").exists(): - return "yarn" - if (project_dir / "package-lock.json").exists(): - return "npm" - if (project_dir / "bun.lockb").exists(): - return "bun" - if (project_dir / "uv.lock").exists(): - return "uv" - if (project_dir / "poetry.lock").exists(): - return "poetry" - if (project_dir / "Pipfile.lock").exists(): - return "pipenv" - if (project_dir / "Cargo.lock").exists(): - return "cargo" - if (project_dir / "go.sum").exists(): - return "go" - if (project_dir / "Gemfile.lock").exists(): - return "bundler" - return "" - - def _discover_js_frameworks( - self, project_dir: Path, result: TestDiscoveryResult - ) -> None: - """Discover JavaScript/TypeScript test frameworks.""" - package_json = project_dir / "package.json" - if not package_json.exists(): - return - - try: - with open(package_json, encoding="utf-8") as f: - pkg = json.load(f) - except (OSError, json.JSONDecodeError): - return - - deps = pkg.get("dependencies", {}) - dev_deps = pkg.get("devDependencies", {}) - all_deps = {**deps, **dev_deps} - scripts = pkg.get("scripts", {}) - - # Check for test frameworks in dependencies - for name, pattern in FRAMEWORK_PATTERNS.items(): - if "package_key" not in pattern: - continue - - if pattern["package_key"] in all_deps: - # Check for config file - config_file = None - for cf in pattern.get("config_files", []): - if (project_dir / cf).exists(): - config_file = cf - break - - # Get version - version = all_deps.get(pattern["package_key"], "") - if version.startswith("^") or version.startswith("~"): - version = version[1:] - - # Determine command - prefer npm scripts if available - command = pattern["command"] - if "test" in scripts and pattern["package_key"] in scripts.get( - "test", "" - ): - command = f"{result.package_manager or 'npm'} test" - - result.frameworks.append( - TestFramework( - name=name, - type=pattern["type"], - command=command, - config_file=config_file, - version=version, - coverage_command=pattern.get("coverage_command"), - ) - ) - - # Check npm scripts for test commands - if not result.frameworks and "test" in scripts: - test_script = scripts["test"] - if ( - test_script - and test_script != 'echo "Error: no test specified" && exit 1' - ): - # Try to infer framework from script - framework_name = "npm_test" - framework_type = "unit" - - if "jest" in test_script: - framework_name = "jest" - elif "vitest" in test_script: - framework_name = "vitest" - elif "mocha" in test_script: - framework_name = "mocha" - elif "playwright" in test_script: - framework_name = "playwright" - framework_type = "e2e" - elif "cypress" in test_script: - framework_name = "cypress" - framework_type = "e2e" - - result.frameworks.append( - TestFramework( - name=framework_name, - type=framework_type, - command=f"{result.package_manager or 'npm'} test", - config_file=None, - ) - ) - - def _discover_python_frameworks( - self, project_dir: Path, result: TestDiscoveryResult - ) -> None: - """Discover Python test frameworks.""" - # Check for pytest.ini first (explicit pytest config) - if (project_dir / "pytest.ini").exists(): - if not any(f.name == "pytest" for f in result.frameworks): - result.frameworks.append( - TestFramework( - name="pytest", - type="all", - command="pytest", - config_file="pytest.ini", - ) - ) - - # Check pyproject.toml - pyproject = project_dir / "pyproject.toml" - if pyproject.exists(): - content = pyproject.read_text() - - # Check for pytest - if "pytest" in content: - if not any(f.name == "pytest" for f in result.frameworks): - config_file = ( - "pyproject.toml" if "[tool.pytest" in content else None - ) - result.frameworks.append( - TestFramework( - name="pytest", - type="all", - command="pytest", - config_file=config_file, - ) - ) - - # Check requirements.txt - requirements = project_dir / "requirements.txt" - if requirements.exists(): - content = requirements.read_text().lower() - if "pytest" in content and not any( - f.name == "pytest" for f in result.frameworks - ): - result.frameworks.append( - TestFramework( - name="pytest", - type="all", - command="pytest", - config_file=None, - ) - ) - - # Check for conftest.py (pytest marker) - conftest_root = project_dir / "conftest.py" - conftest_tests = project_dir / "tests" / "conftest.py" - if conftest_root.exists() or conftest_tests.exists(): - if not any(f.name == "pytest" for f in result.frameworks): - result.frameworks.append( - TestFramework( - name="pytest", - type="all", - command="pytest", - config_file="conftest.py", - ) - ) - - # Fall back to unittest if test files exist but no framework detected - if not result.frameworks: - test_dirs = self._find_test_directories(project_dir) - if test_dirs: - result.frameworks.append( - TestFramework( - name="unittest", - type="unit", - command="python -m unittest discover", - config_file=None, - ) - ) - - def _discover_rust_frameworks( - self, project_dir: Path, result: TestDiscoveryResult - ) -> None: - """Discover Rust test frameworks.""" - cargo_toml = project_dir / "Cargo.toml" - if cargo_toml.exists(): - result.frameworks.append( - TestFramework( - name="cargo_test", - type="all", - command="cargo test", - config_file="Cargo.toml", - ) - ) - - def _discover_go_frameworks( - self, project_dir: Path, result: TestDiscoveryResult - ) -> None: - """Discover Go test frameworks.""" - go_mod = project_dir / "go.mod" - if go_mod.exists(): - result.frameworks.append( - TestFramework( - name="go_test", - type="all", - command="go test ./...", - config_file="go.mod", - ) - ) - - def _discover_ruby_frameworks( - self, project_dir: Path, result: TestDiscoveryResult - ) -> None: - """Discover Ruby test frameworks.""" - gemfile = project_dir / "Gemfile" - if not gemfile.exists(): - return - - content = gemfile.read_text().lower() - - if "rspec" in content or (project_dir / ".rspec").exists(): - result.frameworks.append( - TestFramework( - name="rspec", - type="all", - command="bundle exec rspec", - config_file=".rspec" if (project_dir / ".rspec").exists() else None, - ) - ) - elif "minitest" in content: - result.frameworks.append( - TestFramework( - name="minitest", - type="unit", - command="bundle exec rake test", - config_file=None, - ) - ) - - def _find_test_directories(self, project_dir: Path) -> list[str]: - """Find test directories in the project.""" - test_dir_patterns = [ - "tests", - "test", - "spec", - "__tests__", - "specs", - "test_*", - ] - - found_dirs = [] - for pattern in test_dir_patterns: - if pattern.endswith("*"): - # Glob pattern - for d in project_dir.glob(pattern): - if d.is_dir(): - found_dirs.append(str(d.relative_to(project_dir))) - else: - # Exact name - test_dir = project_dir / pattern - if test_dir.is_dir(): - found_dirs.append(pattern) - - return found_dirs - - def _has_test_files(self, project_dir: Path, test_directories: list[str]) -> bool: - """Check if any test files exist.""" - test_file_patterns = [ - "**/test_*.py", - "**/*_test.py", - "**/*.test.js", - "**/*.test.ts", - "**/*.test.tsx", - "**/*.spec.js", - "**/*.spec.ts", - "**/*.spec.tsx", - "**/test_*.go", - "**/*_test.go", - "**/*_test.rs", - "**/spec/**/*_spec.rb", - ] - - # Check in test directories - for test_dir in test_directories: - test_path = project_dir / test_dir - if test_path.exists(): - for pattern in test_file_patterns: - if list(test_path.glob(pattern.replace("**/", ""))): - return True - - # Check project-wide - for pattern in test_file_patterns: - if list(project_dir.glob(pattern)): - return True - - return False - - def to_dict(self, result: TestDiscoveryResult) -> dict[str, Any]: - """Convert result to dictionary for JSON serialization.""" - return { - "frameworks": [ - { - "name": f.name, - "type": f.type, - "command": f.command, - "config_file": f.config_file, - "version": f.version, - "coverage_command": f.coverage_command, - } - for f in result.frameworks - ], - "test_command": result.test_command, - "test_directories": result.test_directories, - "package_manager": result.package_manager, - "has_tests": result.has_tests, - "coverage_command": result.coverage_command, - } - - def clear_cache(self) -> None: - """Clear the internal cache.""" - self._cache.clear() - - -# ============================================================================= -# CONVENIENCE FUNCTIONS -# ============================================================================= - - -def discover_tests(project_dir: Path) -> TestDiscoveryResult: - """ - Convenience function to discover tests in a project. - - Args: - project_dir: Path to project root - - Returns: - TestDiscoveryResult with detected frameworks - """ - discovery = TestDiscovery() - return discovery.discover(project_dir) - - -def get_test_command(project_dir: Path) -> str: - """ - Get the primary test command for a project. - - Args: - project_dir: Path to project root - - Returns: - Test command string, or empty string if not found - """ - discovery = TestDiscovery() - result = discovery.discover(project_dir) - return result.test_command - - -def get_test_frameworks(project_dir: Path) -> list[str]: - """ - Get list of test framework names in a project. - - Args: - project_dir: Path to project root - - Returns: - List of framework names - """ - discovery = TestDiscovery() - result = discovery.discover(project_dir) - return [f.name for f in result.frameworks] - - -# ============================================================================= -# CLI -# ============================================================================= - - -def main() -> None: - """CLI entry point for testing.""" - import argparse - - parser = argparse.ArgumentParser(description="Discover test frameworks") - parser.add_argument("project_dir", type=Path, help="Path to project root") - parser.add_argument("--json", action="store_true", help="Output as JSON") - - args = parser.parse_args() - - discovery = TestDiscovery() - result = discovery.discover(args.project_dir) - - if args.json: - print(json.dumps(discovery.to_dict(result), indent=2)) - else: - print(f"Package Manager: {result.package_manager or 'unknown'}") - print(f"Has Tests: {result.has_tests}") - print(f"Test Command: {result.test_command or 'none'}") - print(f"Test Directories: {', '.join(result.test_directories) or 'none'}") - print(f"Coverage Command: {result.coverage_command or 'none'}") - print(f"\nFrameworks ({len(result.frameworks)}):") - for f in result.frameworks: - print(f" - {f.name} ({f.type})") - print(f" Command: {f.command}") - if f.config_file: - print(f" Config: {f.config_file}") - if f.version: - print(f" Version: {f.version}") - - -if __name__ == "__main__": - main() +"""Backward compatibility shim - import from analysis.test_discovery instead.""" +from analysis.test_discovery import ( + TestFramework, + TestDiscoveryResult, + TestDiscovery, + discover_tests, + get_test_command, + get_test_frameworks, + FRAMEWORK_PATTERNS, +) + +__all__ = [ + "TestFramework", + "TestDiscoveryResult", + "TestDiscovery", + "discover_tests", + "get_test_command", + "get_test_frameworks", + "FRAMEWORK_PATTERNS", +] diff --git a/auto-claude/ui.py b/auto-claude/ui/main.py similarity index 100% rename from auto-claude/ui.py rename to auto-claude/ui/main.py diff --git a/auto-claude/statusline.py b/auto-claude/ui/statusline.py similarity index 100% rename from auto-claude/statusline.py rename to auto-claude/ui/statusline.py diff --git a/auto-claude/validate_spec.py.backup b/auto-claude/validate_spec.py.backup deleted file mode 100644 index 3f56b167..00000000 --- a/auto-claude/validate_spec.py.backup +++ /dev/null @@ -1,633 +0,0 @@ -#!/usr/bin/env python3 -""" -Spec Validation System -====================== - -Validates spec outputs at each checkpoint to ensure reliability. -This is the enforcement layer that catches errors before they propagate. - -The spec creation process has mandatory checkpoints: -1. Prerequisites (project_index.json exists) -2. Context (context.json created with required fields) -3. Spec document (spec.md with required sections) -4. Implementation plan (implementation_plan.json with valid schema) - -Usage: - python auto-claude/validate_spec.py --spec-dir auto-claude/specs/001-feature/ --checkpoint prereqs - python auto-claude/validate_spec.py --spec-dir auto-claude/specs/001-feature/ --checkpoint context - python auto-claude/validate_spec.py --spec-dir auto-claude/specs/001-feature/ --checkpoint spec - python auto-claude/validate_spec.py --spec-dir auto-claude/specs/001-feature/ --checkpoint plan - python auto-claude/validate_spec.py --spec-dir auto-claude/specs/001-feature/ --checkpoint all -""" - -import json -import re -import sys -from dataclasses import dataclass -from pathlib import Path - -# JSON Schemas for validation -IMPLEMENTATION_PLAN_SCHEMA = { - "required_fields": ["feature", "workflow_type", "phases"], - "optional_fields": [ - "services_involved", - "final_acceptance", - "created_at", - "updated_at", - "spec_file", - "qa_acceptance", - "qa_signoff", - "summary", - "description", - "workflow_rationale", - "status", - ], - "workflow_types": ["feature", "refactor", "investigation", "migration", "simple"], - "phase_schema": { - # Support both old format ("phase" number) and new format ("id" string) - "required_fields_either": [["phase", "id"]], # At least one of these - "required_fields": ["name", "subtasks"], - "optional_fields": [ - "type", - "depends_on", - "parallel_safe", - "description", - "phase", - "id", - ], - "phase_types": [ - "setup", - "implementation", - "investigation", - "integration", - "cleanup", - ], - }, - "subtask_schema": { - "required_fields": ["id", "description", "status"], - "optional_fields": [ - "service", - "all_services", - "files_to_modify", - "files_to_create", - "patterns_from", - "verification", - "expected_output", - "actual_output", - "started_at", - "completed_at", - "session_id", - "critique_result", - ], - "status_values": ["pending", "in_progress", "completed", "blocked", "failed"], - }, - "verification_schema": { - "required_fields": ["type"], - "optional_fields": [ - "run", - "url", - "method", - "expect_status", - "expect_contains", - "scenario", - "steps", - ], - "verification_types": [ - "command", - "api", - "browser", - "component", - "manual", - "none", - "e2e", - ], - }, -} - -CONTEXT_SCHEMA = { - "required_fields": ["task_description"], - "optional_fields": [ - "scoped_services", - "files_to_modify", - "files_to_reference", - "patterns", - "service_contexts", - "created_at", - ], -} - -PROJECT_INDEX_SCHEMA = { - "required_fields": ["project_type"], - "optional_fields": [ - "services", - "infrastructure", - "conventions", - "root_path", - "created_at", - "git_info", - ], - "project_types": ["single", "monorepo"], -} - -SPEC_REQUIRED_SECTIONS = [ - "Overview", - "Workflow Type", - "Task Scope", - "Success Criteria", -] - -SPEC_RECOMMENDED_SECTIONS = [ - "Files to Modify", - "Files to Reference", - "Requirements", - "QA Acceptance Criteria", -] - - -@dataclass -class ValidationResult: - """Result of a validation check.""" - - valid: bool - checkpoint: str - errors: list[str] - warnings: list[str] - fixes: list[str] # Suggested fixes - - def __str__(self) -> str: - lines = [f"Checkpoint: {self.checkpoint}"] - lines.append(f"Status: {'PASS' if self.valid else 'FAIL'}") - - if self.errors: - lines.append("\nErrors:") - for err in self.errors: - lines.append(f" ✗ {err}") - - if self.warnings: - lines.append("\nWarnings:") - for warn in self.warnings: - lines.append(f" ⚠ {warn}") - - if self.fixes and not self.valid: - lines.append("\nSuggested Fixes:") - for fix in self.fixes: - lines.append(f" → {fix}") - - return "\n".join(lines) - - -class SpecValidator: - """Validates spec outputs at each checkpoint.""" - - def __init__(self, spec_dir: Path): - self.spec_dir = Path(spec_dir) - - def validate_all(self) -> list[ValidationResult]: - """Run all validations.""" - results = [ - self.validate_prereqs(), - self.validate_context(), - self.validate_spec_document(), - self.validate_implementation_plan(), - ] - return results - - def validate_prereqs(self) -> ValidationResult: - """Validate prerequisites exist.""" - errors = [] - warnings = [] - fixes = [] - - # Check spec directory exists - if not self.spec_dir.exists(): - errors.append(f"Spec directory does not exist: {self.spec_dir}") - fixes.append(f"Create directory: mkdir -p {self.spec_dir}") - return ValidationResult(False, "prereqs", errors, warnings, fixes) - - # Check project_index.json - project_index = self.spec_dir / "project_index.json" - if not project_index.exists(): - # Check if it exists at auto-claude level - auto_build_index = self.spec_dir.parent.parent / "project_index.json" - if auto_build_index.exists(): - warnings.append( - "project_index.json exists at auto-claude/ but not in spec folder" - ) - fixes.append(f"Copy: cp {auto_build_index} {project_index}") - else: - errors.append("project_index.json not found") - fixes.append( - "Run: python auto-claude/analyzer.py --output auto-claude/project_index.json" - ) - - return ValidationResult( - valid=len(errors) == 0, - checkpoint="prereqs", - errors=errors, - warnings=warnings, - fixes=fixes, - ) - - def validate_context(self) -> ValidationResult: - """Validate context.json exists and has required structure.""" - errors = [] - warnings = [] - fixes = [] - - context_file = self.spec_dir / "context.json" - - if not context_file.exists(): - errors.append("context.json not found") - fixes.append( - "Run: python auto-claude/context.py --task '[task]' --services '[services]' --output context.json" - ) - return ValidationResult(False, "context", errors, warnings, fixes) - - try: - with open(context_file) as f: - context = json.load(f) - except json.JSONDecodeError as e: - errors.append(f"context.json is invalid JSON: {e}") - fixes.append("Regenerate context.json or fix JSON syntax") - return ValidationResult(False, "context", errors, warnings, fixes) - - # Check required fields - for field in CONTEXT_SCHEMA["required_fields"]: - if field not in context: - errors.append(f"Missing required field: {field}") - fixes.append(f"Add '{field}' to context.json") - - # Check optional but recommended fields - recommended = ["files_to_modify", "files_to_reference", "scoped_services"] - for field in recommended: - if field not in context or not context[field]: - warnings.append(f"Missing recommended field: {field}") - - return ValidationResult( - valid=len(errors) == 0, - checkpoint="context", - errors=errors, - warnings=warnings, - fixes=fixes, - ) - - def validate_spec_document(self) -> ValidationResult: - """Validate spec.md exists and has required sections.""" - errors = [] - warnings = [] - fixes = [] - - spec_file = self.spec_dir / "spec.md" - - if not spec_file.exists(): - errors.append("spec.md not found") - fixes.append("Create spec.md with required sections") - return ValidationResult(False, "spec", errors, warnings, fixes) - - content = spec_file.read_text() - - # Check for required sections - for section in SPEC_REQUIRED_SECTIONS: - # Look for ## Section or # Section - pattern = rf"^##?\s+{re.escape(section)}" - if not re.search(pattern, content, re.MULTILINE | re.IGNORECASE): - errors.append(f"Missing required section: '{section}'") - fixes.append(f"Add '## {section}' section to spec.md") - - # Check for recommended sections - for section in SPEC_RECOMMENDED_SECTIONS: - pattern = rf"^##?\s+{re.escape(section)}" - if not re.search(pattern, content, re.MULTILINE | re.IGNORECASE): - warnings.append(f"Missing recommended section: '{section}'") - - # Check minimum content length - if len(content) < 500: - warnings.append("spec.md seems too short (< 500 chars)") - - return ValidationResult( - valid=len(errors) == 0, - checkpoint="spec", - errors=errors, - warnings=warnings, - fixes=fixes, - ) - - def validate_implementation_plan(self) -> ValidationResult: - """Validate implementation_plan.json exists and has valid schema.""" - errors = [] - warnings = [] - fixes = [] - - plan_file = self.spec_dir / "implementation_plan.json" - - if not plan_file.exists(): - errors.append("implementation_plan.json not found") - fixes.append( - f"Run: python auto-claude/planner.py --spec-dir {self.spec_dir}" - ) - return ValidationResult(False, "plan", errors, warnings, fixes) - - try: - with open(plan_file) as f: - plan = json.load(f) - except json.JSONDecodeError as e: - errors.append(f"implementation_plan.json is invalid JSON: {e}") - fixes.append( - "Regenerate with: python auto-claude/planner.py --spec-dir " - + str(self.spec_dir) - ) - return ValidationResult(False, "plan", errors, warnings, fixes) - - # Validate top-level required fields - schema = IMPLEMENTATION_PLAN_SCHEMA - for field in schema["required_fields"]: - if field not in plan: - errors.append(f"Missing required field: {field}") - fixes.append(f"Add '{field}' to implementation_plan.json") - - # Validate workflow_type - if "workflow_type" in plan: - if plan["workflow_type"] not in schema["workflow_types"]: - errors.append(f"Invalid workflow_type: {plan['workflow_type']}") - fixes.append(f"Use one of: {schema['workflow_types']}") - - # Validate phases - phases = plan.get("phases", []) - if not phases: - errors.append("No phases defined") - fixes.append("Add at least one phase with subtasks") - else: - for i, phase in enumerate(phases): - phase_errors = self._validate_phase(phase, i) - errors.extend(phase_errors) - - # Check for at least one subtask - total_subtasks = sum(len(p.get("subtasks", [])) for p in phases) - if total_subtasks == 0: - errors.append("No subtasks defined in any phase") - fixes.append("Add subtasks to phases") - - # Validate dependencies don't create cycles - dep_errors = self._validate_dependencies(phases) - errors.extend(dep_errors) - - return ValidationResult( - valid=len(errors) == 0, - checkpoint="plan", - errors=errors, - warnings=warnings, - fixes=fixes, - ) - - def _validate_phase(self, phase: dict, index: int) -> list[str]: - """Validate a single phase. - - Supports both legacy format (using 'phase' number) and new format (using 'id' string). - """ - errors = [] - schema = IMPLEMENTATION_PLAN_SCHEMA["phase_schema"] - - # Check required fields - for field in schema["required_fields"]: - if field not in phase: - errors.append(f"Phase {index + 1}: missing required field '{field}'") - - # Check either-or required fields (must have at least one from each group) - for field_group in schema.get("required_fields_either", []): - if not any(f in phase for f in field_group): - errors.append( - f"Phase {index + 1}: missing required field (need one of: {', '.join(field_group)})" - ) - - if "type" in phase and phase["type"] not in schema["phase_types"]: - errors.append(f"Phase {index + 1}: invalid type '{phase['type']}'") - - # Validate subtasks - subtasks = phase.get("subtasks", []) - for j, subtask in enumerate(subtasks): - subtask_errors = self._validate_subtask(subtask, index, j) - errors.extend(subtask_errors) - - return errors - - def _validate_subtask( - self, subtask: dict, phase_idx: int, subtask_idx: int - ) -> list[str]: - """Validate a single subtask.""" - errors = [] - schema = IMPLEMENTATION_PLAN_SCHEMA["subtask_schema"] - - for field in schema["required_fields"]: - if field not in subtask: - errors.append( - f"Phase {phase_idx + 1}, Subtask {subtask_idx + 1}: missing required field '{field}'" - ) - - if "status" in subtask and subtask["status"] not in schema["status_values"]: - errors.append( - f"Phase {phase_idx + 1}, Subtask {subtask_idx + 1}: invalid status '{subtask['status']}'" - ) - - # Validate verification if present - if "verification" in subtask: - ver = subtask["verification"] - ver_schema = IMPLEMENTATION_PLAN_SCHEMA["verification_schema"] - - if "type" not in ver: - errors.append( - f"Phase {phase_idx + 1}, Subtask {subtask_idx + 1}: verification missing 'type'" - ) - elif ver["type"] not in ver_schema["verification_types"]: - errors.append( - f"Phase {phase_idx + 1}, Subtask {subtask_idx + 1}: invalid verification type '{ver['type']}'" - ) - - return errors - - def _validate_dependencies(self, phases: list[dict]) -> list[str]: - """Check for circular dependencies. - - Supports both legacy numeric phase IDs and new string-based phase IDs. - """ - errors = [] - - # Build a map of phase identifiers (supports both "id" and "phase" fields) - # and track their position/order for cycle detection - phase_ids = set() - phase_order = {} # Maps phase id -> position index - - for i, p in enumerate(phases): - # Support both "id" field (new format) and "phase" field (legacy format) - phase_id = p.get("id") or p.get("phase", i + 1) - phase_ids.add(phase_id) - phase_order[phase_id] = i - - for i, phase in enumerate(phases): - phase_id = phase.get("id") or phase.get("phase", i + 1) - depends_on = phase.get("depends_on", []) - - for dep in depends_on: - if dep not in phase_ids: - errors.append( - f"Phase {phase_id}: depends on non-existent phase {dep}" - ) - # Check for forward references (cycles) by comparing positions - elif phase_order.get(dep, -1) >= i: - errors.append( - f"Phase {phase_id}: cannot depend on phase {dep} (would create cycle)" - ) - - return errors - - -def auto_fix_plan(spec_dir: Path) -> bool: - """Attempt to auto-fix common implementation_plan.json issues.""" - plan_file = spec_dir / "implementation_plan.json" - - if not plan_file.exists(): - return False - - try: - with open(plan_file) as f: - plan = json.load(f) - except json.JSONDecodeError: - return False - - fixed = False - - # Fix missing top-level fields - if "feature" not in plan: - plan["feature"] = "Unnamed Feature" - fixed = True - - if "workflow_type" not in plan: - plan["workflow_type"] = "feature" - fixed = True - - if "phases" not in plan: - plan["phases"] = [] - fixed = True - - # Fix phases - for i, phase in enumerate(plan.get("phases", [])): - if "phase" not in phase: - phase["phase"] = i + 1 - fixed = True - - if "name" not in phase: - phase["name"] = f"Phase {i + 1}" - fixed = True - - if "subtasks" not in phase: - phase["subtasks"] = [] - fixed = True - - # Fix subtasks - for j, subtask in enumerate(phase.get("subtasks", [])): - if "id" not in subtask: - subtask["id"] = f"subtask-{i + 1}-{j + 1}" - fixed = True - - if "description" not in subtask: - subtask["description"] = "No description" - fixed = True - - if "status" not in subtask: - subtask["status"] = "pending" - fixed = True - - if fixed: - with open(plan_file, "w") as f: - json.dump(plan, f, indent=2) - print(f"Auto-fixed: {plan_file}") - - return fixed - - -def main(): - """CLI entry point.""" - import argparse - - parser = argparse.ArgumentParser(description="Validate spec outputs at checkpoints") - parser.add_argument( - "--spec-dir", - type=Path, - required=True, - help="Directory containing spec files", - ) - parser.add_argument( - "--checkpoint", - choices=["prereqs", "context", "spec", "plan", "all"], - default="all", - help="Which checkpoint to validate", - ) - parser.add_argument( - "--auto-fix", - action="store_true", - help="Attempt to auto-fix common issues", - ) - parser.add_argument( - "--json", - action="store_true", - help="Output results as JSON", - ) - - args = parser.parse_args() - - validator = SpecValidator(args.spec_dir) - - if args.auto_fix: - auto_fix_plan(args.spec_dir) - - # Run validations - if args.checkpoint == "all": - results = validator.validate_all() - elif args.checkpoint == "prereqs": - results = [validator.validate_prereqs()] - elif args.checkpoint == "context": - results = [validator.validate_context()] - elif args.checkpoint == "spec": - results = [validator.validate_spec_document()] - elif args.checkpoint == "plan": - results = [validator.validate_implementation_plan()] - - # Output - all_valid = all(r.valid for r in results) - - if args.json: - output = { - "valid": all_valid, - "results": [ - { - "checkpoint": r.checkpoint, - "valid": r.valid, - "errors": r.errors, - "warnings": r.warnings, - "fixes": r.fixes, - } - for r in results - ], - } - print(json.dumps(output, indent=2)) - else: - print("=" * 60) - print(" SPEC VALIDATION REPORT") - print("=" * 60) - print() - - for result in results: - print(result) - print() - - print("=" * 60) - if all_valid: - print(" ✓ ALL CHECKPOINTS PASSED") - else: - print(" ✗ VALIDATION FAILED - See errors above") - print("=" * 60) - - sys.exit(0 if all_valid else 1) - - -if __name__ == "__main__": - main() diff --git a/auto-claude/validate_spec/__init__.py b/auto-claude/validate_spec/__init__.py index 9f4061e9..43290c49 100644 --- a/auto-claude/validate_spec/__init__.py +++ b/auto-claude/validate_spec/__init__.py @@ -1,19 +1,43 @@ """ -Spec Validation System -====================== +Backward compatibility shim for validate_spec package. -Validates spec outputs at each checkpoint to ensure reliability. -This is the enforcement layer that catches errors before they propagate. +DEPRECATED: This package has been moved to spec.validate_pkg. -The spec creation process has mandatory checkpoints: -1. Prerequisites (project_index.json exists) -2. Context (context.json created with required fields) -3. Spec document (spec.md with required sections) -4. Implementation plan (implementation_plan.json with valid schema) +Please update your imports: + OLD: from validate_spec import SpecValidator, ValidationResult, auto_fix_plan + NEW: from spec.validate_pkg import SpecValidator, ValidationResult, auto_fix_plan + +This shim provides compatibility but will be removed in a future version. """ -from .auto_fix import auto_fix_plan -from .models import ValidationResult -from .spec_validator import SpecValidator +import sys +from pathlib import Path + +# Lazy import to avoid circular dependencies +def __getattr__(name): + """Lazy import mechanism to avoid circular imports.""" + if name in ("SpecValidator", "ValidationResult", "auto_fix_plan"): + # Add spec directory to path temporarily to allow direct imports + # without triggering spec.__init__ + spec_dir = Path(__file__).parent.parent / "spec" + if str(spec_dir) not in sys.path: + sys.path.insert(0, str(spec_dir)) + + try: + # Import directly from validate_pkg without going through spec package + from validate_pkg import SpecValidator, ValidationResult, auto_fix_plan + + # Cache the imported values in this module + globals()["SpecValidator"] = SpecValidator + globals()["ValidationResult"] = ValidationResult + globals()["auto_fix_plan"] = auto_fix_plan + + return globals()[name] + finally: + # Clean up path modification + if str(spec_dir) in sys.path: + sys.path.remove(str(spec_dir)) + + raise AttributeError(f"module 'validate_spec' has no attribute '{name}'") __all__ = ["SpecValidator", "ValidationResult", "auto_fix_plan"] diff --git a/auto-claude/validate_spec/auto_fix.py b/auto-claude/validate_spec/auto_fix.py index e89e8ca6..bfe2c6b6 100644 --- a/auto-claude/validate_spec/auto_fix.py +++ b/auto-claude/validate_spec/auto_fix.py @@ -1,80 +1,40 @@ """ -Auto-Fix Utilities -================== +Backward compatibility shim for auto_fix module. -Automated fixes for common implementation plan issues. +DEPRECATED: This module has been moved to spec.validate_pkg.auto_fix. + +Please update your imports: + OLD: from validate_spec.auto_fix import auto_fix_plan + NEW: from spec.validate_pkg.auto_fix import auto_fix_plan + +This shim provides compatibility but will be removed in a future version. """ -import json +import sys from pathlib import Path +# Lazy import to avoid circular dependencies +def __getattr__(name): + """Lazy import mechanism to avoid circular imports.""" + if name == "auto_fix_plan": + # Add spec directory to path temporarily to allow direct imports + # without triggering spec.__init__ + spec_dir = Path(__file__).parent.parent / "spec" + if str(spec_dir) not in sys.path: + sys.path.insert(0, str(spec_dir)) -def auto_fix_plan(spec_dir: Path) -> bool: - """Attempt to auto-fix common implementation_plan.json issues. + try: + # Import directly from validate_pkg without going through spec package + from validate_pkg.auto_fix import auto_fix_plan - Args: - spec_dir: Path to the spec directory + # Cache the imported value in this module + globals()["auto_fix_plan"] = auto_fix_plan + return auto_fix_plan + finally: + # Clean up path modification + if str(spec_dir) in sys.path: + sys.path.remove(str(spec_dir)) - Returns: - True if fixes were applied, False otherwise - """ - plan_file = spec_dir / "implementation_plan.json" + raise AttributeError(f"module 'validate_spec.auto_fix' has no attribute '{name}'") - if not plan_file.exists(): - return False - - try: - with open(plan_file) as f: - plan = json.load(f) - except json.JSONDecodeError: - return False - - fixed = False - - # Fix missing top-level fields - if "feature" not in plan: - plan["feature"] = "Unnamed Feature" - fixed = True - - if "workflow_type" not in plan: - plan["workflow_type"] = "feature" - fixed = True - - if "phases" not in plan: - plan["phases"] = [] - fixed = True - - # Fix phases - for i, phase in enumerate(plan.get("phases", [])): - if "phase" not in phase: - phase["phase"] = i + 1 - fixed = True - - if "name" not in phase: - phase["name"] = f"Phase {i + 1}" - fixed = True - - if "subtasks" not in phase: - phase["subtasks"] = [] - fixed = True - - # Fix subtasks - for j, subtask in enumerate(phase.get("subtasks", [])): - if "id" not in subtask: - subtask["id"] = f"subtask-{i + 1}-{j + 1}" - fixed = True - - if "description" not in subtask: - subtask["description"] = "No description" - fixed = True - - if "status" not in subtask: - subtask["status"] = "pending" - fixed = True - - if fixed: - with open(plan_file, "w") as f: - json.dump(plan, f, indent=2) - print(f"Auto-fixed: {plan_file}") - - return fixed +__all__ = ["auto_fix_plan"] diff --git a/auto-claude/validate_spec/spec_validator.py b/auto-claude/validate_spec/spec_validator.py index 1b8064de..1b341bb2 100644 --- a/auto-claude/validate_spec/spec_validator.py +++ b/auto-claude/validate_spec/spec_validator.py @@ -1,80 +1,40 @@ """ -Spec Validator -============== +Backward compatibility shim for spec_validator module. -Main validator class that orchestrates all validation checkpoints. +DEPRECATED: This module has been moved to spec.validate_pkg.spec_validator. + +Please update your imports: + OLD: from validate_spec.spec_validator import SpecValidator + NEW: from spec.validate_pkg.spec_validator import SpecValidator + +This shim provides compatibility but will be removed in a future version. """ +import sys from pathlib import Path -from .models import ValidationResult -from .validators import ( - ContextValidator, - ImplementationPlanValidator, - PrereqsValidator, - SpecDocumentValidator, -) +# Lazy import to avoid circular dependencies +def __getattr__(name): + """Lazy import mechanism to avoid circular imports.""" + if name == "SpecValidator": + # Add spec directory to path temporarily to allow direct imports + # without triggering spec.__init__ + spec_dir = Path(__file__).parent.parent / "spec" + if str(spec_dir) not in sys.path: + sys.path.insert(0, str(spec_dir)) + try: + # Import directly from validate_pkg without going through spec package + from validate_pkg.spec_validator import SpecValidator -class SpecValidator: - """Validates spec outputs at each checkpoint.""" + # Cache the imported value in this module + globals()["SpecValidator"] = SpecValidator + return SpecValidator + finally: + # Clean up path modification + if str(spec_dir) in sys.path: + sys.path.remove(str(spec_dir)) - def __init__(self, spec_dir: Path): - """Initialize the spec validator. + raise AttributeError(f"module 'validate_spec.spec_validator' has no attribute '{name}'") - Args: - spec_dir: Path to the spec directory - """ - self.spec_dir = Path(spec_dir) - - # Initialize individual validators - self._prereqs_validator = PrereqsValidator(self.spec_dir) - self._context_validator = ContextValidator(self.spec_dir) - self._spec_document_validator = SpecDocumentValidator(self.spec_dir) - self._implementation_plan_validator = ImplementationPlanValidator(self.spec_dir) - - def validate_all(self) -> list[ValidationResult]: - """Run all validations. - - Returns: - List of validation results for all checkpoints - """ - results = [ - self.validate_prereqs(), - self.validate_context(), - self.validate_spec_document(), - self.validate_implementation_plan(), - ] - return results - - def validate_prereqs(self) -> ValidationResult: - """Validate prerequisites exist. - - Returns: - ValidationResult for prerequisites checkpoint - """ - return self._prereqs_validator.validate() - - def validate_context(self) -> ValidationResult: - """Validate context.json exists and has required structure. - - Returns: - ValidationResult for context checkpoint - """ - return self._context_validator.validate() - - def validate_spec_document(self) -> ValidationResult: - """Validate spec.md exists and has required sections. - - Returns: - ValidationResult for spec document checkpoint - """ - return self._spec_document_validator.validate() - - def validate_implementation_plan(self) -> ValidationResult: - """Validate implementation_plan.json exists and has valid schema. - - Returns: - ValidationResult for implementation plan checkpoint - """ - return self._implementation_plan_validator.validate() +__all__ = ["SpecValidator"] diff --git a/auto-claude/validation_strategy.py b/auto-claude/validation_strategy.py index cad467e2..a6ee734f 100644 --- a/auto-claude/validation_strategy.py +++ b/auto-claude/validation_strategy.py @@ -1,1033 +1,2 @@ -#!/usr/bin/env python3 -""" -Validation Strategy Module -========================== - -Builds validation strategies based on project type and risk level. -This module determines how the QA agent should validate implementations. - -The validation strategy is used by: -- Planner Agent: To define verification requirements in the implementation plan -- QA Agent: To determine what tests to create and run - -Usage: - from validation_strategy import ValidationStrategyBuilder - - builder = ValidationStrategyBuilder() - strategy = builder.build_strategy(project_dir, spec_dir, "medium") - - for step in strategy: - print(f"Run: {step.command}") -""" - -import json -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from risk_classifier import RiskClassifier - -# ============================================================================= -# DATA CLASSES -# ============================================================================= - - -@dataclass -class ValidationStep: - """ - A single validation step to execute. - - Attributes: - name: Human-readable name of the step - command: Command to execute (or "manual" for manual steps) - expected_outcome: Description of what success looks like - step_type: Type of validation (test, visual, api, security, manual) - required: Whether this step is mandatory - blocking: Whether failure blocks approval - """ - - name: str - command: str - expected_outcome: str - step_type: str # test, visual, api, security, manual - required: bool = True - blocking: bool = True - - -@dataclass -class ValidationStrategy: - """ - Complete validation strategy for a task. - - Attributes: - risk_level: Risk level (trivial, low, medium, high, critical) - project_type: Detected project type - steps: List of validation steps to execute - test_types_required: List of test types to create - security_scan_required: Whether security scanning is needed - staging_deployment_required: Whether staging deployment is needed - skip_validation: Whether validation can be skipped entirely - reasoning: Explanation of the strategy - """ - - risk_level: str - project_type: str - steps: list[ValidationStep] = field(default_factory=list) - test_types_required: list[str] = field(default_factory=list) - security_scan_required: bool = False - staging_deployment_required: bool = False - skip_validation: bool = False - reasoning: str = "" - - -# ============================================================================= -# PROJECT TYPE DETECTION -# ============================================================================= - - -# Project type indicators -PROJECT_TYPE_INDICATORS = { - "html_css": { - "files": ["index.html", "style.css", "styles.css"], - "extensions": [".html", ".css"], - "no_package_manager": True, - }, - "react_spa": { - "dependencies": ["react", "react-dom"], - "files": ["package.json"], - }, - "vue_spa": { - "dependencies": ["vue"], - "files": ["package.json"], - }, - "nextjs": { - "dependencies": ["next"], - "files": ["next.config.js", "next.config.mjs", "next.config.ts"], - }, - "nodejs": { - "files": ["package.json"], - "not_dependencies": ["react", "vue", "next", "angular"], - }, - "python_api": { - "dependencies_python": ["fastapi", "flask", "django"], - "files": ["pyproject.toml", "setup.py", "requirements.txt"], - }, - "python_cli": { - "files": ["pyproject.toml", "setup.py"], - "entry_points": True, - }, - "rust": { - "files": ["Cargo.toml"], - }, - "go": { - "files": ["go.mod"], - }, - "ruby": { - "files": ["Gemfile"], - }, -} - - -def detect_project_type(project_dir: Path) -> str: - """ - Detect the project type based on files and dependencies. - - Args: - project_dir: Path to the project directory - - Returns: - Project type string (e.g., "react_spa", "python_api", "nodejs") - """ - project_dir = Path(project_dir) - - # Check for specific frameworks first - package_json = project_dir / "package.json" - if package_json.exists(): - try: - with open(package_json, encoding="utf-8") as f: - pkg = json.load(f) - deps = pkg.get("dependencies", {}) - dev_deps = pkg.get("devDependencies", {}) - all_deps = {**deps, **dev_deps} - - if "electron" in all_deps: - return "electron" - if "next" in all_deps: - return "nextjs" - if "react" in all_deps: - return "react_spa" - if "vue" in all_deps: - return "vue_spa" - if "@angular/core" in all_deps: - return "angular_spa" - return "nodejs" - except (OSError, json.JSONDecodeError): - return "nodejs" - - # Check for Python projects - pyproject = project_dir / "pyproject.toml" - requirements = project_dir / "requirements.txt" - if pyproject.exists() or requirements.exists(): - # Try to detect API framework - deps_text = "" - if requirements.exists(): - deps_text = requirements.read_text().lower() - if pyproject.exists(): - deps_text += pyproject.read_text().lower() - - if "fastapi" in deps_text or "flask" in deps_text or "django" in deps_text: - return "python_api" - if "click" in deps_text or "typer" in deps_text or "argparse" in deps_text: - return "python_cli" - return "python" - - # Check for other languages - if (project_dir / "Cargo.toml").exists(): - return "rust" - if (project_dir / "go.mod").exists(): - return "go" - if (project_dir / "Gemfile").exists(): - return "ruby" - - # Check for simple HTML/CSS - html_files = list(project_dir.glob("*.html")) - if html_files: - return "html_css" - - return "unknown" - - -# ============================================================================= -# VALIDATION STRATEGY BUILDER -# ============================================================================= - - -class ValidationStrategyBuilder: - """ - Builds validation strategies based on project type and risk level. - - The builder uses the risk assessment from complexity_assessment.json - and adapts the validation strategy to the detected project type. - """ - - def __init__(self) -> None: - """Initialize the strategy builder.""" - self._risk_classifier = RiskClassifier() - - def build_strategy( - self, - project_dir: Path, - spec_dir: Path, - risk_level: str | None = None, - ) -> ValidationStrategy: - """ - Build a validation strategy for the given project and spec. - - Args: - project_dir: Path to the project root - spec_dir: Path to the spec directory - risk_level: Override risk level (if not provided, reads from assessment) - - Returns: - ValidationStrategy with appropriate steps - """ - project_dir = Path(project_dir) - spec_dir = Path(spec_dir) - - # Get risk level from assessment if not provided - if risk_level is None: - assessment = self._risk_classifier.load_assessment(spec_dir) - if assessment: - risk_level = assessment.validation.risk_level - else: - risk_level = "medium" # Default to medium - - # Detect project type - project_type = detect_project_type(project_dir) - - # Build strategy based on project type - strategy_builders = { - "html_css": self._strategy_for_html_css, - "react_spa": self._strategy_for_spa, - "vue_spa": self._strategy_for_spa, - "angular_spa": self._strategy_for_spa, - "nextjs": self._strategy_for_fullstack, - "nodejs": self._strategy_for_nodejs, - "electron": self._strategy_for_electron, - "python_api": self._strategy_for_python_api, - "python_cli": self._strategy_for_cli, - "python": self._strategy_for_python, - "rust": self._strategy_for_rust, - "go": self._strategy_for_go, - "ruby": self._strategy_for_ruby, - } - - builder_func = strategy_builders.get(project_type, self._strategy_default) - strategy = builder_func(project_dir, risk_level) - - # Add security scanning for high+ risk - if risk_level in ["high", "critical"]: - strategy = self._add_security_steps(strategy, project_type) - - # Set common properties - strategy.risk_level = risk_level - strategy.project_type = project_type - strategy.skip_validation = risk_level == "trivial" - - return strategy - - def _strategy_for_html_css( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for simple HTML/CSS projects. - - Focus on visual verification and accessibility. - """ - steps = [ - ValidationStep( - name="Start HTTP Server", - command="python -m http.server 8000 &", - expected_outcome="Server running on port 8000", - step_type="setup", - required=True, - blocking=True, - ), - ValidationStep( - name="Visual Verification", - command="npx playwright screenshot http://localhost:8000 screenshot.png", - expected_outcome="Screenshot captured without errors", - step_type="visual", - required=True, - blocking=False, - ), - ValidationStep( - name="Console Error Check", - command="npx playwright test --grep 'console-errors'", - expected_outcome="No JavaScript console errors", - step_type="test", - required=True, - blocking=True, - ), - ] - - # Add Lighthouse for medium+ risk - if risk_level in ["medium", "high", "critical"]: - steps.append( - ValidationStep( - name="Lighthouse Audit", - command="npx lighthouse http://localhost:8000 --output=json --output-path=lighthouse.json", - expected_outcome="Performance > 90, Accessibility > 90", - step_type="visual", - required=True, - blocking=risk_level in ["high", "critical"], - ) - ) - - return ValidationStrategy( - risk_level=risk_level, - project_type="html_css", - steps=steps, - test_types_required=["visual"] if risk_level != "trivial" else [], - reasoning="HTML/CSS project requires visual verification and accessibility checks.", - ) - - def _strategy_for_spa( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for Single Page Applications (React, Vue, Angular). - - Focus on component tests and E2E testing. - """ - steps = [] - - # Unit/component tests for all non-trivial - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Unit/Component Tests", - command="npm test", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - # E2E tests for medium+ risk - if risk_level in ["medium", "high", "critical"]: - steps.append( - ValidationStep( - name="E2E Tests", - command="npx playwright test", - expected_outcome="All E2E tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - # Browser console check - steps.append( - ValidationStep( - name="Console Error Check", - command="npm run dev & sleep 5 && npx playwright test --grep 'no-console-errors'", - expected_outcome="No console errors in browser", - step_type="test", - required=True, - blocking=risk_level in ["high", "critical"], - ) - ) - - # Determine test types - test_types = ["unit"] - if risk_level in ["medium", "high", "critical"]: - test_types.append("integration") - if risk_level in ["high", "critical"]: - test_types.append("e2e") - - return ValidationStrategy( - risk_level=risk_level, - project_type="spa", - steps=steps, - test_types_required=test_types, - reasoning="SPA requires component tests for logic and E2E for user flows.", - ) - - def _strategy_for_fullstack( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for fullstack frameworks (Next.js, Rails, Django). - - Focus on API tests, frontend tests, and integration. - """ - steps = [] - - # Unit tests - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Unit Tests", - command="npm test", - expected_outcome="All unit tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - # API tests for medium+ risk - if risk_level in ["medium", "high", "critical"]: - steps.append( - ValidationStep( - name="API Integration Tests", - command="npm run test:api", - expected_outcome="All API tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - # E2E tests for high+ risk - if risk_level in ["high", "critical"]: - steps.append( - ValidationStep( - name="E2E Tests", - command="npm run test:e2e", - expected_outcome="All E2E tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - # Database migration check - steps.append( - ValidationStep( - name="Database Migration Check", - command="npm run db:migrate:status", - expected_outcome="All migrations applied successfully", - step_type="api", - required=risk_level in ["medium", "high", "critical"], - blocking=True, - ) - ) - - # Determine test types - test_types = ["unit"] - if risk_level in ["medium", "high", "critical"]: - test_types.append("integration") - if risk_level in ["high", "critical"]: - test_types.append("e2e") - - return ValidationStrategy( - risk_level=risk_level, - project_type="fullstack", - steps=steps, - test_types_required=test_types, - reasoning="Fullstack requires API tests, frontend tests, and DB migration checks.", - ) - - def _strategy_for_nodejs( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for Node.js backend projects. - """ - steps = [] - - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Unit Tests", - command="npm test", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - if risk_level in ["medium", "high", "critical"]: - steps.append( - ValidationStep( - name="Integration Tests", - command="npm run test:integration", - expected_outcome="All integration tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - test_types = ["unit"] - if risk_level in ["medium", "high", "critical"]: - test_types.append("integration") - - return ValidationStrategy( - risk_level=risk_level, - project_type="nodejs", - steps=steps, - test_types_required=test_types, - reasoning="Node.js backend requires unit and integration tests.", - ) - - def _strategy_for_python_api( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for Python API projects (FastAPI, Flask, Django). - """ - steps = [] - - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Unit Tests", - command="pytest tests/ -v", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - if risk_level in ["medium", "high", "critical"]: - steps.append( - ValidationStep( - name="API Tests", - command="pytest tests/api/ -v", - expected_outcome="All API tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - steps.append( - ValidationStep( - name="Coverage Check", - command="pytest --cov=src --cov-report=term-missing", - expected_outcome="Coverage >= 80%", - step_type="test", - required=True, - blocking=risk_level == "critical", - ) - ) - - if risk_level in ["high", "critical"]: - steps.append( - ValidationStep( - name="Database Migration Check", - command="alembic current && alembic check", - expected_outcome="Migrations are current and valid", - step_type="api", - required=True, - blocking=True, - ) - ) - - test_types = ["unit"] - if risk_level in ["medium", "high", "critical"]: - test_types.append("integration") - if risk_level in ["high", "critical"]: - test_types.append("e2e") - - return ValidationStrategy( - risk_level=risk_level, - project_type="python_api", - steps=steps, - test_types_required=test_types, - reasoning="Python API requires pytest tests and migration checks.", - ) - - def _strategy_for_cli( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for CLI tools. - """ - steps = [] - - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Unit Tests", - command="pytest tests/ -v", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - steps.append( - ValidationStep( - name="CLI Help Check", - command="python -m module_name --help", - expected_outcome="Help text displays without errors", - step_type="test", - required=True, - blocking=True, - ) - ) - - if risk_level in ["medium", "high", "critical"]: - steps.append( - ValidationStep( - name="CLI Output Verification", - command="python -m module_name --version", - expected_outcome="Version displays correctly", - step_type="test", - required=True, - blocking=False, - ) - ) - - return ValidationStrategy( - risk_level=risk_level, - project_type="python_cli", - steps=steps, - test_types_required=["unit"], - reasoning="CLI tools require output verification and unit tests.", - ) - - def _strategy_for_python( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for generic Python projects. - """ - steps = [] - - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Unit Tests", - command="pytest tests/ -v", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - test_types = ["unit"] - if risk_level in ["medium", "high", "critical"]: - test_types.append("integration") - - return ValidationStrategy( - risk_level=risk_level, - project_type="python", - steps=steps, - test_types_required=test_types, - reasoning="Python project requires pytest unit tests.", - ) - - def _strategy_for_rust( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for Rust projects. - """ - steps = [] - - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Cargo Test", - command="cargo test", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - steps.append( - ValidationStep( - name="Cargo Clippy", - command="cargo clippy -- -D warnings", - expected_outcome="No clippy warnings", - step_type="test", - required=True, - blocking=risk_level in ["high", "critical"], - ) - ) - - return ValidationStrategy( - risk_level=risk_level, - project_type="rust", - steps=steps, - test_types_required=["unit"], - reasoning="Rust project requires cargo test and clippy checks.", - ) - - def _strategy_for_go( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for Go projects. - """ - steps = [] - - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Go Test", - command="go test ./...", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - steps.append( - ValidationStep( - name="Go Vet", - command="go vet ./...", - expected_outcome="No issues found", - step_type="test", - required=True, - blocking=risk_level in ["high", "critical"], - ) - ) - - return ValidationStrategy( - risk_level=risk_level, - project_type="go", - steps=steps, - test_types_required=["unit"], - reasoning="Go project requires go test and vet checks.", - ) - - def _strategy_for_ruby( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for Ruby projects. - """ - steps = [] - - if risk_level != "trivial": - steps.append( - ValidationStep( - name="RSpec Tests", - command="bundle exec rspec", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - return ValidationStrategy( - risk_level=risk_level, - project_type="ruby", - steps=steps, - test_types_required=["unit"], - reasoning="Ruby project requires RSpec tests.", - ) - - def _strategy_for_electron( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Validation strategy for Electron desktop applications. - - Focus on main/renderer process tests, E2E testing, and app packaging. - """ - steps = [] - - # Unit tests for all non-trivial - if risk_level != "trivial": - steps.append( - ValidationStep( - name="Unit Tests", - command="npm test", - expected_outcome="All tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - # E2E tests for medium+ risk (Electron apps need GUI testing) - if risk_level in ["medium", "high", "critical"]: - steps.append( - ValidationStep( - name="E2E Tests", - command="npm run test:e2e", - expected_outcome="All E2E tests pass", - step_type="test", - required=True, - blocking=True, - ) - ) - - # App build/package verification for medium+ risk - if risk_level in ["medium", "high", "critical"]: - steps.append( - ValidationStep( - name="Build Verification", - command="npm run build", - expected_outcome="App builds without errors", - step_type="test", - required=True, - blocking=True, - ) - ) - - # Console error check for high+ risk - if risk_level in ["high", "critical"]: - steps.append( - ValidationStep( - name="Console Error Check", - command="npm run test:console", - expected_outcome="No console errors in main or renderer process", - step_type="test", - required=True, - blocking=True, - ) - ) - - # Determine test types - test_types = ["unit"] - if risk_level in ["medium", "high", "critical"]: - test_types.append("integration") - test_types.append("e2e") - - return ValidationStrategy( - risk_level=risk_level, - project_type="electron", - steps=steps, - test_types_required=test_types, - reasoning="Electron app requires unit tests, E2E tests for GUI, and build verification.", - ) - - def _strategy_default( - self, project_dir: Path, risk_level: str - ) -> ValidationStrategy: - """ - Default validation strategy for unknown project types. - """ - steps = [ - ValidationStep( - name="Manual Verification", - command="manual", - expected_outcome="Code changes reviewed and tested manually", - step_type="manual", - required=True, - blocking=True, - ), - ] - - return ValidationStrategy( - risk_level=risk_level, - project_type="unknown", - steps=steps, - test_types_required=[], - reasoning="Unknown project type - manual verification required.", - ) - - def _add_security_steps( - self, strategy: ValidationStrategy, project_type: str - ) -> ValidationStrategy: - """ - Add security scanning steps to a strategy. - """ - security_steps = [] - - # Secrets scanning (always for high+ risk) - security_steps.append( - ValidationStep( - name="Secrets Scan", - command="python auto-claude/scan_secrets.py --all-files --json", - expected_outcome="No secrets detected", - step_type="security", - required=True, - blocking=True, - ) - ) - - # Language-specific SAST - if project_type in ["python", "python_api", "python_cli"]: - security_steps.append( - ValidationStep( - name="Bandit Security Scan", - command="bandit -r src/ -f json", - expected_outcome="No high severity issues", - step_type="security", - required=True, - blocking=True, - ) - ) - - if project_type in ["nodejs", "react_spa", "vue_spa", "nextjs"]: - security_steps.append( - ValidationStep( - name="npm audit", - command="npm audit --json", - expected_outcome="No critical vulnerabilities", - step_type="security", - required=True, - blocking=True, - ) - ) - - strategy.steps.extend(security_steps) - strategy.security_scan_required = True - - return strategy - - def to_dict(self, strategy: ValidationStrategy) -> dict[str, Any]: - """ - Convert a ValidationStrategy to a dictionary for JSON serialization. - """ - return { - "risk_level": strategy.risk_level, - "project_type": strategy.project_type, - "skip_validation": strategy.skip_validation, - "test_types_required": strategy.test_types_required, - "security_scan_required": strategy.security_scan_required, - "staging_deployment_required": strategy.staging_deployment_required, - "reasoning": strategy.reasoning, - "steps": [ - { - "name": step.name, - "command": step.command, - "expected_outcome": step.expected_outcome, - "type": step.step_type, - "required": step.required, - "blocking": step.blocking, - } - for step in strategy.steps - ], - } - - -# ============================================================================= -# CONVENIENCE FUNCTIONS -# ============================================================================= - - -def build_validation_strategy( - project_dir: Path, - spec_dir: Path, - risk_level: str | None = None, -) -> ValidationStrategy: - """ - Convenience function to build a validation strategy. - - Args: - project_dir: Path to project root - spec_dir: Path to spec directory - risk_level: Optional override for risk level - - Returns: - ValidationStrategy object - """ - builder = ValidationStrategyBuilder() - return builder.build_strategy(project_dir, spec_dir, risk_level) - - -def get_strategy_as_dict( - project_dir: Path, - spec_dir: Path, - risk_level: str | None = None, -) -> dict[str, Any]: - """ - Get validation strategy as a dictionary. - - Args: - project_dir: Path to project root - spec_dir: Path to spec directory - risk_level: Optional override for risk level - - Returns: - Dictionary representation of strategy - """ - builder = ValidationStrategyBuilder() - strategy = builder.build_strategy(project_dir, spec_dir, risk_level) - return builder.to_dict(strategy) - - -# ============================================================================= -# CLI -# ============================================================================= - - -def main() -> None: - """CLI entry point for testing.""" - import argparse - - parser = argparse.ArgumentParser(description="Build validation strategy") - parser.add_argument("project_dir", type=Path, help="Path to project root") - parser.add_argument("--spec-dir", type=Path, help="Path to spec directory") - parser.add_argument("--risk-level", type=str, help="Override risk level") - parser.add_argument("--json", action="store_true", help="Output as JSON") - - args = parser.parse_args() - - spec_dir = args.spec_dir or args.project_dir - builder = ValidationStrategyBuilder() - strategy = builder.build_strategy(args.project_dir, spec_dir, args.risk_level) - - if args.json: - print(json.dumps(builder.to_dict(strategy), indent=2)) - else: - print(f"Project Type: {strategy.project_type}") - print(f"Risk Level: {strategy.risk_level}") - print(f"Skip Validation: {strategy.skip_validation}") - print(f"Test Types: {', '.join(strategy.test_types_required)}") - print(f"Security Scan: {strategy.security_scan_required}") - print(f"Reasoning: {strategy.reasoning}") - print(f"\nValidation Steps ({len(strategy.steps)}):") - for i, step in enumerate(strategy.steps, 1): - print(f" {i}. {step.name}") - print(f" Command: {step.command}") - print(f" Expected: {step.expected_outcome}") - - -if __name__ == "__main__": - main() +"""Backward compatibility shim - import from spec.validation_strategy instead.""" +from spec.validation_strategy import * diff --git a/auto-claude/workspace.py b/auto-claude/workspace.py index 60acb25b..d1498d39 100644 --- a/auto-claude/workspace.py +++ b/auto-claude/workspace.py @@ -1,2399 +1,45 @@ -#!/usr/bin/env python3 """ -Workspace Management - Per-Spec Architecture -============================================= +Backward compatibility shim - import from core.workspace package. -Handles workspace isolation through Git worktrees, where each spec -gets its own isolated worktree in .worktrees/{spec-name}/. +This file exists to maintain backward compatibility for code that imports +from 'workspace' instead of 'core.workspace'. The workspace module has been +refactored into a package (core/workspace/) with multiple sub-modules. -This module has been refactored for better maintainability: -- Models and enums: workspace/models.py -- Git utilities: workspace/git_utils.py -- Setup functions: workspace/setup.py -- Display functions: workspace/display.py -- Finalization: workspace/finalization.py -- Complex merge operations: remain here (workspace.py) +IMPLEMENTATION: To avoid triggering core/__init__.py (which imports modules +with heavy dependencies like claude_agent_sdk), we: +1. Create a minimal fake 'core' module to satisfy Python's import system +2. Load core.workspace package directly using importlib +3. Register it in sys.modules +4. Re-export everything -Public API is exported via workspace/__init__.py for backward compatibility. +This allows 'from workspace import X' to work without requiring all of core's dependencies. """ -import asyncio -import json -import shutil -import subprocess import sys -from dataclasses import dataclass +import importlib.util from pathlib import Path -from typing import Optional - -from ui import ( - Icons, - MenuOption, - bold, - box, - error, - highlight, - icon, - info, - muted, - print_status, - select_menu, - success, - warning, -) -from worktree import WorktreeInfo, WorktreeManager - -# Import debug utilities -try: - from debug import debug, debug_detailed, debug_verbose, debug_success, debug_error, debug_warning, is_debug_enabled -except ImportError: - def debug(*args, **kwargs): pass - def debug_detailed(*args, **kwargs): pass - def debug_verbose(*args, **kwargs): pass - def debug_success(*args, **kwargs): pass - def debug_error(*args, **kwargs): pass - def debug_warning(*args, **kwargs): pass - def is_debug_enabled(): return False - -# Import merge system -from merge import ( - MergeOrchestrator, - MergeDecision, - ConflictSeverity, - FileEvolutionTracker, - FileTimelineTracker, -) - -# Import from refactored modules -from workspace.models import ( - WorkspaceMode, - WorkspaceChoice, - ParallelMergeTask, - ParallelMergeResult, - MergeLock, - MergeLockError, -) - -from workspace.git_utils import ( - has_uncommitted_changes, - get_current_branch, - get_existing_build_worktree, - get_file_content_from_ref as _get_file_content_from_ref, - get_changed_files_from_branch as _get_changed_files_from_branch, - is_process_running as _is_process_running, - is_binary_file as _is_binary_file, - is_lock_file as _is_lock_file, - validate_merged_syntax as _validate_merged_syntax, - create_conflict_file_with_git as _create_conflict_file_with_git, - MAX_FILE_LINES_FOR_AI, - MAX_PARALLEL_AI_MERGES, - MAX_SYNTAX_FIX_RETRIES, - BINARY_EXTENSIONS, - LOCK_FILES, - MERGE_LOCK_TIMEOUT, -) - -from workspace.setup import ( - choose_workspace, - copy_spec_to_worktree, - setup_workspace, - ensure_timeline_hook_installed as _ensure_timeline_hook_installed, - initialize_timeline_tracking as _initialize_timeline_tracking, -) - -from workspace.display import ( - show_build_summary, - show_changed_files, - print_merge_success as _print_merge_success, - print_conflict_info as _print_conflict_info, -) - -from workspace.finalization import ( - finalize_workspace, - handle_workspace_choice, - review_existing_build, - discard_existing_build, - check_existing_build, - list_all_worktrees, - cleanup_all_worktrees, -) - -MODULE = "workspace" - -# The following functions are now imported from refactored modules above. -# They are kept here only to avoid breaking the existing code that still needs -# the complex merge operations below. - -# Remaining complex merge operations that reference each other: -# - merge_existing_build -# - _try_smart_merge -# - _try_smart_merge_inner -# - _check_git_conflicts -# - _resolve_git_conflicts_with_ai -# - _create_async_claude_client -# - _async_ai_call -# - _merge_file_with_ai_async -# - _run_parallel_merges -# - _record_merge_completion -# - _get_task_intent -# - _get_recent_merges_context -# - _merge_file_with_ai -# - _heuristic_merge - -def merge_existing_build( - project_dir: Path, - spec_name: str, - no_commit: bool = False, - use_smart_merge: bool = True, -) -> bool: - """ - Merge an existing build into the project using intent-aware merge. - - Called when user runs: python auto-claude/run.py --spec X --merge - - This uses the MergeOrchestrator to: - 1. Analyze semantic changes from the task - 2. Detect potential conflicts with main branch - 3. Auto-merge compatible changes - 4. Use AI for ambiguous conflicts (if enabled) - 5. Fall back to git merge for remaining changes - - Args: - project_dir: The project directory - spec_name: Name of the spec - no_commit: If True, merge changes but don't commit (stage only for review in IDE) - use_smart_merge: If True, use intent-aware merge (default True) - - Returns: - True if merge succeeded - """ - worktree_path = get_existing_build_worktree(project_dir, spec_name) - - if not worktree_path: - print() - print_status(f"No existing build found for '{spec_name}'.", "warning") - print() - print("To start a new build:") - print(highlight(f" python auto-claude/run.py --spec {spec_name}")) - return False - - if no_commit: - content = [ - bold(f"{icon(Icons.SUCCESS)} STAGING BUILD FOR REVIEW"), - "", - muted("Changes will be staged but NOT committed."), - muted("Review in your IDE, then commit when ready."), - ] - else: - content = [ - bold(f"{icon(Icons.SUCCESS)} ADDING BUILD TO YOUR PROJECT"), - ] - print() - print(box(content, width=60, style="heavy")) - - manager = WorktreeManager(project_dir) - show_build_summary(manager, spec_name) - print() - - # Try smart merge first if enabled - if use_smart_merge: - smart_result = _try_smart_merge( - project_dir, spec_name, worktree_path, manager, no_commit=no_commit - ) - - if smart_result is not None: - # Smart merge handled it (success or identified conflicts) - if smart_result.get("success"): - # Check if smart merge resolved git conflicts directly - if smart_result.get("stats", {}).get("ai_assisted"): - # AI resolved git conflicts - changes are already staged - _print_merge_success(no_commit, smart_result.get("stats")) - - # Cleanup the worktree since merge is done - try: - manager.remove_worktree(spec_name, delete_branch=True) - except Exception: - pass # Best effort cleanup - - return True - else: - # No git conflicts, do standard git merge - success_result = manager.merge_worktree( - spec_name, delete_after=True, no_commit=no_commit - ) - if success_result: - _print_merge_success(no_commit, smart_result.get("stats")) - return True - elif smart_result.get("git_conflicts"): - # Had git conflicts that AI couldn't fully resolve - resolved = smart_result.get("resolved", []) - remaining = smart_result.get("conflicts", []) - - if resolved: - print() - print_status(f"AI resolved {len(resolved)} file(s)", "success") - - if remaining: - print() - print_status( - f"{len(remaining)} conflict(s) require manual resolution:", - "warning" - ) - _print_conflict_info(smart_result) - - # Changes for resolved files are staged, remaining need manual work - print() - print("The resolved files are staged. For remaining conflicts:") - print(muted(" 1. Manually resolve the conflicting files")) - print(muted(" 2. git add ")) - print(muted(" 3. git commit")) - return False - elif smart_result.get("conflicts"): - # Has semantic conflicts that need resolution - _print_conflict_info(smart_result) - print() - print(muted("Attempting git merge anyway...")) - print() - - # Fall back to standard git merge - success_result = manager.merge_worktree( - spec_name, delete_after=True, no_commit=no_commit - ) - - if success_result: - print() - if no_commit: - print_status("Changes are staged in your working directory.", "success") - print() - print("Review the changes in your IDE, then commit:") - print(highlight(" git commit -m 'your commit message'")) - print() - print("Or discard if not satisfied:") - print(muted(" git reset --hard HEAD")) - else: - print_status("Your feature has been added to your project.", "success") - return True - else: - print() - print_status("There was a conflict merging the changes.", "error") - print(muted("You may need to merge manually.")) - return False - - -def _try_smart_merge( - project_dir: Path, - spec_name: str, - worktree_path: Path, - manager: WorktreeManager, - no_commit: bool = False, -) -> Optional[dict]: - """ - Try to use the intent-aware merge system. - - This handles both semantic conflicts (parallel tasks) and git conflicts - (branch divergence) by using AI to intelligently merge files. - - Uses a lock file to prevent concurrent merges for the same spec. - - Returns: - Dict with results, or None if smart merge not applicable - """ - # Quick Win 5: Acquire merge lock to prevent concurrent operations - try: - with MergeLock(project_dir, spec_name): - return _try_smart_merge_inner( - project_dir, spec_name, worktree_path, manager, no_commit - ) - except MergeLockError as e: - print(warning(f" {e}")) - return { - "success": False, - "error": str(e), - "conflicts": [], - } - - -def _try_smart_merge_inner( - project_dir: Path, - spec_name: str, - worktree_path: Path, - manager: WorktreeManager, - no_commit: bool = False, -) -> Optional[dict]: - """Inner implementation of smart merge (called with lock held).""" - debug(MODULE, "=== SMART MERGE START ===", - spec_name=spec_name, - worktree_path=str(worktree_path), - no_commit=no_commit) - - try: - print(muted(" Analyzing changes with intent-aware merge...")) - - # Capture worktree state in FileTimelineTracker before merge - try: - timeline_tracker = FileTimelineTracker(project_dir) - timeline_tracker.capture_worktree_state(spec_name, worktree_path) - debug(MODULE, "Captured worktree state for timeline tracking") - except Exception as e: - debug_warning(MODULE, f"Could not capture worktree state: {e}") - - # Initialize the orchestrator - debug(MODULE, "Initializing MergeOrchestrator", - project_dir=str(project_dir), - enable_ai=True) - orchestrator = MergeOrchestrator( - project_dir, - enable_ai=True, # Enable AI for ambiguous conflicts - dry_run=False, - ) - - # Refresh evolution data from the worktree - debug(MODULE, "Refreshing evolution data from git", - spec_name=spec_name) - orchestrator.evolution_tracker.refresh_from_git(spec_name, worktree_path) - - # Check for git-level conflicts first (branch divergence) - debug(MODULE, "Checking for git-level conflicts") - git_conflicts = _check_git_conflicts(project_dir, spec_name) - - debug_detailed(MODULE, "Git conflict check result", - has_conflicts=git_conflicts.get("has_conflicts"), - conflicting_files=git_conflicts.get("conflicting_files", []), - base_branch=git_conflicts.get("base_branch")) - - if git_conflicts.get("has_conflicts"): - print(muted(f" Branch has diverged from {git_conflicts.get('base_branch', 'main')}")) - print(muted(f" Conflicting files: {len(git_conflicts.get('conflicting_files', []))}")) - - debug(MODULE, "Starting AI conflict resolution", - num_conflicts=len(git_conflicts.get("conflicting_files", []))) - - # Try to resolve git conflicts with AI - resolution_result = _resolve_git_conflicts_with_ai( - project_dir, - spec_name, - worktree_path, - git_conflicts, - orchestrator, - no_commit=no_commit, - ) - - if resolution_result.get("success"): - debug_success(MODULE, "AI conflict resolution succeeded", - resolved_files=resolution_result.get("resolved_files", []), - stats=resolution_result.get("stats", {})) - return resolution_result - else: - # AI couldn't resolve all conflicts - debug_error(MODULE, "AI conflict resolution failed", - remaining_conflicts=resolution_result.get("remaining_conflicts", []), - resolved_files=resolution_result.get("resolved_files", []), - error=resolution_result.get("error")) - return { - "success": False, - "conflicts": resolution_result.get("remaining_conflicts", []), - "resolved": resolution_result.get("resolved_files", []), - "git_conflicts": True, - "error": resolution_result.get("error"), - } - - # No git conflicts - proceed with semantic analysis - debug(MODULE, "No git conflicts, proceeding with semantic analysis") - preview = orchestrator.preview_merge([spec_name]) - - files_to_merge = len(preview.get("files_to_merge", [])) - conflicts = preview.get("conflicts", []) - auto_mergeable = preview.get("summary", {}).get("auto_mergeable", 0) - - print(muted(f" Found {files_to_merge} files to merge")) - - if conflicts: - print(muted(f" Detected {len(conflicts)} potential conflict(s)")) - print(muted(f" Auto-mergeable: {auto_mergeable}/{len(conflicts)}")) - - # Check if any conflicts need human review - needs_human = [ - c for c in conflicts - if not c.get("can_auto_merge") - ] - - if needs_human: - return { - "success": False, - "conflicts": needs_human, - "preview": preview, - } - - # All conflicts can be auto-merged or no conflicts - print(muted(" All changes compatible, proceeding with merge...")) - return { - "success": True, - "stats": { - "files_merged": files_to_merge, - "auto_resolved": auto_mergeable, - }, - } - - except Exception as e: - # If smart merge fails, fall back to git - import traceback - print(muted(f" Smart merge error: {e}")) - traceback.print_exc() - return None - - -def _check_git_conflicts(project_dir: Path, spec_name: str) -> dict: - """ - Check for git-level conflicts WITHOUT modifying the working directory. - - Uses git merge-tree to check conflicts in-memory, avoiding HMR triggers - from file system changes. - - Returns: - Dict with has_conflicts, conflicting_files, etc. - """ - import re - import subprocess - - spec_branch = f"auto-claude/{spec_name}" - result = { - "has_conflicts": False, - "conflicting_files": [], - "base_branch": "main", - "spec_branch": spec_branch, - } - - try: - # Get current branch - base_result = subprocess.run( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], - cwd=project_dir, - capture_output=True, - text=True, - ) - if base_result.returncode == 0: - result["base_branch"] = base_result.stdout.strip() - - # Get merge base - merge_base_result = subprocess.run( - ["git", "merge-base", result["base_branch"], spec_branch], - cwd=project_dir, - capture_output=True, - text=True, - ) - if merge_base_result.returncode != 0: - debug_warning(MODULE, "Could not find merge base") - return result - - merge_base = merge_base_result.stdout.strip() - - # Get commit hashes - main_commit_result = subprocess.run( - ["git", "rev-parse", result["base_branch"]], - cwd=project_dir, - capture_output=True, - text=True, - ) - spec_commit_result = subprocess.run( - ["git", "rev-parse", spec_branch], - cwd=project_dir, - capture_output=True, - text=True, - ) - - if main_commit_result.returncode != 0 or spec_commit_result.returncode != 0: - debug_warning(MODULE, "Could not resolve branch commits") - return result - - main_commit = main_commit_result.stdout.strip() - spec_commit = spec_commit_result.stdout.strip() - - # Use git merge-tree to check for conflicts WITHOUT touching working directory - merge_tree_result = subprocess.run( - ["git", "merge-tree", "--write-tree", "--no-messages", merge_base, main_commit, spec_commit], - cwd=project_dir, - capture_output=True, - text=True, - ) - - # merge-tree returns exit code 1 if there are conflicts - if merge_tree_result.returncode != 0: - result["has_conflicts"] = True - - # Parse the output for conflicting files - output = merge_tree_result.stdout + merge_tree_result.stderr - for line in output.split("\n"): - if "CONFLICT" in line: - match = re.search(r"(?:Merge conflict in|CONFLICT.*?:)\s*(.+?)(?:\s*$|\s+\()", line) - if match: - file_path = match.group(1).strip() - if file_path and file_path not in result["conflicting_files"]: - result["conflicting_files"].append(file_path) - - # Fallback: if we didn't parse conflicts, use diff to find files changed in both branches - if not result["conflicting_files"]: - main_files_result = subprocess.run( - ["git", "diff", "--name-only", merge_base, main_commit], - cwd=project_dir, - capture_output=True, - text=True, - ) - main_files = set(main_files_result.stdout.strip().split("\n")) if main_files_result.stdout.strip() else set() - - spec_files_result = subprocess.run( - ["git", "diff", "--name-only", merge_base, spec_commit], - cwd=project_dir, - capture_output=True, - text=True, - ) - spec_files = set(spec_files_result.stdout.strip().split("\n")) if spec_files_result.stdout.strip() else set() - - # Files modified in both = potential conflicts - conflicting = main_files & spec_files - result["conflicting_files"] = list(conflicting) - - except Exception as e: - print(muted(f" Error checking git conflicts: {e}")) - - return result - - -def _resolve_git_conflicts_with_ai( - project_dir: Path, - spec_name: str, - worktree_path: Path, - git_conflicts: dict, - orchestrator: MergeOrchestrator, - no_commit: bool = False, -) -> dict: - """ - Resolve git-level conflicts using AI. - - This handles the case where main has diverged from the worktree branch. - For each conflicting file, it: - 1. Gets the content from the main branch - 2. Gets the content from the worktree branch - 3. Gets the common ancestor (merge-base) content - 4. Uses AI to intelligently merge them - 5. Writes the merged content to main and stages it - - Returns: - Dict with success, resolved_files, remaining_conflicts - """ - import subprocess - - debug(MODULE, "=== AI CONFLICT RESOLUTION START ===", - spec_name=spec_name, - num_conflicting_files=len(git_conflicts.get("conflicting_files", []))) - - conflicting_files = git_conflicts.get("conflicting_files", []) - base_branch = git_conflicts.get("base_branch", "main") - spec_branch = git_conflicts.get("spec_branch", f"auto-claude/{spec_name}") - - debug_detailed(MODULE, "Conflict resolution params", - base_branch=base_branch, - spec_branch=spec_branch, - conflicting_files=conflicting_files) - - resolved_files = [] - remaining_conflicts = [] - auto_merged_count = 0 - ai_merged_count = 0 - - print() - print_status(f"Resolving {len(conflicting_files)} conflicting file(s) with AI...", "progress") - - # Get merge-base commit - merge_base_result = subprocess.run( - ["git", "merge-base", base_branch, spec_branch], - cwd=project_dir, - capture_output=True, - text=True, - ) - merge_base = merge_base_result.stdout.strip() if merge_base_result.returncode == 0 else None - debug(MODULE, "Found merge-base commit", merge_base=merge_base[:12] if merge_base else None) - - # FIX: Copy NEW files FIRST before resolving conflicts - # This ensures dependencies exist before files that import them are written - changed_files = _get_changed_files_from_branch(project_dir, base_branch, spec_branch) - new_files = [(f, s) for f, s in changed_files if s == "A" and f not in conflicting_files] - - if new_files: - print(muted(f" Copying {len(new_files)} new file(s) first (dependencies)...")) - for file_path, status in new_files: - try: - content = _get_file_content_from_ref(project_dir, spec_branch, file_path) - if content is not None: - target_path = project_dir / file_path - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_text(content, encoding="utf-8") - subprocess.run(["git", "add", file_path], cwd=project_dir, capture_output=True) - resolved_files.append(file_path) - debug(MODULE, f"Copied new file: {file_path}") - except Exception as e: - debug_warning(MODULE, f"Could not copy new file {file_path}: {e}") - - # Categorize conflicting files for processing - files_needing_ai_merge: list[ParallelMergeTask] = [] - simple_merges: list[tuple[str, Optional[str]]] = [] # (file_path, merged_content or None for delete) - - debug(MODULE, "Categorizing conflicting files for parallel processing") - - for file_path in conflicting_files: - debug(MODULE, f"Categorizing conflicting file: {file_path}") - - try: - # Get content from main branch - main_content = _get_file_content_from_ref(project_dir, base_branch, file_path) - - # Get content from worktree branch - worktree_content = _get_file_content_from_ref(project_dir, spec_branch, file_path) - - # Get content from merge-base (common ancestor) - base_content = None - if merge_base: - base_content = _get_file_content_from_ref(project_dir, merge_base, file_path) - - if main_content is None and worktree_content is None: - # File doesn't exist in either - skip - continue - - if main_content is None: - # File only exists in worktree - it's a new file (no AI needed) - simple_merges.append((file_path, worktree_content)) - debug(MODULE, f" {file_path}: new file (no AI needed)") - elif worktree_content is None: - # File only exists in main - was deleted in worktree (no AI needed) - simple_merges.append((file_path, None)) # None = delete - debug(MODULE, f" {file_path}: deleted (no AI needed)") - else: - # File exists in both - check if it's a lock file - if _is_lock_file(file_path): - # Lock files should never go through AI - just take worktree version - # User can run package manager install to regenerate if needed - simple_merges.append((file_path, worktree_content)) - debug(MODULE, f" {file_path}: lock file (taking worktree version)") - else: - # Regular file - needs AI merge - files_needing_ai_merge.append(ParallelMergeTask( - file_path=file_path, - main_content=main_content, - worktree_content=worktree_content, - base_content=base_content, - spec_name=spec_name, - )) - debug(MODULE, f" {file_path}: needs AI merge") - - except Exception as e: - print(error(f" ✗ Failed to categorize {file_path}: {e}")) - remaining_conflicts.append({ - "file": file_path, - "reason": str(e), - "severity": "high", - }) - - # Process simple merges first (fast, no AI) - if simple_merges: - print(muted(f" Processing {len(simple_merges)} simple file(s)...")) - for file_path, merged_content in simple_merges: - try: - if merged_content is not None: - target_path = project_dir / file_path - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_text(merged_content, encoding="utf-8") - subprocess.run(["git", "add", file_path], cwd=project_dir, capture_output=True) - resolved_files.append(file_path) - # Determine the type for display - if _is_lock_file(file_path): - print(success(f" ✓ {file_path} (lock file - took worktree version)")) - else: - print(success(f" ✓ {file_path} (new file)")) - else: - # Delete the file - target_path = project_dir / file_path - if target_path.exists(): - target_path.unlink() - subprocess.run(["git", "add", file_path], cwd=project_dir, capture_output=True) - resolved_files.append(file_path) - print(success(f" ✓ {file_path} (deleted)")) - except Exception as e: - print(error(f" ✗ {file_path}: {e}")) - remaining_conflicts.append({ - "file": file_path, - "reason": str(e), - "severity": "high", - }) - - # Process AI merges in parallel - if files_needing_ai_merge: - print() - print_status(f"Merging {len(files_needing_ai_merge)} file(s) with AI (parallel)...", "progress") - - import time - start_time = time.time() - - # Run parallel merges - parallel_results = asyncio.run(_run_parallel_merges( - tasks=files_needing_ai_merge, - project_dir=project_dir, - max_concurrent=MAX_PARALLEL_AI_MERGES, - )) - - elapsed = time.time() - start_time - - # Process results - for result in parallel_results: - if result.success: - target_path = project_dir / result.file_path - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_text(result.merged_content, encoding="utf-8") - subprocess.run(["git", "add", result.file_path], cwd=project_dir, capture_output=True) - resolved_files.append(result.file_path) - - if result.was_auto_merged: - auto_merged_count += 1 - print(success(f" ✓ {result.file_path} (git auto-merged)")) - else: - ai_merged_count += 1 - print(success(f" ✓ {result.file_path} (AI merged)")) - else: - print(error(f" ✗ {result.file_path}: {result.error}")) - remaining_conflicts.append({ - "file": result.file_path, - "reason": result.error or "AI could not resolve the conflict", - "severity": "high", - }) - - # Print summary - print() - print(muted(f" Parallel merge completed in {elapsed:.1f}s")) - print(muted(f" Git auto-merged: {auto_merged_count}")) - print(muted(f" AI merged: {ai_merged_count}")) - if remaining_conflicts: - print(muted(f" Failed: {len(remaining_conflicts)}")) - - # ALWAYS process non-conflicting files, even if some conflicts failed - # This ensures we get as much of the build as possible - # (New files were already copied at the start) - print(muted(" Merging remaining files...")) - - # Get list of modified/deleted files (new files already copied at start) - non_conflicting = [ - (f, s) for f, s in changed_files - if f not in conflicting_files and s != "A" # Skip new files, already copied - ] - - for file_path, status in non_conflicting: - try: - if status == "D": - # Deleted in worktree - target_path = project_dir / file_path - if target_path.exists(): - target_path.unlink() - subprocess.run(["git", "add", file_path], cwd=project_dir, capture_output=True) - else: - # Added or modified - copy from worktree - content = _get_file_content_from_ref(project_dir, spec_branch, file_path) - if content is not None: - target_path = project_dir / file_path - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_text(content, encoding="utf-8") - subprocess.run(["git", "add", file_path], cwd=project_dir, capture_output=True) - resolved_files.append(file_path) - except Exception as e: - print(muted(f" Warning: Could not process {file_path}: {e}")) - - # V2: Record merge completion in Evolution Tracker for future context - if resolved_files: - _record_merge_completion(project_dir, spec_name, resolved_files) - - # Build result - partial success if some files failed but we got others - result = { - "success": len(remaining_conflicts) == 0, - "resolved_files": resolved_files, - "stats": { - "files_merged": len(resolved_files), - "conflicts_resolved": len(conflicting_files) - len(remaining_conflicts), - "ai_assisted": ai_merged_count, - "auto_merged": auto_merged_count, - "parallel_ai_merges": len(files_needing_ai_merge), - }, - } - - # Add remaining conflicts if any (for UI to show what needs manual attention) - if remaining_conflicts: - result["remaining_conflicts"] = remaining_conflicts - result["partial_success"] = len(resolved_files) > 0 - print() - print(warning(f" ⚠ {len(remaining_conflicts)} file(s) could not be auto-merged:")) - for conflict in remaining_conflicts: - print(muted(f" - {conflict['file']}: {conflict['reason']}")) - print(muted(" These files may need manual review.")) - - return result - - -def _get_file_content_from_ref(project_dir: Path, ref: str, file_path: str) -> Optional[str]: - """Get file content from a git ref (branch, commit, etc.).""" - import subprocess - - result = subprocess.run( - ["git", "show", f"{ref}:{file_path}"], - cwd=project_dir, - capture_output=True, - text=True, - ) - if result.returncode == 0: - return result.stdout - return None - - -def _get_changed_files_from_branch( - project_dir: Path, - base_branch: str, - spec_branch: str, -) -> list[tuple[str, str]]: - """Get list of changed files between branches.""" - import subprocess - - result = subprocess.run( - ["git", "diff", "--name-status", f"{base_branch}...{spec_branch}"], - cwd=project_dir, - capture_output=True, - text=True, - ) - - files = [] - if result.returncode == 0: - for line in result.stdout.strip().split("\n"): - if line: - parts = line.split("\t", 1) - if len(parts) == 2: - files.append((parts[1], parts[0])) # (file_path, status) - return files - - -# Constants for merge limits -MAX_FILE_LINES_FOR_AI = 5000 # Skip AI for files larger than this -MAX_PARALLEL_AI_MERGES = 5 # Limit concurrent AI merge operations - -BINARY_EXTENSIONS = { - '.png', '.jpg', '.jpeg', '.gif', '.ico', '.webp', '.bmp', '.svg', - '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', - '.zip', '.tar', '.gz', '.rar', '.7z', - '.exe', '.dll', '.so', '.dylib', '.bin', - '.mp3', '.mp4', '.wav', '.avi', '.mov', '.mkv', - '.woff', '.woff2', '.ttf', '.otf', '.eot', - '.pyc', '.pyo', '.class', '.o', '.obj', -} - -# Merge lock timeout in seconds -MERGE_LOCK_TIMEOUT = 300 # 5 minutes - - -class MergeLock: - """ - Context manager for merge locking to prevent concurrent merges. - - Uses a lock file in .auto-claude/ to ensure only one merge operation - runs at a time for a given project. - """ - - def __init__(self, project_dir: Path, spec_name: str): - self.project_dir = project_dir - self.spec_name = spec_name - self.lock_dir = project_dir / ".auto-claude" / ".locks" - self.lock_file = self.lock_dir / f"merge-{spec_name}.lock" - self.acquired = False - - def __enter__(self): - """Acquire the merge lock.""" - import time - import os - - self.lock_dir.mkdir(parents=True, exist_ok=True) - - # Check if lock exists and is stale - if self.lock_file.exists(): - try: - lock_data = json.loads(self.lock_file.read_text()) - lock_time = lock_data.get("timestamp", 0) - lock_pid = lock_data.get("pid", 0) - - # Check if lock is stale (older than timeout) - if time.time() - lock_time > MERGE_LOCK_TIMEOUT: - print(muted(f" Removing stale merge lock (older than {MERGE_LOCK_TIMEOUT}s)")) - self.lock_file.unlink() - # Check if locking process is still alive - elif lock_pid and not _is_process_running(lock_pid): - print(muted(f" Removing orphaned merge lock (PID {lock_pid} not running)")) - self.lock_file.unlink() - else: - raise MergeLockError( - f"Another merge operation is in progress for {self.spec_name}. " - f"If this is an error, delete {self.lock_file}" - ) - except json.JSONDecodeError: - # Corrupted lock file, remove it - self.lock_file.unlink() - - # Create lock file - lock_data = { - "spec_name": self.spec_name, - "timestamp": time.time(), - "pid": os.getpid(), - } - self.lock_file.write_text(json.dumps(lock_data)) - self.acquired = True - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Release the merge lock.""" - if self.acquired and self.lock_file.exists(): - try: - self.lock_file.unlink() - except Exception: - pass # Best effort cleanup - return False - - -class MergeLockError(Exception): - """Raised when a merge lock cannot be acquired.""" - pass - - -@dataclass -class ParallelMergeTask: - """A file merge task to be executed in parallel.""" - file_path: str - main_content: str - worktree_content: str - base_content: Optional[str] - spec_name: str - - -@dataclass -class ParallelMergeResult: - """Result of a parallel merge task.""" - file_path: str - merged_content: Optional[str] - success: bool - error: Optional[str] = None - was_auto_merged: bool = False # True if git auto-merged without AI - - -async def _create_async_claude_client(): - """Create an async Claude client for merge resolution.""" - import os - - oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") - if not oauth_token: - debug_warning(MODULE, "CLAUDE_CODE_OAUTH_TOKEN not set") - return None - - try: - from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient - - client = ClaudeSDKClient( - options=ClaudeAgentOptions( - model="sonnet", - system_prompt="You are an expert code merge assistant. Be concise and precise.", - allowed_tools=[], # No tools needed for merge - max_turns=1, - ) - ) - return client - except ImportError: - debug_warning(MODULE, "claude_agent_sdk not installed") - return None - - -async def _async_ai_call(client, system: str, user: str) -> str: - """Make an async AI call using an existing client.""" - try: - await client.query(user) - - response_text = "" - async for msg in client.receive_response(): - msg_type = type(msg).__name__ - if msg_type == "AssistantMessage" and hasattr(msg, "content"): - for block in msg.content: - if hasattr(block, "text"): - response_text += block.text - - return response_text - except Exception as e: - debug_error(MODULE, f"Async AI call failed: {e}") - return "" - - -async def _merge_file_with_ai_async( - task: ParallelMergeTask, - project_dir: Path, - semaphore: asyncio.Semaphore, -) -> ParallelMergeResult: - """ - Async version of file merge with AI. - - Uses a semaphore to limit concurrent AI calls. - """ - from merge.prompts import ( - build_conflict_only_prompt, - parse_conflict_markers, - reassemble_with_resolutions, - extract_conflict_resolutions, - build_simple_merge_prompt, - ) - from merge import AIResolver - - file_path = task.file_path - main_content = task.main_content - worktree_content = task.worktree_content - base_content = task.base_content - spec_name = task.spec_name - - debug(MODULE, f"[PARALLEL] Starting async merge for: {file_path}") - - # Quick checks before acquiring semaphore - if _is_binary_file(file_path): - return ParallelMergeResult( - file_path=file_path, - merged_content=None, - success=False, - error="Binary file - skipped" - ) - - main_lines = main_content.count('\n') if main_content else 0 - worktree_lines = worktree_content.count('\n') if worktree_content else 0 - max_lines = max(main_lines, worktree_lines) - - if max_lines > MAX_FILE_LINES_FOR_AI: - return ParallelMergeResult( - file_path=file_path, - merged_content=None, - success=False, - error=f"File too large ({max_lines} lines)" - ) - - # Try git merge-file first (doesn't need AI) - if base_content: - merged_content, has_conflicts = _create_conflict_file_with_git( - main_content, worktree_content, base_content, project_dir - ) - - # Case 1: Git cleanly merged - no AI needed! - if merged_content and not has_conflicts: - is_valid, error_msg = _validate_merged_syntax(file_path, merged_content, project_dir) - if is_valid: - debug_success(MODULE, f"[PARALLEL] Git auto-merged (no AI): {file_path}") - return ParallelMergeResult( - file_path=file_path, - merged_content=merged_content, - success=True, - was_auto_merged=True - ) - else: - merged_content = None - has_conflicts = False - - # Acquire semaphore for AI calls - async with semaphore: - debug(MODULE, f"[PARALLEL] Acquired semaphore for AI merge: {file_path}") - - # Create client for this task - client = await _create_async_claude_client() - if not client: - return ParallelMergeResult( - file_path=file_path, - merged_content=_heuristic_merge(main_content, worktree_content, base_content), - success=True if _heuristic_merge(main_content, worktree_content, base_content) else False, - error="AI unavailable, used heuristic" - ) - - try: - async with client: - # Determine language - resolver = AIResolver() - language = resolver._infer_language(file_path) - - # Get task intent - task_intent = _get_task_intent(project_dir, spec_name) - - # Case 2: Has conflict markers - use conflict-only AI merge - if merged_content and has_conflicts: - conflicts, _ = parse_conflict_markers(merged_content) - - if conflicts: - total_conflict_lines = sum( - len(c['main_lines'].split('\n')) + len(c['worktree_lines'].split('\n')) - for c in conflicts - ) - savings_pct = 100 - (total_conflict_lines * 100 // max(max_lines, 1)) - - debug(MODULE, f"[PARALLEL] Conflict-only merge for {file_path}", - num_conflicts=len(conflicts), savings_pct=savings_pct) - - prompt = build_conflict_only_prompt( - file_path=file_path, - conflicts=conflicts, - spec_name=spec_name, - language=language, - task_intent=task_intent, - ) - - response = await _async_ai_call( - client, - "You are an expert code merge assistant. Resolve ONLY the specific conflicts shown.", - prompt, - ) - - if response: - resolutions = extract_conflict_resolutions(response, conflicts, language) - if resolutions: - merged = reassemble_with_resolutions(merged_content, conflicts, resolutions) - is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) - - # Retry loop: if syntax is invalid, give AI feedback to fix it - retry_count = 0 - while not is_valid and retry_count < MAX_SYNTAX_FIX_RETRIES: - retry_count += 1 - debug(MODULE, f"[PARALLEL] Syntax error in merge, retry {retry_count}/{MAX_SYNTAX_FIX_RETRIES}: {file_path}") - - # Build fix prompt with error feedback - fix_prompt = f"""The merged code you produced has a syntax error: - -{error_msg} - -Here is your merged code that has the error: -```{language} -{merged} -``` - -Please fix the syntax error and output the corrected code. Make sure: -1. All lines are properly separated (no concatenated lines) -2. All brackets/braces are properly matched -3. All statements end correctly - -Output ONLY the fixed code, no explanations.""" - - fix_response = await _async_ai_call( - client, - "You are an expert code fixer. Fix the syntax error in the code.", - fix_prompt, - ) - - if fix_response: - fixed = resolver._extract_code_block(fix_response, language) - if not fixed and resolver._looks_like_code(fix_response, language): - fixed = fix_response.strip() - if fixed: - merged = fixed - is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) - - if is_valid: - debug_success(MODULE, f"[PARALLEL] Conflict-only merge succeeded: {file_path}" + - (f" (after {retry_count} fix retries)" if retry_count > 0 else "")) - return ParallelMergeResult( - file_path=file_path, - merged_content=merged, - success=True - ) - else: - debug(MODULE, f"[PARALLEL] Conflict-only merge failed validation after {retry_count} retries, falling back to full-file: {file_path}") - - # Case 3: Full-file AI merge (fallback) - debug(MODULE, f"[PARALLEL] Full-file merge for: {file_path}") - - prompt = build_simple_merge_prompt( - file_path=file_path, - main_content=main_content, - worktree_content=worktree_content, - base_content=base_content, - spec_name=spec_name, - language=language, - task_intent=task_intent, - ) - - response = await _async_ai_call( - client, - "You are an expert code merge assistant. Output only the merged code.", - prompt, - ) - - if response: - merged = resolver._extract_code_block(response, language) - if not merged and resolver._looks_like_code(response, language): - merged = response.strip() - - if merged: - is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) - - # Retry loop: if syntax is invalid, give AI feedback to fix it - retry_count = 0 - while not is_valid and retry_count < MAX_SYNTAX_FIX_RETRIES: - retry_count += 1 - debug(MODULE, f"[PARALLEL] Syntax error in full-file merge, retry {retry_count}/{MAX_SYNTAX_FIX_RETRIES}: {file_path}") - - # Build fix prompt with error feedback - fix_prompt = f"""The merged code you produced has a syntax error: - -{error_msg} - -Here is your merged code that has the error: -```{language} -{merged} -``` - -Please fix the syntax error and output the corrected code. Make sure: -1. All lines are properly separated (no concatenated lines) -2. All brackets/braces are properly matched -3. All statements end correctly - -Output ONLY the fixed code, no explanations.""" - - fix_response = await _async_ai_call( - client, - "You are an expert code fixer. Fix the syntax error in the code.", - fix_prompt, - ) - - if fix_response: - fixed = resolver._extract_code_block(fix_response, language) - if not fixed and resolver._looks_like_code(fix_response, language): - fixed = fix_response.strip() - if fixed: - merged = fixed - is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) - - if is_valid: - debug_success(MODULE, f"[PARALLEL] Full-file merge succeeded: {file_path}" + - (f" (after {retry_count} fix retries)" if retry_count > 0 else "")) - return ParallelMergeResult( - file_path=file_path, - merged_content=merged, - success=True - ) - else: - debug_error(MODULE, f"[PARALLEL] Full-file merge failed validation after {retry_count} retries: {file_path}") - - # AI couldn't merge - return ParallelMergeResult( - file_path=file_path, - merged_content=None, - success=False, - error="AI could not merge file - syntax validation failed after retries" - ) - - except Exception as e: - debug_error(MODULE, f"[PARALLEL] Async merge failed for {file_path}: {e}") - return ParallelMergeResult( - file_path=file_path, - merged_content=_heuristic_merge(main_content, worktree_content, base_content), - success=False, - error=str(e) - ) - - -async def _run_parallel_merges( - tasks: list[ParallelMergeTask], - project_dir: Path, - max_concurrent: int = MAX_PARALLEL_AI_MERGES, -) -> list[ParallelMergeResult]: - """ - Run multiple file merges in parallel. - - Uses asyncio.gather with a semaphore to limit concurrency. - - Args: - tasks: List of merge tasks to execute - project_dir: Project directory for validation - max_concurrent: Maximum concurrent AI calls - - Returns: - List of merge results in same order as tasks - """ - if not tasks: - return [] - - debug(MODULE, f"[PARALLEL] Starting {len(tasks)} parallel merges (max concurrent: {max_concurrent})") - - # Create semaphore to limit concurrent AI calls - semaphore = asyncio.Semaphore(max_concurrent) - - # Create coroutines for all tasks - coroutines = [ - _merge_file_with_ai_async(task, project_dir, semaphore) - for task in tasks - ] - - # Run all in parallel - results = await asyncio.gather(*coroutines, return_exceptions=True) - - # Convert exceptions to failed results - processed_results = [] - for i, result in enumerate(results): - if isinstance(result, Exception): - processed_results.append(ParallelMergeResult( - file_path=tasks[i].file_path, - merged_content=None, - success=False, - error=str(result) - )) - else: - processed_results.append(result) - - # Log summary - successful = sum(1 for r in processed_results if r.success) - auto_merged = sum(1 for r in processed_results if r.was_auto_merged) - debug_success(MODULE, f"[PARALLEL] Completed: {successful}/{len(tasks)} successful, {auto_merged} auto-merged") - - return processed_results - - -def _is_process_running(pid: int) -> bool: - """Check if a process with the given PID is running.""" - import os - try: - os.kill(pid, 0) - return True - except (OSError, ProcessLookupError): - return False - - -def _is_binary_file(file_path: str) -> bool: - """Check if a file is binary based on extension.""" - from pathlib import Path - return Path(file_path).suffix.lower() in BINARY_EXTENSIONS - - -def _record_merge_completion( - project_dir: Path, - spec_name: str, - merged_files: list[str], - task_intent: str = "", - merge_commit: str = "", -) -> None: - """ - Record completed merge in both Evolution Tracker and FileTimelineTracker. - - This enables future AI merges to understand the history of file changes, - creating a knowledge chain for intelligent conflict resolution. - - Args: - project_dir: Project root directory - spec_name: The task/spec that was merged - merged_files: List of file paths that were merged - task_intent: Description of what the task accomplished - merge_commit: The commit hash of the merge (for timeline tracking) - """ - # Get intent from implementation plan if not provided - if not task_intent: - intent_data = _get_task_intent(project_dir, spec_name) - if intent_data: - task_intent = intent_data.get("description", "") or intent_data.get("title", spec_name) - - # Track in FileEvolutionTracker (legacy system) - try: - tracker = FileEvolutionTracker(project_dir) - - # Mark the task as completed for all its tracked files - tracker.mark_task_completed(spec_name) - - # Record merge metadata for each file - for file_path in merged_files: - evolution = tracker.get_file_evolution(file_path) - if evolution: - # The task snapshot should already exist from refresh_from_git - # Just ensure it's marked as completed with intent - snapshot = evolution.get_task_snapshot(spec_name) - if snapshot: - snapshot.task_intent = task_intent - - # Save updates - tracker._save_evolutions() - - debug(MODULE, f"Recorded merge in FileEvolutionTracker", - spec_name=spec_name, files=len(merged_files)) - - except Exception as e: - debug_warning(MODULE, f"Could not record in FileEvolutionTracker: {e}") - - # Track in FileTimelineTracker (new intent-aware system) - try: - timeline_tracker = FileTimelineTracker(project_dir) - - # Get merge commit if not provided - if not merge_commit: - import subprocess - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=project_dir, - capture_output=True, - text=True, - ) - merge_commit = result.stdout.strip() if result.returncode == 0 else "unknown" - - # Mark task as merged in timeline tracker - timeline_tracker.on_task_merged(spec_name, merge_commit) - - debug(MODULE, f"Recorded merge in FileTimelineTracker", - spec_name=spec_name, merge_commit=merge_commit[:8]) - print(muted(f" Recorded merge completion for {len(merged_files)} files")) - - except Exception as e: - # Non-fatal - this is supplementary tracking - debug_warning(MODULE, f"Could not record in FileTimelineTracker: {e}") - print(muted(f" Note: Could not record merge completion: {e}")) - - -def _get_task_intent(project_dir: Path, spec_name: str) -> Optional[dict]: - """ - Load task intent from implementation_plan.json. - - Returns dict with: - - title: Task title - - description: What the task does - - files_to_modify: List of files the task planned to modify - - current_subtask: What the agent was working on - """ - try: - # Try worktree location first, then main project - for base_path in [ - project_dir / ".worktrees" / spec_name / ".auto-claude" / "specs" / spec_name, - project_dir / ".auto-claude" / "specs" / spec_name, - ]: - plan_path = base_path / "implementation_plan.json" - if plan_path.exists(): - with open(plan_path) as f: - plan = json.load(f) - - # Extract key intent information - intent = { - "title": plan.get("title", spec_name), - "description": plan.get("description", ""), - "files_to_modify": [], - "subtasks": [], - } - - # Get files_to_modify from phases/subtasks - for phase in plan.get("phases", []): - for subtask in phase.get("subtasks", []): - intent["subtasks"].append({ - "title": subtask.get("title", ""), - "description": subtask.get("description", ""), - "status": subtask.get("status", "pending"), - }) - # Extract files from subtask if present - files = subtask.get("files", []) - intent["files_to_modify"].extend(files) - - # Also check spec.md for high-level context - spec_path = base_path / "spec.md" - if spec_path.exists(): - spec_content = spec_path.read_text() - # Extract first paragraph as summary - lines = spec_content.split("\n\n") - if len(lines) > 1: - intent["spec_summary"] = lines[1][:500] # First content paragraph - - return intent - - return None - except Exception as e: - print(muted(f" Could not load task intent: {e}")) - return None - - -def _get_recent_merges_context(project_dir: Path, file_path: str, limit: int = 3) -> list[dict]: - """ - Get context about recent merges that touched this file. - - Uses the FileEvolutionTracker to retrieve historical information about - recent tasks that have modified this file. This enables the AI to understand - the file's evolution when resolving merge conflicts. - - Args: - project_dir: Project root directory - file_path: Path to the file (relative or absolute) - limit: Maximum number of recent merges to return - - Returns: - List of {task_id, intent, timestamp, changes} for recent tasks that modified this file. - """ - try: - from merge import FileEvolutionTracker - - tracker = FileEvolutionTracker(project_dir) - evolution = tracker.get_file_evolution(file_path) - - if not evolution: - return [] - - # Get task snapshots that have completed modifications - completed_snapshots = [ - ts for ts in evolution.task_snapshots - if ts.completed_at is not None and ts.semantic_changes - ] - - # Sort by completion time (most recent first) - completed_snapshots.sort( - key=lambda ts: ts.completed_at or ts.started_at, - reverse=True - ) - - # Limit results - recent = completed_snapshots[:limit] - - # Build context for each merge - result = [] - for snapshot in recent: - # Summarize the semantic changes - change_summary = [] - for change in snapshot.semantic_changes[:5]: # Limit to 5 changes - change_summary.append( - f"{change.change_type.value}: {change.symbol_name or change.description}" - ) - - result.append({ - "task_id": snapshot.task_id, - "intent": snapshot.task_intent, - "timestamp": (snapshot.completed_at or snapshot.started_at).isoformat(), - "changes": change_summary, - }) - - return result - - except Exception as e: - # Log but don't fail - this is supplementary context - print(muted(f" Could not load merge history for {file_path}: {e}")) - return [] - - -def _validate_merged_syntax(file_path: str, content: str, project_dir: Path) -> tuple[bool, str]: - """ - Validate the syntax of merged code. - - Returns (is_valid, error_message). - """ - import subprocess - import tempfile - from pathlib import Path as P - - ext = P(file_path).suffix.lower() - - # TypeScript/JavaScript validation - if ext in {'.ts', '.tsx', '.js', '.jsx'}: - try: - # Write to temp file in system temp dir (NOT project dir to avoid HMR triggers) - with tempfile.NamedTemporaryFile( - mode='w', - suffix=ext, - delete=False, - # Don't set dir= to avoid writing to project directory which triggers HMR - ) as tmp: - tmp.write(content) - tmp_path = tmp.name - - try: - # Try tsc first (TypeScript) - if ext in {'.ts', '.tsx'}: - result = subprocess.run( - ['npx', 'tsc', '--noEmit', '--skipLibCheck', tmp_path], - cwd=project_dir, - capture_output=True, - text=True, - timeout=30, - ) - if result.returncode != 0: - # Filter out npm warnings (they go to stderr but aren't errors) - error_lines = [ - line for line in result.stderr.strip().split('\n') - if line and not line.startswith('npm warn') and not line.startswith('npm WARN') - ] - # Only treat as error if there are actual TypeScript errors - if error_lines: - return False, '\n'.join(error_lines[:3]) - # No actual errors, just npm warnings - syntax is valid - - # Try eslint for all JS/TS - result = subprocess.run( - ['npx', 'eslint', '--no-eslintrc', '--parser', '@typescript-eslint/parser', tmp_path], - cwd=project_dir, - capture_output=True, - text=True, - timeout=30, - ) - # eslint exit 1 for errors, 0 for clean - if result.returncode > 1: # 2+ is config error, ignore - pass - elif result.returncode == 1 and 'Parsing error' in result.stdout: - return False, "Syntax error in merged code" - - finally: - P(tmp_path).unlink(missing_ok=True) - - return True, "" - - except subprocess.TimeoutExpired: - return True, "" # Timeout = assume ok - except FileNotFoundError: - return True, "" # No tsc/eslint = skip validation - except Exception as e: - return True, "" # Other errors = skip validation - - # Python validation - elif ext == '.py': - try: - compile(content, file_path, 'exec') - return True, "" - except SyntaxError as e: - return False, f"Python syntax error: {e.msg} at line {e.lineno}" - - # JSON validation - elif ext == '.json': - try: - json.loads(content) - return True, "" - except json.JSONDecodeError as e: - return False, f"JSON error: {e.msg} at line {e.lineno}" - - # Other file types - skip validation - return True, "" - - -def _create_conflict_file_with_git( - main_content: str, - worktree_content: str, - base_content: Optional[str], - project_dir: Path, -) -> tuple[Optional[str], bool]: - """ - Use git merge-file to create a file with conflict markers. - - This produces a file with standard git conflict markers that can be - parsed to extract only the conflicting regions. - - Returns: - Tuple of (merged_content, has_conflicts): - - (content, True) if there are conflict markers - - (content, False) if git cleanly merged (USE THIS RESULT!) - - (None, False) if merge failed - """ - import subprocess - import tempfile - - if not base_content: - # Without a base, we can't do proper 3-way merge with markers - debug(MODULE, "git merge-file: no base content available") - return None, False - - try: - # Create temp files for git merge-file - with tempfile.NamedTemporaryFile(mode='w', suffix='.main', delete=False) as main_f: - main_f.write(main_content) - main_path = main_f.name - - with tempfile.NamedTemporaryFile(mode='w', suffix='.base', delete=False) as base_f: - base_f.write(base_content) - base_path = base_f.name - - with tempfile.NamedTemporaryFile(mode='w', suffix='.worktree', delete=False) as worktree_f: - worktree_f.write(worktree_content) - worktree_path = worktree_f.name - - try: - # git merge-file modifies the first file in place - # Returns 0 if no conflicts, >0 if conflicts - result = subprocess.run( - ['git', 'merge-file', '-p', main_path, base_path, worktree_path], - cwd=project_dir, - capture_output=True, - text=True, - timeout=30, - ) - - debug(MODULE, "git merge-file result", - return_code=result.returncode, - output_length=len(result.stdout) if result.stdout else 0, - has_conflict_markers='<<<<<<' in result.stdout if result.stdout else False) - - # Return code > 0 means conflicts exist - if result.returncode > 0 and '<<<<<<' in result.stdout: - debug(MODULE, "git merge-file: has conflicts") - return result.stdout, True - elif result.returncode == 0: - # Clean merge, no conflicts - THIS IS STILL USEFUL! - debug(MODULE, "git merge-file: clean merge (no conflicts)") - return result.stdout, False - else: - # Some error occurred - debug_warning(MODULE, "git merge-file: unexpected result", - return_code=result.returncode, - stderr=result.stderr[:200] if result.stderr else None) - return None, False - - finally: - # Cleanup temp files - Path(main_path).unlink(missing_ok=True) - Path(base_path).unlink(missing_ok=True) - Path(worktree_path).unlink(missing_ok=True) - - except Exception as e: - debug_warning(MODULE, f"git merge-file failed: {e}") - - return None, False - - -def _merge_file_with_ai( - file_path: str, - main_content: str, - worktree_content: str, - base_content: Optional[str], - spec_name: str, - orchestrator: MergeOrchestrator, - project_dir: Optional[Path] = None, -) -> Optional[str]: - """ - Use AI to merge a conflicting file. - - OPTIMIZED: First tries to identify specific conflict regions and only - sends those to the AI, rather than regenerating the entire file. - - This enhanced version includes: - - Conflict-region-only merging (FAST - only sends conflict lines) - - Fallback to full-file merge for complex cases - - FileTimelineTracker context for full situational awareness - - Task intent from implementation_plan.json - - Binary file detection - - File size limits - - Syntax validation after merge - - Returns merged content, or None if AI couldn't resolve. - """ - from merge import create_claude_resolver - from merge.prompts import ( - build_timeline_merge_prompt, - build_simple_merge_prompt, - build_conflict_only_prompt, - parse_conflict_markers, - reassemble_with_resolutions, - extract_conflict_resolutions, - ) - - debug(MODULE, f"AI merge starting for: {file_path}", - spec_name=spec_name, - has_base_content=base_content is not None) - - # Quick Win 2: Binary file detection - if _is_binary_file(file_path): - debug_warning(MODULE, "Skipping binary file", file_path=file_path) - print(warning(f" Binary file detected, skipping AI merge")) - return None - - # Quick Win 4: File size limit - main_lines = main_content.count('\n') if main_content else 0 - worktree_lines = worktree_content.count('\n') if worktree_content else 0 - max_lines = max(main_lines, worktree_lines) - - debug_detailed(MODULE, "File size check", - main_lines=main_lines, - worktree_lines=worktree_lines, - max_allowed=MAX_FILE_LINES_FOR_AI) - - if max_lines > MAX_FILE_LINES_FOR_AI: - debug_warning(MODULE, "File too large for AI merge", - max_lines=max_lines, - limit=MAX_FILE_LINES_FOR_AI) - print(warning(f" File too large ({max_lines} lines > {MAX_FILE_LINES_FOR_AI}), skipping AI merge")) - return None - - # Create an AI resolver - resolver = create_claude_resolver() - - if not resolver.ai_call_fn: - debug_warning(MODULE, "AI not available, using heuristic merge") - print(muted(f" AI not available, trying heuristic merge...")) - return _heuristic_merge(main_content, worktree_content, base_content) - - # Determine language - language = resolver._infer_language(file_path) - debug(MODULE, "Detected language", language=language) - - # Get task intent for context - task_intent = None - if project_dir: - task_intent = _get_task_intent(project_dir, spec_name) - - # OPTIMIZATION: Try git merge-file first - # This can either: - # 1. Cleanly merge (no AI needed - FASTEST) - # 2. Produce conflict markers (only send conflict regions to AI - FAST) - # 3. Fail (fall back to full-file AI merge - SLOW) - if project_dir and base_content: - merged_content, has_conflicts = _create_conflict_file_with_git( - main_content, worktree_content, base_content, project_dir - ) - - # Case 1: Git cleanly merged - no AI needed! - if merged_content and not has_conflicts: - debug_success(MODULE, "Git merge-file cleanly merged (no AI needed)", - file_path=file_path) - print(success(f" ✓ Git auto-merged (no conflicts)")) - - # Validate syntax before returning - is_valid, error_msg = _validate_merged_syntax(file_path, merged_content, project_dir) - if is_valid: - return merged_content - else: - debug_warning(MODULE, "Git auto-merge produced invalid syntax, falling back to AI", - error=error_msg) - print(muted(f" Git auto-merge had syntax issues, trying AI...")) - - # Case 2: Has conflict markers - use conflict-only AI merge (FAST) - if merged_content and has_conflicts: - conflicts, _ = parse_conflict_markers(merged_content) - - if conflicts: - # Calculate how much smaller this approach is - total_conflict_lines = sum( - len(c['main_lines'].split('\n')) + len(c['worktree_lines'].split('\n')) - for c in conflicts - ) - savings_pct = 100 - (total_conflict_lines * 100 // max(max_lines, 1)) - - debug(MODULE, "Using conflict-only merge (optimized)", - num_conflicts=len(conflicts), - conflict_lines=total_conflict_lines, - file_lines=max_lines, - savings_pct=savings_pct) - print(muted(f" Found {len(conflicts)} conflict region(s) ({total_conflict_lines} lines vs {max_lines} total - {savings_pct}% smaller prompt)")) - - # Build focused prompt with only conflict regions - prompt = build_conflict_only_prompt( - file_path=file_path, - conflicts=conflicts, - spec_name=spec_name, - language=language, - task_intent=task_intent, - ) - - try: - debug(MODULE, "Calling AI for conflict-only merge") - response = resolver.ai_call_fn( - "You are an expert code merge assistant. Resolve ONLY the specific conflicts shown. Output the resolved code for each conflict.", - prompt, - ) - - if response: - debug(MODULE, "Conflict-only AI response received", - response_length=len(response), - preview=response[:200] if len(response) > 200 else response) - - # Extract resolutions for each conflict - resolutions = extract_conflict_resolutions(response, conflicts, language) - - if resolutions: - debug(MODULE, "Extracted conflict resolutions", - num_resolutions=len(resolutions), - expected=len(conflicts)) - - # Reassemble the file with resolved conflicts - merged = reassemble_with_resolutions(merged_content, conflicts, resolutions) - - # Validate syntax - if project_dir: - is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) - if is_valid: - debug_success(MODULE, "Conflict-only merge succeeded", - file_path=file_path, - conflicts_resolved=len(resolutions)) - print(success(f" ✓ Resolved {len(resolutions)} conflict(s)")) - return merged - else: - debug_warning(MODULE, "Conflict-only merge produced invalid syntax, falling back", - error=error_msg) - print(muted(f" Conflict-only merge had syntax issues, trying full-file merge...")) - else: - return merged - else: - debug_warning(MODULE, "No resolutions extracted from AI response", - response_preview=response[:500] if len(response) > 500 else response) - print(muted(f" Could not extract conflict resolutions from AI response, trying full-file merge...")) - else: - debug_warning(MODULE, "Conflict-only AI returned empty response") - print(muted(f" AI returned empty response, trying full-file merge...")) - - except Exception as e: - debug_warning(MODULE, f"Conflict-only merge failed: {e}, falling back to full-file") - print(muted(f" Conflict-only merge failed, trying full-file merge...")) - - # FALLBACK: Full-file merge approach (slower but more comprehensive) - print(muted(f" Using full-file AI merge...")) - - # Try to get timeline context for richer merge prompt - timeline_context = None - if project_dir: - try: - tracker = FileTimelineTracker(project_dir) - timeline_context = tracker.get_merge_context(spec_name, file_path) - if timeline_context: - debug(MODULE, "Using FileTimelineTracker context", - commits_behind=timeline_context.total_commits_behind, - pending_tasks=timeline_context.total_pending_tasks, - main_events=len(timeline_context.main_evolution)) - except Exception as e: - debug_warning(MODULE, f"Could not get timeline context: {e}") - - # Build prompt - use timeline context if available, fallback to simple prompt - if timeline_context and timeline_context.total_commits_behind > 0: - # Use the rich timeline-based prompt with full situational awareness - debug(MODULE, "Building timeline-based merge prompt", - commits_behind=timeline_context.total_commits_behind, - main_events=len(timeline_context.main_evolution), - pending_tasks=timeline_context.total_pending_tasks) - print(muted(f" Using timeline context ({timeline_context.total_commits_behind} commits behind, {timeline_context.total_pending_tasks} pending tasks)")) - prompt = build_timeline_merge_prompt(timeline_context) - else: - # Fallback to simple three-way merge prompt - debug(MODULE, "Building simple merge prompt (no timeline context)") - - if task_intent: - debug(MODULE, "Loaded task intent", - title=task_intent.get('title'), - num_subtasks=len(task_intent.get('subtasks', []))) - - prompt = build_simple_merge_prompt( - file_path=file_path, - main_content=main_content, - worktree_content=worktree_content, - base_content=base_content, - spec_name=spec_name, - language=language, - task_intent=task_intent, - ) - - try: - debug(MODULE, "Calling AI for full-file merge", - file_path=file_path, - has_timeline_context=timeline_context is not None) - - response = resolver.ai_call_fn( - "You are an expert code merge assistant. Output only the merged code. The code MUST be syntactically valid.", - prompt, - ) - - debug_detailed(MODULE, "AI response received", - response_length=len(response) if response else 0) - - # Log response content for debugging (truncated) - if response: - preview = response[:200] if len(response) > 200 else response - print(f" [DEBUG] AI response preview: {repr(preview)}", file=sys.stderr) - else: - print(f" [DEBUG] AI response was empty", file=sys.stderr) - - # Extract code from response - merged = resolver._extract_code_block(response, language) - if not merged: - # If extraction failed, try using the whole response if it looks like code - if resolver._looks_like_code(response, language): - merged = response.strip() - - if not merged: - debug_error(MODULE, "Could not extract merged code from AI response") - print(muted(f" Could not extract merged code from AI response")) - return None - - debug(MODULE, "Extracted merged code", - merged_lines=merged.count('\n') + 1) - - # Quick Win 3: Validate syntax before returning - if project_dir: - debug(MODULE, "Validating merged syntax") - is_valid, error_msg = _validate_merged_syntax(file_path, merged, project_dir) - if not is_valid: - debug_warning(MODULE, "AI merge produced invalid syntax", - error=error_msg) - print(warning(f" AI merge produced invalid syntax: {error_msg}")) - print(muted(f" Retrying with syntax fix...")) - - # Try once more with explicit syntax fix request - retry_prompt = f'''The previous merge attempt produced invalid {language} code. -Error: {error_msg} - -Please fix the syntax error and output valid {language} code: - -{merged} - -Output ONLY the fixed code, wrapped in triple backticks: -```{language} -fixed code here -``` -''' - retry_response = resolver.ai_call_fn( - f"Fix the syntax error in this {language} code. Output only valid code.", - retry_prompt, - ) - retry_merged = resolver._extract_code_block(retry_response, language) - if retry_merged: - is_valid, _ = _validate_merged_syntax(file_path, retry_merged, project_dir) - if is_valid: - debug_success(MODULE, "Syntax fix retry succeeded", file_path=file_path) - return retry_merged - else: - debug_error(MODULE, "Syntax fix retry also failed", file_path=file_path) - print(warning(f" Retry also produced invalid syntax")) - return None - else: - debug_error(MODULE, "Could not extract code from retry response") - return None - - debug_success(MODULE, "AI merge completed successfully", - file_path=file_path, - merged_lines=merged.count('\n') + 1) - return merged - - except Exception as e: - debug_error(MODULE, "AI merge failed with exception", - file_path=file_path, - error=str(e)) - print(muted(f" AI merge failed: {e}")) - return _heuristic_merge(main_content, worktree_content, base_content) - - -def _heuristic_merge( - main_content: str, - worktree_content: str, - base_content: Optional[str], -) -> Optional[str]: - """ - Try a simple heuristic merge when AI is unavailable. - - This uses Python's difflib to attempt a three-way merge. - """ - import difflib - - if base_content is None: - # Without a base, we can't do a proper three-way merge - # Just prefer worktree content (the feature being merged) - return worktree_content - - try: - # Use diff3-style merge - main_lines = main_content.splitlines(keepends=True) - worktree_lines = worktree_content.splitlines(keepends=True) - base_lines = base_content.splitlines(keepends=True) - - # Simple approach: find what's changed in each branch and try to combine - # This is a simplified version - real diff3 is more complex - - # Get diffs from base to each branch - main_diff = list(difflib.unified_diff(base_lines, main_lines)) - worktree_diff = list(difflib.unified_diff(base_lines, worktree_lines)) - - # If one has no changes, use the other - if not main_diff: - return worktree_content - if not worktree_diff: - return main_content - - # If both have changes, this simple heuristic won't work reliably - # Return None to indicate AI is needed - return None - - except Exception: - return None - - -def _print_merge_success(no_commit: bool, stats: Optional[dict] = None) -> None: - """Print success message after merge.""" - print() - if stats: - print(muted(f" Files merged: {stats.get('files_merged', 0)}")) - if stats.get('auto_resolved'): - print(muted(f" Conflicts auto-resolved: {stats.get('auto_resolved', 0)}")) - print() - - if no_commit: - print_status("Changes are staged in your working directory.", "success") - print() - print("Review the changes in your IDE, then commit:") - print(highlight(" git commit -m 'your commit message'")) - print() - print("Or discard if not satisfied:") - print(muted(" git reset --hard HEAD")) - else: - print_status("Your feature has been added to your project.", "success") - - -def _print_conflict_info(result: dict) -> None: - """Print information about detected conflicts.""" - conflicts = result.get("conflicts", []) - - print() - print_status(f"Detected {len(conflicts)} conflict(s) that need attention:", "warning") - print() - - for i, conflict in enumerate(conflicts[:5], 1): # Show first 5 - file_path = conflict.get("file", "unknown") - location = conflict.get("location", "") - reason = conflict.get("reason", "") - severity = conflict.get("severity", "unknown") - - print(f" {i}. {highlight(file_path)}") - if location: - print(f" Location: {muted(location)}") - if reason: - print(f" Reason: {muted(reason)}") - print(f" Severity: {severity}") - print() - - if len(conflicts) > 5: - print(muted(f" ... and {len(conflicts) - 5} more")) - - -def review_existing_build(project_dir: Path, spec_name: str) -> bool: - """ - Show what an existing build contains. - - Called when user runs: python auto-claude/run.py --spec X --review - - Args: - project_dir: The project directory - spec_name: Name of the spec - - Returns: - True if build exists - """ - worktree_path = get_existing_build_worktree(project_dir, spec_name) - - if not worktree_path: - print() - print_status(f"No existing build found for '{spec_name}'.", "warning") - print() - print("To start a new build:") - print(highlight(f" python auto-claude/run.py --spec {spec_name}")) - return False - - content = [ - bold(f"{icon(Icons.FILE)} BUILD CONTENTS"), - ] - print() - print(box(content, width=60, style="heavy")) - - manager = WorktreeManager(project_dir) - worktree_info = manager.get_worktree_info(spec_name) - - show_build_summary(manager, spec_name) - show_changed_files(manager, spec_name) - - print() - print(muted("-" * 60)) - print() - print("To test the feature:") - print(highlight(f" cd {worktree_path}")) - print() - print("To add these changes to your project:") - print(highlight(f" python auto-claude/run.py --spec {spec_name} --merge")) - print() - print("To see full diff:") - if worktree_info: - print(muted(f" git diff {worktree_info.base_branch}...{worktree_info.branch}")) - print() - - return True - - -def discard_existing_build(project_dir: Path, spec_name: str) -> bool: - """ - Discard an existing build (with confirmation). - - Called when user runs: python auto-claude/run.py --spec X --discard - - Requires typing "delete" to confirm - prevents accidents. - - Args: - project_dir: The project directory - spec_name: Name of the spec - - Returns: - True if discarded - """ - worktree_path = get_existing_build_worktree(project_dir, spec_name) - - if not worktree_path: - print() - print_status(f"No existing build found for '{spec_name}'.", "warning") - return False - - content = [ - warning(f"{icon(Icons.WARNING)} DELETE BUILD RESULTS?"), - "", - "This will permanently delete all work for this build.", - ] - print() - print(box(content, width=60, style="heavy")) - - manager = WorktreeManager(project_dir) - - show_build_summary(manager, spec_name) - - print() - print(f"Are you sure? Type {highlight('delete')} to confirm: ", end="") - - try: - confirmation = input().strip().lower() - except KeyboardInterrupt: - print() - print_status("Cancelled. Your build is still saved.", "info") - return False - - if confirmation != "delete": - print() - print_status("Cancelled. Your build is still saved.", "info") - return False - - # Actually delete - manager.remove_worktree(spec_name, delete_branch=True) - - print() - print_status("Build deleted.", "success") - return True - - -def check_existing_build(project_dir: Path, spec_name: str) -> bool: - """ - Check if there's an existing build and offer options. - - Returns True if user wants to continue with existing build, - False if they want to start fresh (after discarding). - """ - worktree_path = get_existing_build_worktree(project_dir, spec_name) - - if not worktree_path: - return False # No existing build - - content = [ - info(f"{icon(Icons.INFO)} EXISTING BUILD FOUND"), - "", - "There's already a build in progress for this spec.", - ] - print() - print(box(content, width=60, style="heavy")) - - options = [ - MenuOption( - key="continue", - label="Continue where it left off", - icon=Icons.PLAY, - description="Resume building from the last checkpoint", - ), - MenuOption( - key="review", - label="Review what was built", - icon=Icons.FILE, - description="See the files that were created/modified", - ), - MenuOption( - key="merge", - label="Add to my project now", - icon=Icons.SUCCESS, - description="Merge the existing build into your project", - ), - MenuOption( - key="fresh", - label="Start fresh", - icon=Icons.ERROR, - description="Discard current build and start over", - ), - ] - - print() - choice = select_menu( - title="What would you like to do?", - options=options, - allow_quit=True, - ) - - if choice is None: - print() - print_status("Cancelled.", "info") - sys.exit(0) - - if choice == "continue": - return True # Continue with existing - elif choice == "review": - review_existing_build(project_dir, spec_name) - print() - input("Press Enter to continue building...") - return True - elif choice == "merge": - merge_existing_build(project_dir, spec_name) - return False # Start fresh after merge - elif choice == "fresh": - discarded = discard_existing_build(project_dir, spec_name) - return not discarded # If discarded, start fresh - else: - return True # Default to continue - - -def list_all_worktrees(project_dir: Path) -> list[WorktreeInfo]: - """ - List all spec worktrees in the project. - - Args: - project_dir: Main project directory - - Returns: - List of WorktreeInfo for each spec worktree - """ - manager = WorktreeManager(project_dir) - return manager.list_all_worktrees() - - -def cleanup_all_worktrees(project_dir: Path, confirm: bool = True) -> bool: - """ - Remove all worktrees and their branches. - - Args: - project_dir: Main project directory - confirm: Whether to ask for confirmation - - Returns: - True if cleanup succeeded - """ - manager = WorktreeManager(project_dir) - worktrees = manager.list_all_worktrees() - - if not worktrees: - print_status("No worktrees found.", "info") - return True - - print() - print("=" * 70) - print(" CLEANUP ALL WORKTREES") - print("=" * 70) - - content = [ - warning(f"{icon(Icons.WARNING)} THIS WILL DELETE ALL BUILD WORKTREES"), - "", - "The following will be removed:", - ] - for wt in worktrees: - content.append(f" - {wt.spec_name} ({wt.branch})") - - print() - print(box(content, width=70, style="heavy")) - - if confirm: - print() - response = input(" Type 'cleanup' to confirm: ").strip() - if response != "cleanup": - print_status("Cleanup cancelled.", "info") - return False - - manager.cleanup_all() - - print() - print_status("All worktrees cleaned up.", "success") - return True +from types import ModuleType + +# Ensure auto-claude is in sys.path +_auto_claude_dir = Path(__file__).parent +if str(_auto_claude_dir) not in sys.path: + sys.path.insert(0, str(_auto_claude_dir)) + +# Create a minimal 'core' module if it doesn't exist (to avoid importing core/__init__.py) +if 'core' not in sys.modules: + _core_module = ModuleType('core') + _core_module.__file__ = str(_auto_claude_dir / 'core' / '__init__.py') + _core_module.__path__ = [str(_auto_claude_dir / 'core')] + sys.modules['core'] = _core_module + +# Now load core.workspace package directly +_workspace_init = _auto_claude_dir / 'core' / 'workspace' / '__init__.py' +_spec = importlib.util.spec_from_file_location('core.workspace', _workspace_init) +_workspace_module = importlib.util.module_from_spec(_spec) +sys.modules['core.workspace'] = _workspace_module +_spec.loader.exec_module(_workspace_module) + +# Re-export everything from core.workspace +from core.workspace import * # noqa: F401, F403 + +__all__ = _workspace_module.__all__ diff --git a/auto-claude/worktree.py b/auto-claude/worktree.py index 2aaae53b..86205746 100644 --- a/auto-claude/worktree.py +++ b/auto-claude/worktree.py @@ -1,561 +1,42 @@ -#!/usr/bin/env python3 """ -Git Worktree Manager - Per-Spec Architecture -============================================= +Backward compatibility shim - import from core.worktree. -Each spec gets its own worktree: -- Worktree path: .worktrees/{spec-name}/ -- Branch name: auto-claude/{spec-name} +This file exists to maintain backward compatibility for code that imports +from 'worktree' instead of 'core.worktree'. -This allows: -1. Multiple specs to be worked on simultaneously -2. Each spec's changes are isolated -3. Branches persist until explicitly merged -4. Clear 1:1:1 mapping: spec → worktree → branch +IMPLEMENTATION: To avoid triggering core/__init__.py (which imports modules +with heavy dependencies like claude_agent_sdk), we: +1. Create a minimal fake 'core' module to satisfy Python's import system +2. Load core.worktree directly using importlib +3. Register it in sys.modules +4. Re-export everything + +This allows 'from worktree import X' to work without requiring all of core's dependencies. """ -import asyncio -import re -import shutil -import subprocess -from dataclasses import dataclass +import sys +import importlib.util from pathlib import Path - - -class WorktreeError(Exception): - """Error during worktree operations.""" - - pass - - -@dataclass -class WorktreeInfo: - """Information about a spec's worktree.""" - - path: Path - branch: str - spec_name: str - base_branch: str - is_active: bool = True - commit_count: int = 0 - files_changed: int = 0 - additions: int = 0 - deletions: int = 0 - - -class WorktreeManager: - """ - Manages per-spec Git worktrees. - - Each spec gets its own worktree in .worktrees/{spec-name}/ with - a corresponding branch auto-claude/{spec-name}. - """ - - def __init__(self, project_dir: Path, base_branch: str | None = None): - self.project_dir = project_dir - self.base_branch = base_branch or self._get_current_branch() - self.worktrees_dir = project_dir / ".worktrees" - self._merge_lock = asyncio.Lock() - - def _get_current_branch(self) -> str: - """Get the current git branch.""" - result = subprocess.run( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], - cwd=self.project_dir, - capture_output=True, - text=True, - ) - if result.returncode != 0: - raise WorktreeError(f"Failed to get current branch: {result.stderr}") - return result.stdout.strip() - - def _run_git( - self, args: list[str], cwd: Path | None = None - ) -> subprocess.CompletedProcess: - """Run a git command and return the result.""" - return subprocess.run( - ["git"] + args, - cwd=cwd or self.project_dir, - capture_output=True, - text=True, - ) - - def _unstage_gitignored_files(self) -> None: - """ - Unstage any staged files that are gitignored in the current branch. - - This is needed after a --no-commit merge because files that exist in the - source branch (like spec files in auto-claude/specs/) get staged even if - they're gitignored in the target branch. - """ - # Get list of staged files - result = self._run_git(["diff", "--cached", "--name-only"]) - if result.returncode != 0 or not result.stdout.strip(): - return - - staged_files = result.stdout.strip().split("\n") - - # Check which staged files are gitignored - # git check-ignore returns the files that ARE ignored - result = self._run_git(["check-ignore", "--stdin"], cwd=self.project_dir) - # We need to pass the files via stdin - result = subprocess.run( - ["git", "check-ignore", "--stdin"], - cwd=self.project_dir, - input="\n".join(staged_files), - capture_output=True, - text=True, - ) - - if not result.stdout.strip(): - return - - ignored_files = result.stdout.strip().split("\n") - - if ignored_files: - print(f"Unstaging {len(ignored_files)} gitignored file(s)...") - # Unstage each ignored file - for file in ignored_files: - if file.strip(): - self._run_git(["reset", "HEAD", "--", file.strip()]) - - def setup(self) -> None: - """Create worktrees directory if needed.""" - self.worktrees_dir.mkdir(exist_ok=True) - - # ==================== Per-Spec Worktree Methods ==================== - - def get_worktree_path(self, spec_name: str) -> Path: - """Get the worktree path for a spec.""" - return self.worktrees_dir / spec_name - - def get_branch_name(self, spec_name: str) -> str: - """Get the branch name for a spec.""" - return f"auto-claude/{spec_name}" - - def worktree_exists(self, spec_name: str) -> bool: - """Check if a worktree exists for a spec.""" - return self.get_worktree_path(spec_name).exists() - - def get_worktree_info(self, spec_name: str) -> WorktreeInfo | None: - """Get info about a spec's worktree.""" - worktree_path = self.get_worktree_path(spec_name) - if not worktree_path.exists(): - return None - - # Verify the branch exists in the worktree - result = self._run_git(["rev-parse", "--abbrev-ref", "HEAD"], cwd=worktree_path) - if result.returncode != 0: - return None - - actual_branch = result.stdout.strip() - - # Get statistics - stats = self._get_worktree_stats(spec_name) - - return WorktreeInfo( - path=worktree_path, - branch=actual_branch, - spec_name=spec_name, - base_branch=self.base_branch, - is_active=True, - **stats, - ) - - def _get_worktree_stats(self, spec_name: str) -> dict: - """Get diff statistics for a worktree.""" - worktree_path = self.get_worktree_path(spec_name) - - stats = { - "commit_count": 0, - "files_changed": 0, - "additions": 0, - "deletions": 0, - } - - if not worktree_path.exists(): - return stats - - # Commit count - result = self._run_git( - ["rev-list", "--count", f"{self.base_branch}..HEAD"], cwd=worktree_path - ) - if result.returncode == 0: - stats["commit_count"] = int(result.stdout.strip() or "0") - - # Diff stats - result = self._run_git( - ["diff", "--shortstat", f"{self.base_branch}...HEAD"], cwd=worktree_path - ) - if result.returncode == 0 and result.stdout.strip(): - # Parse: "3 files changed, 50 insertions(+), 10 deletions(-)" - match = re.search(r"(\d+) files? changed", result.stdout) - if match: - stats["files_changed"] = int(match.group(1)) - match = re.search(r"(\d+) insertions?", result.stdout) - if match: - stats["additions"] = int(match.group(1)) - match = re.search(r"(\d+) deletions?", result.stdout) - if match: - stats["deletions"] = int(match.group(1)) - - return stats - - def create_worktree(self, spec_name: str) -> WorktreeInfo: - """ - Create a worktree for a spec. - - Args: - spec_name: The spec folder name (e.g., "002-implement-memory") - - Returns: - WorktreeInfo for the created worktree - """ - worktree_path = self.get_worktree_path(spec_name) - branch_name = self.get_branch_name(spec_name) - - # Remove existing if present (from crashed previous run) - if worktree_path.exists(): - self._run_git(["worktree", "remove", "--force", str(worktree_path)]) - - # Delete branch if it exists (from previous attempt) - self._run_git(["branch", "-D", branch_name]) - - # Create worktree with new branch from base - result = self._run_git( - ["worktree", "add", "-b", branch_name, str(worktree_path), self.base_branch] - ) - - if result.returncode != 0: - raise WorktreeError( - f"Failed to create worktree for {spec_name}: {result.stderr}" - ) - - print(f"Created worktree: {worktree_path.name} on branch {branch_name}") - - return WorktreeInfo( - path=worktree_path, - branch=branch_name, - spec_name=spec_name, - base_branch=self.base_branch, - is_active=True, - ) - - def get_or_create_worktree(self, spec_name: str) -> WorktreeInfo: - """ - Get existing worktree or create a new one for a spec. - - Args: - spec_name: The spec folder name - - Returns: - WorktreeInfo for the worktree - """ - existing = self.get_worktree_info(spec_name) - if existing: - print(f"Using existing worktree: {existing.path}") - return existing - - return self.create_worktree(spec_name) - - def remove_worktree(self, spec_name: str, delete_branch: bool = False) -> None: - """ - Remove a spec's worktree. - - Args: - spec_name: The spec folder name - delete_branch: Whether to also delete the branch - """ - worktree_path = self.get_worktree_path(spec_name) - branch_name = self.get_branch_name(spec_name) - - if worktree_path.exists(): - result = self._run_git( - ["worktree", "remove", "--force", str(worktree_path)] - ) - if result.returncode == 0: - print(f"Removed worktree: {worktree_path.name}") - else: - print(f"Warning: Could not remove worktree: {result.stderr}") - shutil.rmtree(worktree_path, ignore_errors=True) - - if delete_branch: - self._run_git(["branch", "-D", branch_name]) - print(f"Deleted branch: {branch_name}") - - self._run_git(["worktree", "prune"]) - - def merge_worktree( - self, spec_name: str, delete_after: bool = False, no_commit: bool = False - ) -> bool: - """ - Merge a spec's worktree branch back to base branch. - - Args: - spec_name: The spec folder name - delete_after: Whether to remove worktree and branch after merge - no_commit: If True, merge changes but don't commit (stage only for review) - - Returns: - True if merge succeeded - """ - info = self.get_worktree_info(spec_name) - if not info: - print(f"No worktree found for spec: {spec_name}") - return False - - if no_commit: - print( - f"Merging {info.branch} into {self.base_branch} (staged, not committed)..." - ) - else: - print(f"Merging {info.branch} into {self.base_branch}...") - - # Switch to base branch in main project - result = self._run_git(["checkout", self.base_branch]) - if result.returncode != 0: - print(f"Error: Could not checkout base branch: {result.stderr}") - return False - - # Merge the spec branch - merge_args = ["merge", "--no-ff", info.branch] - if no_commit: - # --no-commit stages the merge but doesn't create the commit - merge_args.append("--no-commit") - else: - merge_args.extend(["-m", f"auto-claude: Merge {info.branch}"]) - - result = self._run_git(merge_args) - - if result.returncode != 0: - print("Merge conflict! Aborting merge...") - self._run_git(["merge", "--abort"]) - return False - - if no_commit: - # Unstage any files that are gitignored in the main branch - # These get staged during merge because they exist in the worktree branch - self._unstage_gitignored_files() - print( - f"Changes from {info.branch} are now staged in your working directory." - ) - print("Review the changes, then commit when ready:") - print(" git commit -m 'your commit message'") - else: - print(f"Successfully merged {info.branch}") - - if delete_after: - self.remove_worktree(spec_name, delete_branch=True) - - return True - - def commit_in_worktree(self, spec_name: str, message: str) -> bool: - """Commit all changes in a spec's worktree.""" - worktree_path = self.get_worktree_path(spec_name) - if not worktree_path.exists(): - return False - - self._run_git(["add", "."], cwd=worktree_path) - result = self._run_git(["commit", "-m", message], cwd=worktree_path) - - if result.returncode == 0: - return True - elif "nothing to commit" in result.stdout + result.stderr: - return True - else: - print(f"Commit failed: {result.stderr}") - return False - - # ==================== Listing & Discovery ==================== - - def list_all_worktrees(self) -> list[WorktreeInfo]: - """List all spec worktrees.""" - worktrees = [] - - if not self.worktrees_dir.exists(): - return worktrees - - for item in self.worktrees_dir.iterdir(): - if item.is_dir(): - info = self.get_worktree_info(item.name) - if info: - worktrees.append(info) - - return worktrees - - def list_all_spec_branches(self) -> list[str]: - """List all auto-claude branches (even if worktree removed).""" - result = self._run_git(["branch", "--list", "auto-claude/*"]) - if result.returncode != 0: - return [] - - branches = [] - for line in result.stdout.strip().split("\n"): - branch = line.strip().lstrip("* ") - if branch: - branches.append(branch) - - return branches - - def get_changed_files(self, spec_name: str) -> list[tuple[str, str]]: - """Get list of changed files in a spec's worktree.""" - worktree_path = self.get_worktree_path(spec_name) - if not worktree_path.exists(): - return [] - - result = self._run_git( - ["diff", "--name-status", f"{self.base_branch}...HEAD"], cwd=worktree_path - ) - - files = [] - for line in result.stdout.strip().split("\n"): - if not line: - continue - parts = line.split("\t", 1) - if len(parts) == 2: - files.append((parts[0], parts[1])) - - return files - - def get_change_summary(self, spec_name: str) -> dict: - """Get a summary of changes in a worktree.""" - files = self.get_changed_files(spec_name) - - new_files = sum(1 for status, _ in files if status == "A") - modified_files = sum(1 for status, _ in files if status == "M") - deleted_files = sum(1 for status, _ in files if status == "D") - - return { - "new_files": new_files, - "modified_files": modified_files, - "deleted_files": deleted_files, - } - - def cleanup_all(self) -> None: - """Remove all worktrees and their branches.""" - for worktree in self.list_all_worktrees(): - self.remove_worktree(worktree.spec_name, delete_branch=True) - - def cleanup_stale_worktrees(self) -> None: - """Remove worktrees that aren't registered with git.""" - if not self.worktrees_dir.exists(): - return - - # Get list of registered worktrees - result = self._run_git(["worktree", "list", "--porcelain"]) - registered_paths = set() - for line in result.stdout.split("\n"): - if line.startswith("worktree "): - registered_paths.add(Path(line.split(" ", 1)[1])) - - # Remove unregistered directories - for item in self.worktrees_dir.iterdir(): - if item.is_dir() and item not in registered_paths: - print(f"Removing stale worktree directory: {item.name}") - shutil.rmtree(item, ignore_errors=True) - - self._run_git(["worktree", "prune"]) - - def get_test_commands(self, spec_name: str) -> list[str]: - """Detect likely test/run commands for the project.""" - worktree_path = self.get_worktree_path(spec_name) - commands = [] - - if (worktree_path / "package.json").exists(): - commands.append("npm install && npm run dev") - commands.append("npm test") - - if (worktree_path / "requirements.txt").exists(): - commands.append("pip install -r requirements.txt") - - if (worktree_path / "Cargo.toml").exists(): - commands.append("cargo run") - commands.append("cargo test") - - if (worktree_path / "go.mod").exists(): - commands.append("go run .") - commands.append("go test ./...") - - if not commands: - commands.append("# Check the project's README for run instructions") - - return commands - - # ==================== Backward Compatibility ==================== - # These methods provide backward compatibility with the old single-worktree API - - def get_staging_path(self) -> Path | None: - """ - Backward compatibility: Get path to any existing spec worktree. - Prefer using get_worktree_path(spec_name) instead. - """ - worktrees = self.list_all_worktrees() - if worktrees: - return worktrees[0].path - return None - - def get_staging_info(self) -> WorktreeInfo | None: - """ - Backward compatibility: Get info about any existing spec worktree. - Prefer using get_worktree_info(spec_name) instead. - """ - worktrees = self.list_all_worktrees() - if worktrees: - return worktrees[0] - return None - - def merge_staging(self, delete_after: bool = True) -> bool: - """ - Backward compatibility: Merge first found worktree. - Prefer using merge_worktree(spec_name) instead. - """ - worktrees = self.list_all_worktrees() - if worktrees: - return self.merge_worktree(worktrees[0].spec_name, delete_after) - return False - - def remove_staging(self, delete_branch: bool = True) -> None: - """ - Backward compatibility: Remove first found worktree. - Prefer using remove_worktree(spec_name) instead. - """ - worktrees = self.list_all_worktrees() - if worktrees: - self.remove_worktree(worktrees[0].spec_name, delete_branch) - - def get_or_create_staging(self, spec_name: str) -> WorktreeInfo: - """ - Backward compatibility: Alias for get_or_create_worktree. - """ - return self.get_or_create_worktree(spec_name) - - def staging_exists(self) -> bool: - """ - Backward compatibility: Check if any spec worktree exists. - Prefer using worktree_exists(spec_name) instead. - """ - return len(self.list_all_worktrees()) > 0 - - def commit_in_staging(self, message: str) -> bool: - """ - Backward compatibility: Commit in first found worktree. - Prefer using commit_in_worktree(spec_name, message) instead. - """ - worktrees = self.list_all_worktrees() - if worktrees: - return self.commit_in_worktree(worktrees[0].spec_name, message) - return False - - def has_uncommitted_changes(self, in_staging: bool = False) -> bool: - """Check if there are uncommitted changes.""" - worktrees = self.list_all_worktrees() - if in_staging and worktrees: - cwd = worktrees[0].path - else: - cwd = None - result = self._run_git(["status", "--porcelain"], cwd=cwd) - return bool(result.stdout.strip()) - - -# Keep STAGING_WORKTREE_NAME for backward compatibility in imports -STAGING_WORKTREE_NAME = "auto-claude" +from types import ModuleType + +# Ensure auto-claude is in sys.path +_auto_claude_dir = Path(__file__).parent +if str(_auto_claude_dir) not in sys.path: + sys.path.insert(0, str(_auto_claude_dir)) + +# Create a minimal 'core' module if it doesn't exist (to avoid importing core/__init__.py) +if 'core' not in sys.modules: + _core_module = ModuleType('core') + _core_module.__file__ = str(_auto_claude_dir / 'core' / '__init__.py') + _core_module.__path__ = [str(_auto_claude_dir / 'core')] + sys.modules['core'] = _core_module + +# Now load core.worktree directly +_worktree_file = _auto_claude_dir / 'core' / 'worktree.py' +_spec = importlib.util.spec_from_file_location('core.worktree', _worktree_file) +_worktree_module = importlib.util.module_from_spec(_spec) +sys.modules['core.worktree'] = _worktree_module +_spec.loader.exec_module(_worktree_module) + +# Re-export everything from core.worktree +from core.worktree import * # noqa: F401, F403 diff --git a/tests/conftest.py b/tests/conftest.py index 524caaad..846b7c48 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,30 @@ from unittest.mock import MagicMock import pytest +# ============================================================================= +# PRE-MOCK EXTERNAL SDK MODULES - Must happen BEFORE adding auto-claude to path +# ============================================================================= +# These SDK modules may not be installed, so we mock them before any imports +# that might trigger loading code that depends on them. + +def _create_sdk_mock(): + """Create a comprehensive mock for SDK modules.""" + mock = MagicMock() + mock.ClaudeAgentOptions = MagicMock + mock.ClaudeSDKClient = MagicMock + mock.HookMatcher = MagicMock + return mock + +# Pre-mock claude_agent_sdk if not installed +if 'claude_agent_sdk' not in sys.modules: + sys.modules['claude_agent_sdk'] = _create_sdk_mock() + sys.modules['claude_agent_sdk.types'] = MagicMock() + +# Pre-mock claude_code_sdk if not installed +if 'claude_code_sdk' not in sys.modules: + sys.modules['claude_code_sdk'] = _create_sdk_mock() + sys.modules['claude_code_sdk.types'] = MagicMock() + # Add auto-claude directory to path for imports sys.path.insert(0, str(Path(__file__).parent.parent / "auto-claude")) @@ -85,7 +109,7 @@ def pytest_runtest_setup(item): 'test_qa_report_project_detection': {'claude_agent_sdk', 'ui', 'progress', 'task_logger', 'linear_updater', 'client'}, 'test_qa_report_manual_plan': {'claude_agent_sdk', 'ui', 'progress', 'task_logger', 'linear_updater', 'client'}, 'test_qa_report_config': {'claude_agent_sdk', 'ui', 'progress', 'task_logger', 'linear_updater', 'client'}, - 'test_qa_loop': {'claude_code_sdk', 'claude_code_sdk.types'}, + 'test_qa_loop': {'claude_code_sdk', 'claude_code_sdk.types', 'claude_agent_sdk', 'claude_agent_sdk.types'}, 'test_spec_pipeline': {'claude_code_sdk', 'claude_code_sdk.types', 'init', 'client', 'review', 'task_logger', 'ui', 'validate_spec'}, 'test_spec_complexity': {'claude_code_sdk', 'claude_code_sdk.types', 'claude_agent_sdk', 'claude_agent_sdk.types'}, 'test_spec_phases': {'claude_code_sdk', 'claude_code_sdk.types', 'claude_agent_sdk', 'graphiti_providers', 'validate_spec', 'client'}, @@ -1000,109 +1024,8 @@ except ImportError: # Module will be available when tests run pass -# Sample React component code -SAMPLE_REACT_COMPONENT = '''import React from 'react'; -import { useState } from 'react'; - -function App() { - const [count, setCount] = useState(0); - - return ( -
-

Hello World

- -
- ); -} - -export default App; -''' - -SAMPLE_REACT_WITH_HOOK = '''import React from 'react'; -import { useState } from 'react'; -import { useAuth } from './hooks/useAuth'; - -function App() { - const [count, setCount] = useState(0); - const { user } = useAuth(); - - return ( -
-

Hello World

- -
- ); -} - -export default App; -''' - -# Sample Python module code -SAMPLE_PYTHON_MODULE = '''"""Sample Python module.""" -import os -from pathlib import Path - -def hello(): - """Say hello.""" - print("Hello") - -def goodbye(): - """Say goodbye.""" - print("Goodbye") - -class Greeter: - """A greeter class.""" - - def greet(self, name: str) -> str: - return f"Hello, {name}" -''' - -SAMPLE_PYTHON_WITH_NEW_IMPORT = '''"""Sample Python module.""" -import os -import logging -from pathlib import Path - -def hello(): - """Say hello.""" - print("Hello") - -def goodbye(): - """Say goodbye.""" - print("Goodbye") - -class Greeter: - """A greeter class.""" - - def greet(self, name: str) -> str: - return f"Hello, {name}" -''' - -SAMPLE_PYTHON_WITH_NEW_FUNCTION = '''"""Sample Python module.""" -import os -from pathlib import Path - -def hello(): - """Say hello.""" - print("Hello") - -def goodbye(): - """Say goodbye.""" - print("Goodbye") - -def new_function(): - """A new function.""" - return 42 - -class Greeter: - """A greeter class.""" - - def greet(self, name: str) -> str: - return f"Hello, {name}" -''' +# Sample data constants moved to test_fixtures.py +# Import from there if needed in test files @pytest.fixture @@ -1152,3 +1075,36 @@ def mock_ai_resolver(): code += "return
Merged
;" return code return AIResolver(ai_call_fn=mock_ai_call) + + +@pytest.fixture +def temp_project(temp_git_repo: Path): + """ + Create a temporary project with mixed language files for testing file tracker. + + Creates: + - src/App.tsx (React component) + - src/utils.py (Python module) + """ + from tests.test_fixtures import SAMPLE_REACT_COMPONENT, SAMPLE_PYTHON_MODULE + + # Create src directory + src_dir = temp_git_repo / "src" + src_dir.mkdir(parents=True, exist_ok=True) + + # Create App.tsx + app_tsx = src_dir / "App.tsx" + app_tsx.write_text(SAMPLE_REACT_COMPONENT) + + # Create utils.py + utils_py = src_dir / "utils.py" + utils_py.write_text(SAMPLE_PYTHON_MODULE) + + # Commit the files + subprocess.run(["git", "add", "."], cwd=temp_git_repo, capture_output=True) + subprocess.run( + ["git", "commit", "-m", "Add source files"], + cwd=temp_git_repo, capture_output=True + ) + + return temp_git_repo diff --git a/tests/test_agent_architecture.py b/tests/test_agent_architecture.py index 0c081b72..c03ce7d8 100644 --- a/tests/test_agent_architecture.py +++ b/tests/test_agent_architecture.py @@ -168,7 +168,7 @@ class TestModuleIntegrity: def test_no_coordinator_imports(self): """Core modules don't import coordinator.""" - for filename in ["run.py", "agent.py"]: + for filename in ["run.py", "core/agent.py"]: filepath = Path(__file__).parent.parent / "auto-claude" / filename content = filepath.read_text() @@ -181,7 +181,7 @@ class TestModuleIntegrity: def test_no_task_tool_imports(self): """Core modules don't import task_tool.""" - for filename in ["run.py", "agent.py"]: + for filename in ["run.py", "core/agent.py"]: filepath = Path(__file__).parent.parent / "auto-claude" / filename content = filepath.read_text() @@ -305,23 +305,24 @@ class TestSubtaskTerminology: def test_implementation_plan_uses_subtask_class(self): """Implementation plan uses Subtask class.""" - impl_plan_path = Path(__file__).parent.parent / "auto-claude" / "implementation_plan.py" + impl_plan_path = Path(__file__).parent.parent / "auto-claude" / "implementation_plan" / "main.py" content = impl_plan_path.read_text() - assert "class Subtask" in content, ( - "implementation_plan.py should define 'class Subtask'" + # Check that it re-exports or imports Subtask and SubtaskStatus + assert "Subtask" in content, ( + "implementation_plan/main.py should reference 'Subtask'" ) assert "SubtaskStatus" in content, ( - "implementation_plan.py should define SubtaskStatus enum" + "implementation_plan/main.py should reference SubtaskStatus enum" ) def test_progress_uses_subtask_terminology(self): """Progress module uses subtask terminology.""" - progress_path = Path(__file__).parent.parent / "auto-claude" / "progress.py" + progress_path = Path(__file__).parent.parent / "auto-claude" / "core" / "progress.py" content = progress_path.read_text() assert "subtask" in content.lower(), ( - "progress.py should use subtask terminology" + "core/progress.py should use subtask terminology" ) diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py new file mode 100644 index 00000000..70b1dd36 --- /dev/null +++ b/tests/test_fixtures.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Test Fixtures - Sample Data Constants +====================================== + +Sample code snippets and data used across multiple test files. +These are separated from conftest.py to allow direct imports. +""" + +# Sample React component code +SAMPLE_REACT_COMPONENT = '''import React from 'react'; +import { useState } from 'react'; + +function App() { + const [count, setCount] = useState(0); + + return ( +
+

Hello World

+ +
+ ); +} + +export default App; +''' + +SAMPLE_REACT_WITH_HOOK = '''import React from 'react'; +import { useState } from 'react'; +import { useAuth } from './hooks/useAuth'; + +function App() { + const [count, setCount] = useState(0); + const { user } = useAuth(); + + return ( +
+

Hello World

+ +
+ ); +} + +export default App; +''' + +# Sample Python module code +SAMPLE_PYTHON_MODULE = '''"""Sample Python module.""" +import os +from pathlib import Path + +def hello(): + """Say hello.""" + print("Hello") + +def goodbye(): + """Say goodbye.""" + print("Goodbye") + +class Greeter: + """A greeter class.""" + + def greet(self, name: str) -> str: + return f"Hello, {name}" +''' + +SAMPLE_PYTHON_WITH_NEW_IMPORT = '''"""Sample Python module.""" +import os +import logging +from pathlib import Path + +def hello(): + """Say hello.""" + print("Hello") + +def goodbye(): + """Say goodbye.""" + print("Goodbye") + +class Greeter: + """A greeter class.""" + + def greet(self, name: str) -> str: + return f"Hello, {name}" +''' + +SAMPLE_PYTHON_WITH_NEW_FUNCTION = '''"""Sample Python module.""" +import os +from pathlib import Path + +def hello(): + """Say hello.""" + print("Hello") + +def goodbye(): + """Say goodbye.""" + print("Goodbye") + +def new_function(): + """A new function.""" + return 42 + +class Greeter: + """A greeter class.""" + + def greet(self, name: str) -> str: + return f"Hello, {name}" +''' diff --git a/tests/test_graphiti.py b/tests/test_graphiti.py index 9a8a81d7..32cf9ed3 100644 --- a/tests/test_graphiti.py +++ b/tests/test_graphiti.py @@ -356,7 +356,7 @@ class TestGraphitiProviders: def test_create_llm_client_missing_openai_key(self): """create_llm_client raises ProviderError when OpenAI key missing.""" - from graphiti_providers import ProviderError + from graphiti_providers import ProviderError, ProviderNotInstalled, create_llm_client with patch.dict(os.environ, { "GRAPHITI_ENABLED": "true", @@ -364,17 +364,18 @@ class TestGraphitiProviders: }, clear=True): config = GraphitiConfig.from_env() - # Only import if graphiti_core is available + # Test raises ProviderError for missing API key, or skip if graphiti-core not installed try: - from graphiti_providers import create_llm_client - with pytest.raises(ProviderError, match="OPENAI_API_KEY"): - create_llm_client(config) - except ImportError: + create_llm_client(config) + pytest.fail("Expected ProviderError to be raised for missing OPENAI_API_KEY") + except ProviderNotInstalled: pytest.skip("graphiti-core not installed") + except ProviderError as e: + assert "OPENAI_API_KEY" in str(e) def test_create_embedder_missing_ollama_model(self): """create_embedder raises ProviderError when Ollama model missing.""" - from graphiti_providers import ProviderError + from graphiti_providers import ProviderError, ProviderNotInstalled, create_embedder with patch.dict(os.environ, { "GRAPHITI_ENABLED": "true", @@ -383,12 +384,14 @@ class TestGraphitiProviders: }, clear=True): config = GraphitiConfig.from_env() + # Test raises ProviderError for missing model config, or skip if graphiti-core not installed try: - from graphiti_providers import create_embedder - with pytest.raises(ProviderError, match="OLLAMA_EMBEDDING_MODEL"): - create_embedder(config) - except ImportError: + create_embedder(config) + pytest.fail("Expected ProviderError to be raised for missing OLLAMA_EMBEDDING_MODEL") + except ProviderNotInstalled: pytest.skip("graphiti-core not installed") + except ProviderError as e: + assert "OLLAMA_EMBEDDING_MODEL" in str(e) def test_embedding_dimensions_lookup(self): """get_expected_embedding_dim returns correct dimensions.""" diff --git a/tests/test_merge_file_tracker.py b/tests/test_merge_file_tracker.py index a5f81101..656fa1a1 100644 --- a/tests/test_merge_file_tracker.py +++ b/tests/test_merge_file_tracker.py @@ -22,8 +22,10 @@ import pytest # Add auto-claude directory to path for imports sys.path.insert(0, str(Path(__file__).parent.parent / "auto-claude")) +# Add tests directory to path for test_fixtures +sys.path.insert(0, str(Path(__file__).parent)) -from conftest import ( +from test_fixtures import ( SAMPLE_PYTHON_MODULE, SAMPLE_PYTHON_WITH_NEW_FUNCTION, SAMPLE_PYTHON_WITH_NEW_IMPORT, diff --git a/tests/test_merge_orchestrator.py b/tests/test_merge_orchestrator.py index 3f0a9114..b6aca437 100644 --- a/tests/test_merge_orchestrator.py +++ b/tests/test_merge_orchestrator.py @@ -17,14 +17,20 @@ Covers: """ import json +import sys from pathlib import Path import pytest +# Add auto-claude directory to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent / "auto-claude")) +# Add tests directory to path for test_fixtures +sys.path.insert(0, str(Path(__file__).parent)) + from merge import MergeOrchestrator from merge.orchestrator import TaskMergeRequest -from conftest import ( +from test_fixtures import ( SAMPLE_PYTHON_MODULE, SAMPLE_PYTHON_WITH_NEW_FUNCTION, SAMPLE_PYTHON_WITH_NEW_IMPORT, @@ -38,7 +44,8 @@ class TestOrchestratorInitialization: """Orchestrator initializes with all components.""" orchestrator = MergeOrchestrator(temp_project) - assert orchestrator.project_dir == temp_project + # Use resolve() to handle symlinks on macOS (/var vs /private/var) + assert orchestrator.project_dir.resolve() == temp_project.resolve() assert orchestrator.analyzer is not None assert orchestrator.conflict_detector is not None assert orchestrator.auto_merger is not None @@ -112,20 +119,21 @@ class TestSingleTaskMerge: """Integration tests for single task merge.""" def test_full_merge_pipeline_single_task(self, temp_project): - """Full pipeline works for single task merge.""" + """Full pipeline works for single task merge (with git-committed changes).""" + import subprocess + orchestrator = MergeOrchestrator(temp_project, dry_run=True) - # Setup: capture baseline and make changes + # Setup: capture baseline files = [temp_project / "src" / "utils.py"] orchestrator.evolution_tracker.capture_baselines("task-001", files, intent="Add new function") - # Simulate task making changes - orchestrator.evolution_tracker.record_modification( - "task-001", - "src/utils.py", - SAMPLE_PYTHON_MODULE, - SAMPLE_PYTHON_WITH_NEW_FUNCTION, - ) + # Create a task branch with actual git changes (the merge pipeline uses git diff main...HEAD) + subprocess.run(["git", "checkout", "-b", "auto-claude/task-001"], cwd=temp_project, capture_output=True) + utils_file = temp_project / "src" / "utils.py" + utils_file.write_text(SAMPLE_PYTHON_WITH_NEW_FUNCTION) + subprocess.run(["git", "add", "."], cwd=temp_project, capture_output=True) + subprocess.run(["git", "commit", "-m", "Add new function"], cwd=temp_project, capture_output=True) # Execute merge - provide worktree_path to avoid lookup report = orchestrator.merge_task("task-001", worktree_path=temp_project) @@ -133,6 +141,7 @@ class TestSingleTaskMerge: # Verify results assert report.success is True assert "task-001" in report.tasks_merged + # The pipeline should detect and process the modified file assert report.stats.files_processed >= 1 diff --git a/tests/test_merge_parallel.py b/tests/test_merge_parallel.py index 20fb3893..109d3a8d 100644 --- a/tests/test_merge_parallel.py +++ b/tests/test_merge_parallel.py @@ -98,10 +98,10 @@ class TestParallelMergeDataclasses: class TestParallelMergeRunner: """Tests for the parallel merge runner.""" - def test_run_parallel_merges_empty_list(self, temp_project): + def test_run_parallel_merges_empty_list(self, project_dir): """Running with empty task list returns empty results.""" import asyncio - results = asyncio.run(_run_parallel_merges([], temp_project)) + results = asyncio.run(_run_parallel_merges([], project_dir)) assert results == [] def test_parallel_merge_task_with_data(self): diff --git a/tests/test_merge_semantic_analyzer.py b/tests/test_merge_semantic_analyzer.py index 60bf2538..6afc049b 100644 --- a/tests/test_merge_semantic_analyzer.py +++ b/tests/test_merge_semantic_analyzer.py @@ -20,9 +20,11 @@ import pytest # Add auto-claude directory to path for imports sys.path.insert(0, str(Path(__file__).parent.parent / "auto-claude")) +# Add tests directory to path for test_fixtures +sys.path.insert(0, str(Path(__file__).parent)) from merge import ChangeType -from conftest import ( +from test_fixtures import ( SAMPLE_PYTHON_MODULE, SAMPLE_PYTHON_WITH_NEW_IMPORT, SAMPLE_PYTHON_WITH_NEW_FUNCTION, diff --git a/tests/test_qa_loop.py b/tests/test_qa_loop.py index 3c61db68..269aabe9 100644 --- a/tests/test_qa_loop.py +++ b/tests/test_qa_loop.py @@ -21,21 +21,31 @@ _original_modules = {} _mocked_module_names = [ 'claude_code_sdk', 'claude_code_sdk.types', + 'claude_agent_sdk', + 'claude_agent_sdk.types', ] for name in _mocked_module_names: if name in sys.modules: _original_modules[name] = sys.modules[name] -# Mock claude_code_sdk and its submodules before importing qa_loop -# The SDK isn't available in the test environment -mock_sdk = MagicMock() -mock_sdk.ClaudeSDKClient = MagicMock() -mock_sdk.ClaudeCodeOptions = MagicMock() -mock_types = MagicMock() -mock_types.HookMatcher = MagicMock() -sys.modules['claude_code_sdk'] = mock_sdk -sys.modules['claude_code_sdk.types'] = mock_types +# Mock claude_code_sdk and claude_agent_sdk before importing qa_loop +# The SDKs aren't available in the test environment +mock_code_sdk = MagicMock() +mock_code_sdk.ClaudeSDKClient = MagicMock() +mock_code_sdk.ClaudeCodeOptions = MagicMock() +mock_code_types = MagicMock() +mock_code_types.HookMatcher = MagicMock() +sys.modules['claude_code_sdk'] = mock_code_sdk +sys.modules['claude_code_sdk.types'] = mock_code_types + +mock_agent_sdk = MagicMock() +mock_agent_sdk.ClaudeSDKClient = MagicMock() +mock_agent_sdk.ClaudeCodeOptions = MagicMock() +mock_agent_types = MagicMock() +mock_agent_types.HookMatcher = MagicMock() +sys.modules['claude_agent_sdk'] = mock_agent_sdk +sys.modules['claude_agent_sdk.types'] = mock_agent_types from qa_loop import ( load_implementation_plan, diff --git a/tests/test_spec_pipeline.py b/tests/test_spec_pipeline.py index 8b9c603d..7b8d0bd6 100644 --- a/tests/test_spec_pipeline.py +++ b/tests/test_spec_pipeline.py @@ -115,7 +115,7 @@ class TestGetSpecsDir: def test_calls_init_auto_claude_dir(self, temp_dir: Path): """Initializes auto-claude directory.""" - with patch('spec.pipeline.init_auto_claude_dir') as mock_init: + with patch('spec.pipeline.models.init_auto_claude_dir') as mock_init: mock_init.return_value = (temp_dir / ".auto-claude", False) get_specs_dir(temp_dir)